Efficiently Stream Large API Responses in Node.js with `fetch`

import fetch from 'node-fetch'; // For Node.js, install: npm install node-fetch
import { Writable } from 'stream';

async function streamLargeApiResponse(url, processChunkCallback) {
    try {
        const response = await fetch(url);

        if (!response.ok) {
            const errorText = await response.text();
            throw new Error(`API Error: ${response.status} - ${errorText}`);
        }

        const reader = response.body.getReader();
        const decoder = new TextDecoder();
        let receivedLength = 0;
        let chunks = [];

        // A custom Writable stream to process data as it arrives
        const processingStream = new Writable({
            write(chunk, encoding, callback) {
                processChunkCallback(decoder.decode(chunk, { stream: true }));
                callback();
            }
        });

        while (true) {
            const { done, value } = await reader.read();

            if (done) {
                // Process any remaining buffered text if necessary (e.g., if JSON objects were split)
                processChunkCallback(decoder.decode()); // Flush the decoder
                console.log('Stream finished.');
                break;
            }

            receivedLength += value.length;
            // Alternatively, pipe directly if you have a full stream processor
            // value is a Uint8Array, decoder.decode converts it to string
            processingStream.write(value);
        }
    } catch (error) {
        console.error('Error streaming API response:', error);
        throw error; // Re-throw for caller to handle
    }
}

// Example Usage:
// This example simulates processing line-by-line or partial JSON.
// For full JSON parsing of streamed data, a more sophisticated parser (like `stream-json`)
// would be needed to handle chunks that split JSON tokens.
// For simpler data (e.g., newline-delimited JSON), this approach works well.

// async function main() {
//     console.log('Starting stream example...');
//     let totalCharacters = 0;
//     await streamLargeApiResponse(
//         'https://jsonplaceholder.typicode.com/comments', // An API that returns a decent amount of JSON
//         (chunkText) => {
//             // In a real scenario, you might accumulate `chunkText` and
//             // then parse complete JSON objects if the API sends them incrementally,
//             // or process line-by-line for newline-delimited data.
//             // For this example, we just count characters and log a part.
//             totalCharacters += chunkText.length;
//             // console.log('Received chunk (first 50 chars):', chunkText.substring(0, 50));
//         }
//     );
//     console.log('Total characters received from stream:', totalCharacters);
// }

// main().catch(err => console.error('Main process failed:', err));