Stream Large API Responses for Progressive Processing

async function streamLargeApiResponse(url, processChunk) {
  try {
    const response = await fetch(url);
    if (!response.ok) {
      throw new Error(`HTTP error! Status: ${response.status}`);
    }

    const reader = response.body.getReader();
    const decoder = new TextDecoder('utf-8');
    let result = '';

    while (true) {
      const { done, value } = await reader.read();
      if (done) break;
      
      const chunk = decoder.decode(value, { stream: true });
      result += chunk;
      
      // Process the accumulated data or just the chunk
      processChunk(chunk, result); 
    }

    // Optionally process the final complete result
    // console.log('Final streamed data:', result);
    return result;

  } catch (error) {
    console.error('Error streaming API response:', error);
    throw error;
  }
}

// Usage example (simulating an endpoint that returns data in chunks):
// Note: Most real-world APIs return complete JSON. This is more useful for
// server-sent events (SSE) or custom chunked responses.
// For a true stream, a server needs to implement chunked transfer encoding.

const CHUNKED_API_URL = 'https://httpbin.org/stream/5'; // Example returning 5 lines

streamLargeApiResponse(CHUNKED_API_URL, (chunk, accumulated) => {
  console.log('Received chunk:', chunk.trim());
  // You can parse incomplete JSON here if the server guarantees valid JSON per chunk
  // or accumulate until you have a full object/line.
})
.then(finalResult => {
  console.log('
Streaming complete. Full response (if applicable):
', finalResult.trim());
  // For httpbin.org/stream, it's newline-delimited JSON objects
  finalResult.trim().split('
').forEach(line => {
    try { console.log('Parsed line:', JSON.parse(line)); }
    catch (e) { /* ignore incomplete lines */ }
  });
})
.catch(error => console.error('Streaming failed:', error));

// Another example: Processing a stream of newline-delimited JSON objects
async function processJsonStream(url) {
  await streamLargeApiResponse(url, (chunk, accumulated) => {
    const lines = accumulated.split('
');
    // Process all full lines received so far
    while (lines.length > 1) {
      const line = lines.shift();
      if (line) {
        try {
          const data = JSON.parse(line);
          console.log('Processed JSON object:', data);
        } catch (e) {
          console.error('Could not parse JSON line:', line, e);
        }
      }
    }
    // Update accumulated to keep only the potentially incomplete last line
    return lines[0] || ''; 
  });
}

// Example for newline-delimited JSON stream:
// processJsonStream(CHUNKED_API_URL);