Fetch All Data from a Paginated REST API Using Next Links or Pages

const fetch = require('node-fetch'); // or use native fetch in browser/Node 18+

async function fetchAllPaginatedData(baseUrl, initialParams = {}) {
  let allData = [];
  let nextUrl = baseUrl;

  while (nextUrl) {
    try {
      // Construct URL with parameters for the first request
      const currentUrl = nextUrl === baseUrl ?
        `${baseUrl}?${new URLSearchParams(initialParams).toString()}` :
        nextUrl;

      console.log(`Fetching from: ${currentUrl}`);
      const response = await fetch(currentUrl);

      if (!response.ok) {
        const errorText = await response.text();
        throw new Error(`API request failed: ${response.status} - ${errorText}`);
      }

      const pageData = await response.json();

      // Assuming API returns an array of items directly or in a 'data' property
      const items = pageData.data || pageData;
      if (Array.isArray(items)) {
        allData = allData.concat(items);
      } else {
        console.warn('Page data is not an array, check API response structure.');
        break; // Stop if unexpected data format
      }

      // Determine next URL based on common pagination patterns
      // 1. Link header (e.g., rel="next")
      const linkHeader = response.headers.get('link');
      if (linkHeader) {
        const nextMatch = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
        nextUrl = nextMatch ? nextMatch[1] : null;
      }
      // 2. 'next' property in the response body (e.g., { data: [...], next: "https://api.example.com/items?page=2" })
      else if (pageData.next) {
        nextUrl = pageData.next;
      }
      // 3. Page number increment (if API uses `page` or `offset` param, needs more complex logic)
      // This example focuses on 'next' links/properties for simplicity and robustness.
      else {
        nextUrl = null; // No more pages indicated
      }

      // Add a small delay to avoid hitting rate limits if fetching many pages quickly
      // await new Promise(resolve => setTimeout(resolve, 200));

    } catch (error) {
      console.error('Error fetching paginated data:', error);
      nextUrl = null; // Stop on error
      throw error;
    }
  }
  return allData;
}

// Usage example:
// const API_BASE_URL = 'https://api.example.com/v1/users';
// const initialQuery = { limit: 100 }; // Example: initial query parameters

// (async () => {
//   try {
//     const allUsers = await fetchAllPaginatedData(API_BASE_URL, initialQuery);
//     console.log(`Fetched ${allUsers.length} users in total.`);
//     // console.log(allUsers);
//   } catch (error) {
//     console.error('Failed to retrieve all paginated data:', error);
//   }
// })();