Fetching All Pages with Cursor-Based API Pagination

/**
 * Fetches all pages from an API endpoint using cursor-based pagination.
 * Assumes the API response structure is { data: [], next_cursor: 'abc' | null }
 *
 * @param {string} baseUrl - The base URL of the API endpoint (e.g., 'https://api.example.com/items').
 * @param {object} initialQueryParams - Initial query parameters (e.g., { limit: 100, status: 'active' }).
 * @returns {Promise<Array<any>>} - A promise that resolves to an array containing all fetched items.
 */
async function fetchAllCursorPaginatedData(baseUrl, initialQueryParams = {}) {
  let allItems = [];
  let nextCursor = null;
  let hasMorePages = true;

  while (hasMorePages) {
    const url = new URL(baseUrl);
    const params = { ...initialQueryParams };

    if (nextCursor) {
      params.cursor = nextCursor; // Add the cursor for subsequent requests
    }

    Object.keys(params).forEach(key => url.searchParams.append(key, params[key]));

    try {
      const response = await fetch(url.toString());
      if (!response.ok) {
        throw new Error(`HTTP error! status: ${response.status}`);
      }
      const result = await response.json();

      if (result && Array.isArray(result.data)) {
        allItems = allItems.concat(result.data);
        nextCursor = result.next_cursor || null; // API might return null or undefined
        hasMorePages = !!nextCursor; // If nextCursor is null/undefined, no more pages
      } else {
        console.warn('API response structure unexpected:', result);
        hasMorePages = false; // Stop if structure is not as expected
      }
    } catch (error) {
      console.error(`Error fetching paginated data from ${url.toString()}:`, error);
      hasMorePages = false; // Stop on error
      throw error; // Re-throw to inform the caller
    }
  }
  return allItems;
}

// Example Usage (assuming a mock API that returns data and a next_cursor):
/*
// Mock API setup for demonstration
let mockData = Array.from({ length: 250 }, (_, i) => ({ id: i + 1, name: `Item ${i + 1}` }));
function mockApiCall(cursor, limit) {
    const start = cursor ? parseInt(cursor) : 0;
    const end = Math.min(start + limit, mockData.length);
    const data = mockData.slice(start, end);
    const newCursor = end < mockData.length ? end.toString() : null;
    return Promise.resolve({ data, next_cursor: newCursor });
}

// Override global fetch for testing (do NOT do this in production)
const originalFetch = global.fetch;
global.fetch = async (input, init) => {
    const urlObj = new URL(input);
    if (urlObj.hostname === 'mockapi.example.com') {
        const cursor = urlObj.searchParams.get('cursor');
        const limit = parseInt(urlObj.searchParams.get('limit') || '50'); // Default limit
        console.log(`Mock API: Fetching with cursor=${cursor}, limit=${limit}`);
        return new Response(JSON.stringify(await mockApiCall(cursor, limit)), {
            status: 200,
            headers: { 'Content-Type': 'application/json' }
        });
    }
    return originalFetch(input, init); // Fallback to actual fetch for other URLs
};


(async () => {
  try {
    console.log('Fetching all items with cursor pagination...');
    const allItems = await fetchAllCursorPaginatedData('https://mockapi.example.com/items', { limit: 50, status: 'active' });
    console.log(`Total items fetched: ${allItems.length}`);
    // console.log('First 5 items:', allItems.slice(0, 5));
    // console.log('Last 5 items:', allItems.slice(-5));
  } catch (error) {
    console.error('Failed to fetch all paginated data:', error);
  } finally {
    global.fetch = originalFetch; // Restore original fetch
  }
})();
*/