Efficiently Handling API Pagination for Large Datasets

async function fetchAllPaginatedData(baseUrl, pageSize = 10, initialPage = 1) {
  let allItems = [];
  let currentPage = initialPage;
  let hasMore = true;

  while (hasMore) {
    try {
      const url = `${baseUrl}?page=${currentPage}&pageSize=${pageSize}`;
      const response = await fetch(url);

      if (!response.ok) {
        const errorData = await response.json();
        throw new Error(errorData.message || `HTTP error! Status: ${response.status}`);
      }

      const { data, totalPages, currentPage: returnedPage } = await response.json(); // Adjust to your API's response structure
      
      allItems = allItems.concat(data);
      currentPage++;
      
      // Determine if there are more pages based on API response (adjust logic as needed)
      hasMore = returnedPage < totalPages; // Common pattern
      // Or check if data array is empty: hasMore = data.length === pageSize;
      // Or check for a 'next_page_url' property
      
    } catch (error) {
      console.error(`Error fetching page ${currentPage}:`, error);
      hasMore = false; // Stop fetching on error
      throw error;
    }
  }

  return allItems;
}

// Example Usage (assuming API returns { data: [...], totalPages: N, currentPage: M })
// async function loadAllItems() {
//   try {
//     const allProducts = await fetchAllPaginatedData('https://api.example.com/products', 20);
//     console.log('All products:', allProducts);
//   } catch (err) {
//     console.error('Failed to load all products:', err);
//   }
// }
// loadAllItems();