Programmatically Fetch All Pages from an Offset/Limit API

const API_PAGINATED_BASE_URL = 'https://jsonplaceholder.typicode.com/comments'; // Example API

async function fetchAllPaginatedData(baseUrl, limit = 10, initialOffset = 0) {
    let allData = [];
    let currentOffset = initialOffset;
    let hasMoreData = true;

    while (hasMoreData) {
        const url = new URL(baseUrl);
        url.searchParams.append('_limit', limit); // Example: API uses _limit
        url.searchParams.append('_start', currentOffset); // Example: API uses _start for offset

        console.log(`Fetching: ${url.toString()}`);
        const response = await fetch(url.toString());
        if (!response.ok) {
            throw new Error(`API error: ${response.status} ${response.statusText}`);
        }

        const pageData = await response.json();
        allData = allData.concat(pageData);

        // Determine if there's more data based on the response length
        // This assumes API returns less than 'limit' items if it's the last page
        hasMoreData = pageData.length === limit;
        currentOffset += limit;

        // Optional: Add a small delay to avoid hitting rate limits too quickly
        // await new Promise(resolve => setTimeout(resolve, 100));
    }

    return allData;
}

// Usage Example:
/*
(async () => {
    try {
        // Fetch first 2 pages (assuming 10 items per page and API has more than 10)
        const allComments = await fetchAllPaginatedData(API_PAGINATED_BASE_URL, 10);
        console.log(`Total comments fetched: ${allComments.length}`);
        console.log('First few comments:', allComments.slice(0, 5));
    } catch (err) {
        console.error('Failed to fetch all paginated data:', err.message);
    }
})();
*/