Fetching All Data from Cursor-Paginated API

const axios = require('axios');

/**
 * Fetches all items from a cursor-paginated API endpoint.
 * Assumes the API response structure has:
 * {
 *   data: [...items],
 *   pagination: {
 *     next_cursor: "...", // or next_page_token, etc.
 *     has_more: true/false // or similar indicator
 *   }
 * }
 * @param {string} url The base URL of the API endpoint.
 * @param {object} initialParams Initial query parameters (e.g., limit, filters).
 * @param {string} cursorKey The name of the cursor parameter (e.g., 'cursor', 'next_token').
 * @returns {Promise<Array>} A promise that resolves to an array of all fetched items.
 */
async function fetchAllCursorPaginatedData(url, initialParams = {}, cursorKey = 'next_cursor') {
    let allItems = [];
    let currentCursor = null;
    let hasMore = true;

    while (hasMore) {
        const params = { ...initialParams };
        if (currentCursor) {
            params[cursorKey] = currentCursor;
        }

        try {
            const response = await axios.get(url, { params });
            const { data, pagination } = response.data;

            if (Array.isArray(data)) {
                allItems = allItems.concat(data);
            }

            if (pagination && pagination[cursorKey]) {
                currentCursor = pagination[cursorKey];
            } else {
                currentCursor = null; // No more cursor
            }

            // Check for has_more or deduce from cursor presence
            hasMore = (pagination && pagination.has_more !== undefined)
                      ? pagination.has_more
                      : !!currentCursor; // If cursor exists, assume more pages

            console.log(`Fetched ${data.length} items. Total: ${allItems.length}. Next cursor: ${currentCursor || 'None'}`);

        } catch (error) {
            console.error('Error fetching paginated data:', error.response ? error.response.data : error.message);
            throw error; // Re-throw to propagate the error
        }
    }
    return allItems;
}

// --- Example Usage (simulated API) ---
async function simulateApiCall(url, params) {
    // Simulate a small delay for network latency
    await new Promise(resolve => setTimeout(resolve, 100));

    let cursor = params.next_cursor || null;
    let limit = parseInt(params.limit || 5);
    let items = Array.from({ length: 20 }, (_, i) => ({ id: i + 1, name: `Item ${i + 1}` }));

    let startIndex = 0;
    if (cursor) {
        const cursorIndex = items.findIndex(item => item.id.toString() === cursor);
        if (cursorIndex !== -1) {
            startIndex = cursorIndex + 1;
        }
    }

    const paginatedItems = items.slice(startIndex, startIndex + limit);
    const nextCursor = (startIndex + limit < items.length) ? items[startIndex + limit -1].id.toString() : null; // Get ID of last item in current page

    return {
        data: paginatedItems,
        pagination: {
            next_cursor: nextCursor,
            has_more: nextCursor !== null
        }
    };
}

async function runExample() {
    console.log("Starting data fetch...");
    try {
        // Replace with your actual API endpoint and parameters
        // For this example, we'll use a simulated API call
        const allItems = await fetchAllCursorPaginatedData(
            'http://simulated.api/items', // Actual URL here
            { limit: 5 },
            'next_cursor'
        );
        console.log('
All fetched items:', allItems);
        console.log('Total items fetched:', allItems.length);
    } catch (error) {
        console.error('Failed to fetch all items.');
    }
}

if (require.main === module) {
    // Mock axios to use our simulated API for the example
    axios.get = async (url, config) => {
        console.log(`Simulated API call to ${url} with params:`, config.params);
        const responseData = await simulateApiCall(url, config.params);
        return { data: responseData, status: 200 };
    };
    runExample();
}