JAVASCRIPT
Fetch All Paginated API Data Sequentially
Efficiently retrieve all available data from a paginated API endpoint by iteratively fetching subsequent pages until no more data is returned, ensuring comprehensive data collection.
async function fetchAllPaginatedData(initialUrl, pageParam = 'page', dataKey = 'results') {
let allData = [];
let nextUrl = initialUrl;
while (nextUrl) {
try {
const response = await fetch(nextUrl);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const data = await response.json();
// Assuming data is an object with a dataKey array and a 'next' URL field
// Adjust 'next' and 'dataKey' based on actual API response structure
allData = allData.concat(data[dataKey] || data.items || []);
nextUrl = data.next || null; // Or `data.next_page_url`, `data.links.next`, etc.
// If API uses page numbers, construct next URL
if (!nextUrl && data.page && data.total_pages && data.page < data.total_pages) {
const urlObj = new URL(initialUrl);
const currentPage = parseInt(urlObj.searchParams.get(pageParam) || '1');
urlObj.searchParams.set(pageParam, currentPage + 1);
nextUrl = urlObj.toString();
} else if (!nextUrl && data.page && data.total_pages && data.page >= data.total_pages) {
nextUrl = null; // Reached last page for page-number based pagination
}
} catch (error) {
console.error('Error fetching paginated data:', error);
// Optionally throw or return partial data on error
nextUrl = null; // Stop fetching on error
}
}
return allData;
}
// Example usage (replace with your actual API endpoint and parameters)
// fetchAllPaginatedData('https://api.example.com/v1/products?limit=10', 'page', 'products')
// .then(products => {
// console.log('Fetched all products:', products.length);
// console.log(products);
// })
// .catch(error => console.error('Failed to fetch all products:', error));
// fetchAllPaginatedData('https://swapi.dev/api/people/', 'page', 'results')
// .then(people => {
// console.log('Fetched all Star Wars people:', people.length);
// console.log(people);
// })
// .catch(error => console.error('Failed to fetch all people:', error));
How it works: This JavaScript snippet demonstrates how to fetch all data from an API that implements pagination. It uses an asynchronous `while` loop to repeatedly make requests, updating the `nextUrl` based on the API's response (e.g., a `next` link or incrementing a `page` number parameter). It aggregates the data from each page into a single array until no further pages are indicated, providing a complete dataset. Error handling is included to gracefully manage network issues or API errors during the fetching process.