PYTHON
Efficiently Paginating Through Large API Datasets in Python
Learn to iterate and fetch all pages of a large dataset from a paginated REST API in Python, handling common pagination patterns like 'next' links or page numbers.
import requests
import time
def fetch_all_paginated_data(base_url, headers=None, params=None,
page_param_name='page', per_page_param_name='per_page',
initial_page=1, page_size=100,
data_key='items', next_page_key=None):
"""
Fetches all data from a paginated API endpoint.
Args:
base_url (str): The base URL of the API endpoint.
headers (dict, optional): HTTP headers to send with the request. Defaults to None.
params (dict, optional): Initial query parameters. Defaults to None.
page_param_name (str): The name of the query parameter for the current page number.
Defaults to 'page'.
per_page_param_name (str): The name of the query parameter for items per page.
Defaults to 'per_page'.
initial_page (int): The starting page number. Defaults to 1.
page_size (int): The number of items to request per page. Defaults to 100.
data_key (str): The key in the API response JSON that contains the list of items.
Defaults to 'items'.
next_page_key (str, optional): If the API provides a 'next' link directly in the response
(e.g., HATEOAS style), specify its key. If None,
page numbers are used. Defaults to None.
Returns:
list: A combined list of all items fetched from all pages.
"""
all_data = []
current_page = initial_page
current_url = base_url
# Prepare initial parameters
current_params = params.copy() if params else {}
current_params[per_page_param_name] = page_size
print(f"Starting pagination from {base_url}...")
while True:
try:
if next_page_key is None:
# Pagination by page number
current_params[page_param_name] = current_page
print(f"Fetching page {current_page} with {per_page_param_name}={page_size}...")
response = requests.get(base_url, headers=headers, params=current_params)
else:
# Pagination by 'next' URL from previous response
print(f"Fetching from URL: {current_url}...")
response = requests.get(current_url, headers=headers)
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
page_data = response.json()
items = page_data.get(data_key, [])
if not items:
print("No more items found or data_key is missing. Exiting pagination.")
break # No items, or empty list means end of data
all_data.extend(items)
print(f"Fetched {len(items)} items from current page. Total fetched: {len(all_data)}")
if next_page_key:
current_url = page_data.get(next_page_key)
if not current_url:
print("No next page URL found. Exiting pagination.")
break # No next link, end of data
else:
# For page number pagination, check if the number of items fetched
# is less than page_size. If so, it's likely the last page.
if len(items) < page_size:
print(f"Less than {page_size} items on page {current_page}. Assuming last page.")
break
current_page += 1
# Add a small delay to avoid hitting rate limits
time.sleep(0.1)
except requests.exceptions.RequestException as e:
print(f"Request failed: {e}")
break
except ValueError as e: # JSONDecodeError inherits from ValueError
print(f"Failed to parse JSON response: {e}")
print(f"Response content: {response.text[:200]}...") # Print first 200 chars of problematic response
break
return all_data
# --- Example Usage (simulated API) ---
# Imagine a simple API that returns data like:
# GET /api/products?page=1&per_page=10
# {
# "items": [ {...}, {...} ],
# "total_items": 123,
# "total_pages": 13,
# "next_page_url": "/api/products?page=2&per_page=10" // if using next_page_key
# }
# Mock server for demonstration (run separately, or replace with actual API endpoint)
# class MockApiServer:
# def __init__(self, total_items, page_size):
# self.total_items = total_items
# self.page_size = page_size
# self.items = [{"id": i, "name": f"Product {i}"} for i in range(1, total_items + 1)]
#
# def get_products(self, page, per_page):
# start_index = (page - 1) * per_page
# end_index = start_index + per_page
# current_page_items = self.items[start_index:end_index]
# total_pages = (self.total_items + per_page - 1) // per_page
# next_page_url = f"http://localhost:5000/api/products?page={page+1}&per_page={per_page}" if page < total_pages else None
# return {
# "items": current_page_items,
# "total_items": self.total_items,
# "total_pages": total_pages,
# "next": next_page_url # For next_page_key example
# }
#
# def run_flask(self):
# from flask import Flask, request, jsonify
# app = Flask(__name__)
#
# @app.route('/api/products')
# def products():
# page = int(request.args.get('page', 1))
# per_page = int(request.args.get('per_page', self.page_size))
# response_data = self.get_products(page, per_page)
# return jsonify(response_data)
#
# app.run(port=5000, debug=False)
#
# # To run the mock server:
# # mock_server = MockApiServer(total_items=255, page_size=50)
# # mock_server.run_flask() # Run this in a separate terminal or thread
#
# # Example 1: Pagination by page number
print("
--- Paginating by page number ---")
# Replace with your actual API endpoint if not using mock server
# For demonstration, assume a mock API at localhost:5000
all_products_page_num = fetch_all_paginated_data(
"http://localhost:5000/api/products",
data_key='items', # Key where actual item list is
page_size=50
)
print(f"
Total products fetched (page number): {len(all_products_page_num)}")
# Example: print(all_products_page_num[:5]) # Print first 5 items
# Example 2: Pagination by 'next' URL (HATEOAS style)
# Ensure your mock server or actual API supports returning a 'next' link.
print("
--- Paginating by 'next' URL ---")
all_products_next_url = fetch_all_paginated_data(
"http://localhost:5000/api/products",
data_key='items',
next_page_key='next', # Key in the response that holds the URL for the next page
page_size=50 # still useful for initial request, can be ignored by API if next_page_key is used
)
print(f"
Total products fetched (next URL): {len(all_products_next_url)}")
How it works: This Python snippet provides a flexible function, `fetch_all_paginated_data`, designed to retrieve all available data from a REST API that uses pagination. It supports two common pagination strategies: using incrementing `page` numbers (and `per_page` limits) and following a `next` URL provided within the API response (HATEOAS style). The function iteratively makes API requests, collecting items from each page until no more items are returned or no `next` URL is available. It includes error handling for network issues and JSON parsing, along with a small delay between requests to help prevent hitting API rate limits. This is crucial for applications that need to process large datasets from external services.