PYTHON

Efficiently Paginating Through Large API Datasets in Python

Learn to iterate and fetch all pages of a large dataset from a paginated REST API in Python, handling common pagination patterns like 'next' links or page numbers.

import requests
import time

def fetch_all_paginated_data(base_url, headers=None, params=None,
                                 page_param_name='page', per_page_param_name='per_page',
                                 initial_page=1, page_size=100,
                                 data_key='items', next_page_key=None):
    """
    Fetches all data from a paginated API endpoint.

    Args:
        base_url (str): The base URL of the API endpoint.
        headers (dict, optional): HTTP headers to send with the request. Defaults to None.
        params (dict, optional): Initial query parameters. Defaults to None.
        page_param_name (str): The name of the query parameter for the current page number.
                               Defaults to 'page'.
        per_page_param_name (str): The name of the query parameter for items per page.
                                   Defaults to 'per_page'.
        initial_page (int): The starting page number. Defaults to 1.
        page_size (int): The number of items to request per page. Defaults to 100.
        data_key (str): The key in the API response JSON that contains the list of items.
                        Defaults to 'items'.
        next_page_key (str, optional): If the API provides a 'next' link directly in the response
                                       (e.g., HATEOAS style), specify its key. If None,
                                       page numbers are used. Defaults to None.

    Returns:
        list: A combined list of all items fetched from all pages.
    """
    all_data = []
    current_page = initial_page
    current_url = base_url

    # Prepare initial parameters
    current_params = params.copy() if params else {}
    current_params[per_page_param_name] = page_size

    print(f"Starting pagination from {base_url}...")

    while True:
        try:
            if next_page_key is None:
                # Pagination by page number
                current_params[page_param_name] = current_page
                print(f"Fetching page {current_page} with {per_page_param_name}={page_size}...")
                response = requests.get(base_url, headers=headers, params=current_params)
            else:
                # Pagination by 'next' URL from previous response
                print(f"Fetching from URL: {current_url}...")
                response = requests.get(current_url, headers=headers)

            response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
            page_data = response.json()

            items = page_data.get(data_key, [])
            if not items:
                print("No more items found or data_key is missing. Exiting pagination.")
                break # No items, or empty list means end of data

            all_data.extend(items)
            print(f"Fetched {len(items)} items from current page. Total fetched: {len(all_data)}")

            if next_page_key:
                current_url = page_data.get(next_page_key)
                if not current_url:
                    print("No next page URL found. Exiting pagination.")
                    break # No next link, end of data
            else:
                # For page number pagination, check if the number of items fetched
                # is less than page_size. If so, it's likely the last page.
                if len(items) < page_size:
                    print(f"Less than {page_size} items on page {current_page}. Assuming last page.")
                    break
                current_page += 1

            # Add a small delay to avoid hitting rate limits
            time.sleep(0.1)

        except requests.exceptions.RequestException as e:
            print(f"Request failed: {e}")
            break
        except ValueError as e: # JSONDecodeError inherits from ValueError
            print(f"Failed to parse JSON response: {e}")
            print(f"Response content: {response.text[:200]}...") # Print first 200 chars of problematic response
            break

    return all_data

# --- Example Usage (simulated API) ---
# Imagine a simple API that returns data like:
# GET /api/products?page=1&per_page=10
# {
#   "items": [ {...}, {...} ],
#   "total_items": 123,
#   "total_pages": 13,
#   "next_page_url": "/api/products?page=2&per_page=10" // if using next_page_key
# }

# Mock server for demonstration (run separately, or replace with actual API endpoint)
# class MockApiServer:
#     def __init__(self, total_items, page_size):
#         self.total_items = total_items
#         self.page_size = page_size
#         self.items = [{"id": i, "name": f"Product {i}"} for i in range(1, total_items + 1)]
#
#     def get_products(self, page, per_page):
#         start_index = (page - 1) * per_page
#         end_index = start_index + per_page
#         current_page_items = self.items[start_index:end_index]
#         total_pages = (self.total_items + per_page - 1) // per_page
#         next_page_url = f"http://localhost:5000/api/products?page={page+1}&per_page={per_page}" if page < total_pages else None
#         return {
#             "items": current_page_items,
#             "total_items": self.total_items,
#             "total_pages": total_pages,
#             "next": next_page_url # For next_page_key example
#         }
#
#     def run_flask(self):
#         from flask import Flask, request, jsonify
#         app = Flask(__name__)
#
#         @app.route('/api/products')
#         def products():
#             page = int(request.args.get('page', 1))
#             per_page = int(request.args.get('per_page', self.page_size))
#             response_data = self.get_products(page, per_page)
#             return jsonify(response_data)
#
#         app.run(port=5000, debug=False)
#
# # To run the mock server:
# # mock_server = MockApiServer(total_items=255, page_size=50)
# # mock_server.run_flask() # Run this in a separate terminal or thread
#
# # Example 1: Pagination by page number
print("
--- Paginating by page number ---")
# Replace with your actual API endpoint if not using mock server
# For demonstration, assume a mock API at localhost:5000
all_products_page_num = fetch_all_paginated_data(
    "http://localhost:5000/api/products",
    data_key='items', # Key where actual item list is
    page_size=50
)
print(f"
Total products fetched (page number): {len(all_products_page_num)}")
# Example: print(all_products_page_num[:5]) # Print first 5 items

# Example 2: Pagination by 'next' URL (HATEOAS style)
# Ensure your mock server or actual API supports returning a 'next' link.
print("
--- Paginating by 'next' URL ---")
all_products_next_url = fetch_all_paginated_data(
    "http://localhost:5000/api/products",
    data_key='items',
    next_page_key='next', # Key in the response that holds the URL for the next page
    page_size=50 # still useful for initial request, can be ignored by API if next_page_key is used
)
print(f"
Total products fetched (next URL): {len(all_products_next_url)}")
How it works: This Python snippet provides a flexible function, `fetch_all_paginated_data`, designed to retrieve all available data from a REST API that uses pagination. It supports two common pagination strategies: using incrementing `page` numbers (and `per_page` limits) and following a `next` URL provided within the API response (HATEOAS style). The function iteratively makes API requests, collecting items from each page until no more items are returned or no `next` URL is available. It includes error handling for network issues and JSON parsing, along with a small delay between requests to help prevent hitting API rate limits. This is crucial for applications that need to process large datasets from external services.

Need help integrating this into your project?

Our team of expert developers can help you build your custom application from scratch.

Hire DigitalCodeLabs