Fetching Paginated API Data Using Python Requests

import requests

def fetch_paginated_data(base_url, initial_params=None, max_pages=5):
    all_items = []
    params = initial_params if initial_params else {'page': 1, 'per_page': 10}
    page_num = params.get('page', 1)

    for _ in range(max_pages):
        print(f"Fetching page {page_num} from {base_url} with params {params}")
        try:
            response = requests.get(base_url, params=params)
            response.raise_for_status()  # Raise an HTTPError for bad responses (4xx or 5xx)
            data = response.json()

            # Assuming the API returns a list of items directly or under a 'results' key
            items = data.get('results', data) 
            if not items: # No more items, break loop
                break
            all_items.extend(items)

            # Update for next page. This logic depends on the API's pagination scheme.
            # Example 1: API uses 'next' link (GitHub style)
            # if 'next' in response.links:
            #     base_url = response.links['next']['url']
            #     params = {}
            # else:
            #     break
            
            # Example 2: API uses 'page' parameter
            if 'next_page' in data: # Assuming a 'next_page' field in response
                page_num = data['next_page']
                params['page'] = page_num
            else:
                # Increment page number if no explicit next_page link/field
                # And assuming total_pages or a similar indicator is not available or handled
                page_num += 1
                params['page'] = page_num
                if len(items) < params['per_page']: # If current page has fewer items than 'per_page', assume last page
                    break

        except requests.exceptions.HTTPError as errh:
            print ("Http Error:",errh)
            break
        except requests.exceptions.ConnectionError as errc:
            print ("Error Connecting:",errc)
            break
        except requests.exceptions.Timeout as errt:
            print ("Timeout Error:",errt)
            break
        except requests.exceptions.RequestException as err:
            print ("OOps: Something Else",err)
            break
    
    return all_items

# Example usage:
# Replace with an actual paginated API endpoint
# data = fetch_paginated_data('https://api.example.com/items', initial_params={'per_page': 5}, max_pages=3)
# print(f"Fetched {len(data)} items.")
# print(data[:2]) # Print first two items as example