How to scrape an infinite scroll page with Playwright (Python)

page.goto(START_URL)
page.wait_for_selector(ITEM_SELECTOR)

page.mouse.wheel(0, 20000)
time.sleep(DELAY_SECONDS)
count = len(page.query_selector_all(ITEM_SELECTOR))

while quiet_rounds < QUIET_ROUNDS_LIMIT and scrolls < MAX_SCROLLS:
    ...
    if count > seen:
        seen = count
        quiet_rounds = 0
    else:
        quiet_rounds += 1

with open("scroll_output.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=["quote", "author"])
    writer.writeheader()
    writer.writerows(rows)

"""Scrape https://quotes.toscrape.com/scroll — a public practice page built for
exactly this exercise. Every quote and author is loaded the same way: more rows
arrive as you scroll down, with no page numbers and no "load more" button.

Run:
    pip3 install playwright
    python3 -m playwright install chromium
    python3 infinite_scroll.py
"""

import csv
import time

from playwright.sync_api import sync_playwright

START_URL = "https://quotes.toscrape.com/scroll"
ITEM_SELECTOR = "div.quote"
DELAY_SECONDS = 1.0    # pause after each scroll, waiting for the next batch
QUIET_ROUNDS_LIMIT = 3  # stop after this many scrolls in a row add nothing
MAX_SCROLLS = 200       # hard stop, in case the feed never runs out


def scrape(page):
    page.goto(START_URL)
    page.wait_for_selector(ITEM_SELECTOR)

    seen = 0
    quiet_rounds = 0
    scrolls = 0

    while quiet_rounds < QUIET_ROUNDS_LIMIT and scrolls < MAX_SCROLLS:
        page.mouse.wheel(0, 20000)
        scrolls += 1
        time.sleep(DELAY_SECONDS)

        count = len(page.query_selector_all(ITEM_SELECTOR))
        if count > seen:
            seen = count
            quiet_rounds = 0
        else:
            quiet_rounds += 1

    rows = []
    for item in page.query_selector_all(ITEM_SELECTOR):
        rows.append({
            "quote": item.query_selector("span.text").inner_text(),
            "author": item.query_selector("small.author").inner_text(),
        })
    return rows


def main():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        rows = scrape(page)
        browser.close()

    with open("scroll_output.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["quote", "author"])
        writer.writeheader()
        writer.writerows(rows)

    print(f"Scraped {len(rows)} rows -> scroll_output.csv")


if __name__ == "__main__":
    main()

$ python3 infinite_scroll.py
Scraped 100 rows -> scroll_output.csv

quote,author
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,Albert Einstein
"“It is our choices, Harry, that show what we truly are, far more than our abilities.”",J.K. Rowling

"""Scrape quotes.toscrape.com — generated by Scrapejig on 2026-09-10.

Setup (one time):
    pip3 install playwright
    python3 -m playwright install chromium

Run:
    python3 scrape_quotes_toscrape_com.py

If pip3 says "externally-managed-environment" (Homebrew, Debian, Ubuntu 23.04+),
create a virtual environment and run all three lines inside it:
    python3 -m venv .venv && source .venv/bin/activate
Every new terminal needs that activate line again before the script will run.

On Windows the launcher is py: py -m pip install playwright, then
py -m playwright install chromium, then py scrape_quotes_toscrape_com.py.
A venv activates there with .venv/Scripts/activate in place of the source line.

Output: scrape_output.csv and scrape_output.json in the working directory.
This file is yours — edit it freely. The constants below are the usual knobs.
"""

import csv
import json
import sys
import time

from playwright.sync_api import sync_playwright

START_URL = "https://quotes.toscrape.com/scroll"
HEADLESS = True
OUTPUT_BASE = "scrape_output"
DELAY_SECONDS = 1.0  # pause between page loads; raise to be gentler
ITEM_SELECTOR = "div.quote"
FIELDS = ["quote", "author"]
MAX_SCROLLS = 200  # hard stop; a feed that never ends would scroll forever


def extract_item(item):
    """Pull one row's fields from a single item element."""
    return {
        "quote": text(item, "span.text"),
        "author": text(item, "small.author"),
    }


def rendered_text(node):
    """The text a node renders. inner_text() is HTML-only and raises on SVG and other
    non-HTML nodes, where text_content() asks the same question and answers it.
    """
    try:
        return node.inner_text().strip()
    except Exception:
        pass
    try:
        return (node.text_content() or "").strip()
    except Exception:
        return ""


def text(el, selector):
    node = el.query_selector(selector)
    return rendered_text(node) if node else ""


def rows_on(page):
    """Every real item on the page in front of us.

    A row whose every column came back empty means the item selector matched something that is
    not an item — a page section sitting beside the real cards — so it is dropped rather than
    padding the output with blank lines.
    """
    rows = []
    for item in page.query_selector_all(ITEM_SELECTOR):
        row = extract_item(item)
        if any(row.values()):
            rows.append(row)
    return rows


# The tail every start-page failure shares. A wall served to a fresh profile explains a
# navigation that hangs, a 403, and a document with no items in it equally well.
BLOCKED_ADVICE = (
    "A site will often serve a bot check or a cookie wall to a fresh headless profile "
    "instead of the page. Set HEADLESS = False to see what came back, and if it is a "
    "banner or a login, record a session with the storage_state note in main()."
)


def open_start_page(page):
    """Load the start page and wait for the items, explaining what went wrong before it bites."""
    try:
        # "domcontentloaded", not Playwright's default "load". The default waits for every
        # subresource the page pulls in, so one hanging tracker or ad pixel times the navigation
        # out thirty seconds after the markup — all of it, items included — already arrived.
        # Readiness here is "are the items there", and the wait below asks exactly that.
        response = page.goto(START_URL, wait_until="domcontentloaded")
    except Exception:
        print(f"Could not load {START_URL}: the document never arrived.", file=sys.stderr)
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise
    # goto() raises on a transport failure but not on an HTTP one: 404, 403 and 500 are all
    # successful navigations to a page that says no. Unchecked, they fall through to the wait
    # below, cost its full timeout, and are then reported as something they are not.
    if response is not None and not response.ok:
        print(
            f"{START_URL} answered HTTP {response.status}, so the items never loaded.",
            file=sys.stderr,
        )
        print(
            "A 404 or 410 means START_URL is wrong or the listing has moved — open it in a "
            "normal browser and check. A 401, 403 or 429 is usually a wall rather than a "
            "verdict on the URL:",
            file=sys.stderr,
        )
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise RuntimeError(f"HTTP {response.status} at {START_URL}")
    try:
        page.wait_for_selector(ITEM_SELECTOR)
    except Exception:
        print(f"Nothing matched ITEM_SELECTOR on {START_URL}.", file=sys.stderr)
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise


def scrape(page):
    """Scroll until the item count stops growing (3 quiet rounds)."""
    open_start_page(page)
    quiet_rounds = 0
    seen = 0
    scrolls = 0
    while quiet_rounds < 3 and scrolls < MAX_SCROLLS:
        try:
            page.mouse.wheel(0, 20000)
        except Exception as error:
            # Whatever stopped the scroll, what has already loaded is on the page and is about
            # to be read. Ending here costs the rest of the feed, not the whole run.
            print(f"Stopped scrolling: {error} — reading what has loaded.", file=sys.stderr)
            break
        scrolls += 1
        time.sleep(DELAY_SECONDS)
        count = len(page.query_selector_all(ITEM_SELECTOR))
        quiet_rounds = quiet_rounds + 1 if count <= seen else 0
        # The highest count seen, not the latest one. A virtualized feed drops the items you
        # scrolled past, so the live count falls and then rises again; against the latest
        # count that rise looks like new content, resets quiet_rounds, and the loop never ends.
        seen = max(seen, count)
    return rows_on(page)


def main():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=HEADLESS)
        page = browser.new_page()
        # Cookie banner or login wall? This browser starts with a clean profile that has
        # agreed to nothing, so plenty of sites — most UK and EU ones — cover the page with a
        # consent overlay that swallows every click, and a login-gated site never shows you
        # the content at all. Fix both the same way: record a session once with
        #     python3 -m playwright codegen --save-storage=auth.json "https://quotes.toscrape.com/scroll"
        # dismiss the banner in the window that opens (and sign in, if you need to), close it,
        # then replace the new_page() line above with:
        #     page = browser.new_context(storage_state="auth.json").new_page()
        rows = scrape(page)
        browser.close()
    write_output(rows)
    print(f"Scraped {len(rows)} rows -> {OUTPUT_BASE}.csv and {OUTPUT_BASE}.json")


def write_output(rows):
    with open(f"{OUTPUT_BASE}.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS)
        writer.writeheader()
        writer.writerows(rows)
    with open(f"{OUTPUT_BASE}.json", "w", encoding="utf-8") as f:
        json.dump(rows, f, indent=2, ensure_ascii=False)


if __name__ == "__main__":
    main()

$ python3 scrape_quotes_toscrape_com.py
Scraped 100 rows -> scrape_output.csv and scrape_output.json