Scrape a paginated listing with Playwright (and stop where you want)

for quote in page.query_selector_all("div.quote"):
    row = {
        "quote": quote.query_selector("span.text").inner_text(),
        "author": quote.query_selector("small.author").inner_text(),
        "author_url": quote.query_selector("a[href^='/author/']").get_attribute("href"),
    }

MAX_PAGES = 2  # demo cap: 2 pages -> 20 rows -> 20 detail visits

rows = []
pages_done = 0
while True:
    for quote in page.query_selector_all("div.quote"):
        rows.append({ ... })  # as in step 1

    pages_done += 1
    if pages_done >= MAX_PAGES:
        break

    next_link = page.query_selector("li.next a")
    if next_link is None:
        break

    next_link.click()
    page.wait_for_load_state()

DELAY_SECONDS = 1.0

    next_link.click()
    page.wait_for_load_state()
    time.sleep(DELAY_SECONDS)

detail_page = page.context.new_page()
for row in rows:
    try:
        detail_page.goto(f"https://quotes.toscrape.com{row['author_url']}")
        row["born"] = detail_page.query_selector("span.author-born-date").inner_text()
    except Exception as error:
        print(f"Skipped {row['author_url']}: {error}")
        row["born"] = ""
    time.sleep(DELAY_SECONDS)
detail_page.close()

"""Scrape every quote on quotes.toscrape.com, page by page, then visit each
author's own page for their date of birth.

Setup:
    pip3 install playwright
    python3 -m playwright install chromium

Run:
    python3 scrape_quotes.py
"""

import csv
import time

from playwright.sync_api import sync_playwright

START_URL = "https://quotes.toscrape.com/"
MAX_PAGES = 2        # demo cap: 2 pages -> 20 rows -> 20 detail visits
DELAY_SECONDS = 1.0  # pause between requests, so we are not hammering the site


def scrape_listing(page):
    """Collect quote/author/author_url rows, following the Next link until it runs out."""
    rows = []
    pages_done = 0
    while True:
        for quote in page.query_selector_all("div.quote"):
            rows.append({
                "quote": quote.query_selector("span.text").inner_text(),
                "author": quote.query_selector("small.author").inner_text(),
                # The "(about)" link's href always starts with /author/ — that is
                # how we tell it apart from the tag links in the same block.
                "author_url": quote.query_selector("a[href^='/author/']").get_attribute("href"),
            })
        pages_done += 1
        # Even on a site this small, cap the walk. Without MAX_PAGES a bug in the
        # "is there a next page" check — or a site that changed underneath you —
        # turns into an unbounded loop instead of a script that just finishes.
        if pages_done >= MAX_PAGES:
            break
        next_link = page.query_selector("li.next a")
        # The last page has no li.next element at all, so query_selector returns
        # None rather than raising. That is the signal the walk is done.
        if next_link is None:
            break
        next_link.click()
        page.wait_for_load_state()
        time.sleep(DELAY_SECONDS)
    return rows


def scrape_born_date(detail_page, author_url):
    """Visit one author's page on the SAME browser (a second tab) and read their born date."""
    detail_page.goto(f"https://quotes.toscrape.com{author_url}")
    return detail_page.query_selector("span.author-born-date").inner_text()


def main():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context()
        page = context.new_page()
        page.goto(START_URL)

        rows = scrape_listing(page)

        # A second tab for the detail visits. Same context, so it shares cookies
        # with the listing tab; a separate page keeps the listing's history and
        # state untouched, which matters once detail visits interleave with a walk.
        detail_page = page.context.new_page()
        for row in rows:
            try:
                row["born"] = scrape_born_date(detail_page, row["author_url"])
            except Exception as error:
                # One broken author link should not sink the whole scrape: keep the
                # row with the fields we already have, and just leave born blank.
                print(f"Skipped {row['author_url']}: {error}")
                row["born"] = ""
            time.sleep(DELAY_SECONDS)
        detail_page.close()

        browser.close()

    with open("tutorial_output.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["quote", "author", "author_url", "born"])
        writer.writeheader()
        writer.writerows(rows)

    print(f"Scraped {len(rows)} rows -> tutorial_output.csv")


if __name__ == "__main__":
    main()

$ python3 scrape_quotes.py
Scraped 20 rows -> tutorial_output.csv

quote,author,author_url,born
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,Albert Einstein,/author/Albert-Einstein,"March 14, 1879"
"“It is our choices, Harry, that show what we truly are, far more than our abilities.”",J.K. Rowling,/author/J-K-Rowling,"July 31, 1965"

"""Scrape quotes.toscrape.com — generated by Scrapejig on 2026-09-10.

Setup (one time):
    pip3 install playwright
    python3 -m playwright install chromium

Run:
    python3 scrape_quotes_toscrape_com.py

If pip3 says "externally-managed-environment" (Homebrew, Debian, Ubuntu 23.04+),
create a virtual environment and run all three lines inside it:
    python3 -m venv .venv && source .venv/bin/activate
Every new terminal needs that activate line again before the script will run.

On Windows the launcher is py: py -m pip install playwright, then
py -m playwright install chromium, then py scrape_quotes_toscrape_com.py.
A venv activates there with .venv/Scripts/activate in place of the source line.

Output: scrape_output.csv and scrape_output.json in the working directory.
This file is yours — edit it freely. The constants below are the usual knobs.
"""

import csv
import json
import sys
import time
from urllib.parse import urljoin

from playwright.sync_api import sync_playwright

START_URL = "https://quotes.toscrape.com/"
HEADLESS = True
OUTPUT_BASE = "scrape_output"
DELAY_SECONDS = 1.0  # pause between page loads; raise to be gentler
ITEM_SELECTOR = "div.quote"
FIELDS = ["quote", "author", "author_url", "born"]
NEXT_SELECTOR = "li.next a"
MAX_PAGES = 2  # set in the panel; change to None to scrape every page


def extract_item(item, base):
    """Pull one row's fields from a single item element.

    base is the URL of the page this item came from. Relative links resolve
    against it, which from page 2 onward is no longer START_URL.
    """
    return {
        "quote": text(item, "span.text"),
        "author": text(item, "small.author"),
        "author_url": attr(item, "a[href^=\"/author/\"]", "href", base),
    }


def rendered_text(node):
    """The text a node renders. inner_text() is HTML-only and raises on SVG and other
    non-HTML nodes, where text_content() asks the same question and answers it.
    """
    try:
        return node.inner_text().strip()
    except Exception:
        pass
    try:
        return (node.text_content() or "").strip()
    except Exception:
        return ""


def text(el, selector):
    node = el.query_selector(selector)
    return rendered_text(node) if node else ""


def attr(el, selector, name, base):
    node = el.query_selector(selector)
    try:
        value = (node.get_attribute(name) or "") if node else ""
    except Exception:
        return ""
    if value and name in ("href", "src"):
        value = urljoin(base, value)
    return value


def rows_on(page):
    """Every real item on the page in front of us.

    A row whose every column came back empty means the item selector matched something that is
    not an item — a page section sitting beside the real cards — so it is dropped rather than
    padding the output with blank lines.
    """
    rows = []
    for item in page.query_selector_all(ITEM_SELECTOR):
        row = extract_item(item, page.url)
        if any(row.values()):
            rows.append(row)
    return rows


# The tail every start-page failure shares. A wall served to a fresh profile explains a
# navigation that hangs, a 403, and a document with no items in it equally well.
BLOCKED_ADVICE = (
    "A site will often serve a bot check or a cookie wall to a fresh headless profile "
    "instead of the page. Set HEADLESS = False to see what came back, and if it is a "
    "banner or a login, record a session with the storage_state note in main()."
)


def open_start_page(page):
    """Load the start page and wait for the items, explaining what went wrong before it bites."""
    try:
        # "domcontentloaded", not Playwright's default "load". The default waits for every
        # subresource the page pulls in, so one hanging tracker or ad pixel times the navigation
        # out thirty seconds after the markup — all of it, items included — already arrived.
        # Readiness here is "are the items there", and the wait below asks exactly that.
        response = page.goto(START_URL, wait_until="domcontentloaded")
    except Exception:
        print(f"Could not load {START_URL}: the document never arrived.", file=sys.stderr)
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise
    # goto() raises on a transport failure but not on an HTTP one: 404, 403 and 500 are all
    # successful navigations to a page that says no. Unchecked, they fall through to the wait
    # below, cost its full timeout, and are then reported as something they are not.
    if response is not None and not response.ok:
        print(
            f"{START_URL} answered HTTP {response.status}, so the items never loaded.",
            file=sys.stderr,
        )
        print(
            "A 404 or 410 means START_URL is wrong or the listing has moved — open it in a "
            "normal browser and check. A 401, 403 or 429 is usually a wall rather than a "
            "verdict on the URL:",
            file=sys.stderr,
        )
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise RuntimeError(f"HTTP {response.status} at {START_URL}")
    try:
        page.wait_for_selector(ITEM_SELECTOR)
    except Exception:
        print(f"Nothing matched ITEM_SELECTOR on {START_URL}.", file=sys.stderr)
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise


def scrape(page):
    """Walk every page via the next link, collecting items as we go."""
    open_start_page(page)
    rows = []
    pages_done = 0
    while True:
        rows += rows_on(page)
        pages_done += 1
        if MAX_PAGES is not None and pages_done >= MAX_PAGES:
            break
        next_link = page.query_selector(NEXT_SELECTOR)
        if next_link is None:
            break
        previous_url = page.url
        try:
            next_link.click()
            page.wait_for_load_state()
            page.wait_for_selector(ITEM_SELECTOR)
        except Exception as error:
            # Usually a consent banner over the link: a fresh profile has agreed to nothing, so
            # the overlay is there and it eats the click (see the storage_state note in main()).
            # The pages already walked are still yours; losing them to the last click is not.
            print(
                f"Stopped paginating: {error} — keeping the {len(rows)} rows collected so far.",
                file=sys.stderr,
            )
            break
        time.sleep(DELAY_SECONDS)
        # A next arrow that never disappears would loop forever, so stop if the URL
        # did not move. Site paginates without changing it? Delete this, set MAX_PAGES.
        if page.url == previous_url:
            break
    return rows


def extract_detail(page, url):
    """Visit one item's detail page and pull its extra fields."""
    # Same cheap wait the start page uses, and it matters more here because the cost is per ROW:
    # under the default "load" one hanging tracker on a product template costs thirty seconds on
    # every item, so a two-hundred-row drill-down spends over an hour waiting for ad pixels on
    # pages it had finished reading. The fields below are read straight off the document.
    response = page.goto(url, wait_until="domcontentloaded")
    # goto() raises on a TRANSPORT failure — refused connection, bad DNS — and an HTTP error is
    # not one of those: 404, 403 and 500 are successful navigations to a page that says no. Left
    # unchecked, the selectors below find nothing on that page and every detail cell records "",
    # which in the CSV is indistinguishable from a detail page that genuinely had no description.
    # Raising here routes it into the caller's per-row arm, so the row keeps its listing fields,
    # carries no detail ones, and says on the way past which URL failed and why.
    #
    # None is not a failure: goto() returns it for a navigation that did not produce a new
    # document, which is a statement about the navigation rather than about the server.
    if response is not None and not response.ok:
        raise RuntimeError(f"HTTP {response.status} at {url}")
    page.wait_for_load_state()
    return {
        "born": text(page, "span.author-born-date"),
    }


def main():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=HEADLESS)
        page = browser.new_page()
        # Cookie banner or login wall? This browser starts with a clean profile that has
        # agreed to nothing, so plenty of sites — most UK and EU ones — cover the page with a
        # consent overlay that swallows every click, and a login-gated site never shows you
        # the content at all. Fix both the same way: record a session once with
        #     python3 -m playwright codegen --save-storage=auth.json "https://quotes.toscrape.com/"
        # dismiss the banner in the window that opens (and sign in, if you need to), close it,
        # then replace the new_page() line above with:
        #     page = browser.new_context(storage_state="auth.json").new_page()
        rows = scrape(page)
        for row in rows:
            url = row["author_url"]
            if url:
                try:
                    row.update(extract_detail(page, url))
                except Exception as error:
                    # One dead link must not cost you a finished scrape. This row keeps the
                    # fields read off the listing and simply has no detail ones. On stderr with
                    # every other warning this script emits: stdout carries the row count, and a
                    # skipped page is the kind of thing you want to see when that is piped away.
                    print(f"Skipped detail page {url}: {error}", file=sys.stderr)
                time.sleep(DELAY_SECONDS)
        browser.close()
    write_output(rows)
    print(f"Scraped {len(rows)} rows -> {OUTPUT_BASE}.csv and {OUTPUT_BASE}.json")


def write_output(rows):
    with open(f"{OUTPUT_BASE}.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS)
        writer.writeheader()
        writer.writerows(rows)
    with open(f"{OUTPUT_BASE}.json", "w", encoding="utf-8") as f:
        json.dump(rows, f, indent=2, ensure_ascii=False)


if __name__ == "__main__":
    main()