Export any HTML table from a website to CSV

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()
    page.goto("https://www.scrapethissite.com/pages/forms/?page_num=1")
    page.wait_for_selector("tr.team")

    rows = []
    for tr in page.query_selector_all("tr.team"):
        rows.append({
            "team": tr.query_selector("td.name").inner_text().strip(),
            "year": tr.query_selector("td.year").inner_text().strip(),
            "wins": tr.query_selector("td.wins").inner_text().strip(),
            "losses": tr.query_selector("td.losses").inner_text().strip(),
        })
    print(len(rows), "rows")
    browser.close()

headers = [th.inner_text().strip() for th in page.query_selector_all("table.table th")]
print(headers[:4])
# ['Team Name', 'Year', 'Wins', 'Losses']

MAX_PAGES = 3  # demo cap; see the note below the full script

pages_done = 0
while True:
    # ...read rows on this page as in step 1...
    pages_done += 1
    if pages_done >= MAX_PAGES:
        break
    next_link = page.query_selector('a[aria-label="Next"]')
    if next_link is None:
        break
    next_link.click()
    page.wait_for_load_state()
    page.wait_for_selector("tr.team")
    time.sleep(1.0)  # be polite between page loads

"""Scrape the hockey team table at scrapethissite.com/pages/forms/ to CSV.

Tutorial script for https://scrapejig.com/guides/export-html-table-to-csv

Setup:
    pip3 install playwright
    python3 -m playwright install chromium

Run:
    python3 tutorial_scrape.py
"""

import csv
import time

from playwright.sync_api import sync_playwright

START_URL = "https://www.scrapethissite.com/pages/forms/?page_num=1"
MAX_PAGES = 3  # demo cap so this runs fast; see the note below to remove it
DELAY_SECONDS = 1.0


def scrape_page(page):
    """Read every row on the table currently loaded."""
    headers = [th.inner_text().strip() for th in page.query_selector_all("table.table th")]
    rows = []
    for tr in page.query_selector_all("tr.team"):
        rows.append({
            "team": tr.query_selector("td.name").inner_text().strip(),
            "year": tr.query_selector("td.year").inner_text().strip(),
            "wins": tr.query_selector("td.wins").inner_text().strip(),
            "losses": tr.query_selector("td.losses").inner_text().strip(),
        })
    return headers, rows


def main():
    all_rows = []
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(START_URL)
        page.wait_for_selector("tr.team")

        pages_done = 0
        while True:
            _, rows = scrape_page(page)
            all_rows += rows
            pages_done += 1
            print(f"page {pages_done}: {len(rows)} rows")

            if pages_done >= MAX_PAGES:
                break

            next_link = page.query_selector('a[aria-label="Next"]')
            if next_link is None:
                break

            next_link.click()
            page.wait_for_load_state()
            page.wait_for_selector("tr.team")
            time.sleep(DELAY_SECONDS)

        browser.close()

    with open("tutorial_output.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["team", "year", "wins", "losses"])
        writer.writeheader()
        writer.writerows(all_rows)

    print(f"Scraped {len(all_rows)} rows -> tutorial_output.csv")


if __name__ == "__main__":
    main()

page 1: 25 rows
page 2: 25 rows
page 3: 25 rows
Scraped 75 rows -> tutorial_output.csv

team,year,wins,losses
Boston Bruins,1990,44,24
Buffalo Sabres,1990,31,30

"""Scrape www.scrapethissite.com — generated by Scrapejig on 2026-09-10.

Setup (one time):
    pip3 install playwright
    python3 -m playwright install chromium

Run:
    python3 scrape_www_scrapethissite_com.py

If pip3 says "externally-managed-environment" (Homebrew, Debian, Ubuntu 23.04+),
create a virtual environment and run all three lines inside it:
    python3 -m venv .venv && source .venv/bin/activate
Every new terminal needs that activate line again before the script will run.

On Windows the launcher is py: py -m pip install playwright, then
py -m playwright install chromium, then py scrape_www_scrapethissite_com.py.
A venv activates there with .venv/Scripts/activate in place of the source line.

Output: scrape_output.csv and scrape_output.json in the working directory.
This file is yours — edit it freely. The constants below are the usual knobs.
"""

import csv
import json
import sys
import time

from playwright.sync_api import sync_playwright

START_URL = "https://www.scrapethissite.com/pages/forms/?page_num=1"
HEADLESS = True
OUTPUT_BASE = "scrape_output"
DELAY_SECONDS = 1.0  # pause between page loads; raise to be gentler
ITEM_SELECTOR = "tr.team"
FIELDS = ["team", "year", "wins", "losses"]
NEXT_SELECTOR = "a[aria-label=\"Next\"]"
MAX_PAGES = 3  # set in the panel; change to None to scrape every page


def extract_item(item):
    """Pull one row's fields from a single item element."""
    return {
        "team": text(item, "td.name"),
        "year": text(item, "td.year"),
        "wins": text(item, "td.wins"),
        "losses": text(item, "td.losses"),
    }


def rendered_text(node):
    """The text a node renders. inner_text() is HTML-only and raises on SVG and other
    non-HTML nodes, where text_content() asks the same question and answers it.
    """
    try:
        return node.inner_text().strip()
    except Exception:
        pass
    try:
        return (node.text_content() or "").strip()
    except Exception:
        return ""


def text(el, selector):
    node = el.query_selector(selector)
    return rendered_text(node) if node else ""


def rows_on(page):
    """Every real item on the page in front of us.

    A row whose every column came back empty means the item selector matched something that is
    not an item — a page section sitting beside the real cards — so it is dropped rather than
    padding the output with blank lines.
    """
    rows = []
    for item in page.query_selector_all(ITEM_SELECTOR):
        row = extract_item(item)
        if any(row.values()):
            rows.append(row)
    return rows


# The tail every start-page failure shares. A wall served to a fresh profile explains a
# navigation that hangs, a 403, and a document with no items in it equally well.
BLOCKED_ADVICE = (
    "A site will often serve a bot check or a cookie wall to a fresh headless profile "
    "instead of the page. Set HEADLESS = False to see what came back, and if it is a "
    "banner or a login, record a session with the storage_state note in main()."
)


def open_start_page(page):
    """Load the start page and wait for the items, explaining what went wrong before it bites."""
    try:
        # "domcontentloaded", not Playwright's default "load". The default waits for every
        # subresource the page pulls in, so one hanging tracker or ad pixel times the navigation
        # out thirty seconds after the markup — all of it, items included — already arrived.
        # Readiness here is "are the items there", and the wait below asks exactly that.
        response = page.goto(START_URL, wait_until="domcontentloaded")
    except Exception:
        print(f"Could not load {START_URL}: the document never arrived.", file=sys.stderr)
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise
    # goto() raises on a transport failure but not on an HTTP one: 404, 403 and 500 are all
    # successful navigations to a page that says no. Unchecked, they fall through to the wait
    # below, cost its full timeout, and are then reported as something they are not.
    if response is not None and not response.ok:
        print(
            f"{START_URL} answered HTTP {response.status}, so the items never loaded.",
            file=sys.stderr,
        )
        print(
            "A 404 or 410 means START_URL is wrong or the listing has moved — open it in a "
            "normal browser and check. A 401, 403 or 429 is usually a wall rather than a "
            "verdict on the URL:",
            file=sys.stderr,
        )
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise RuntimeError(f"HTTP {response.status} at {START_URL}")
    try:
        page.wait_for_selector(ITEM_SELECTOR)
    except Exception:
        print(f"Nothing matched ITEM_SELECTOR on {START_URL}.", file=sys.stderr)
        print(BLOCKED_ADVICE, file=sys.stderr)
        raise


def scrape(page):
    """Walk every page via the next link, collecting items as we go."""
    open_start_page(page)
    rows = []
    pages_done = 0
    while True:
        rows += rows_on(page)
        pages_done += 1
        if MAX_PAGES is not None and pages_done >= MAX_PAGES:
            break
        next_link = page.query_selector(NEXT_SELECTOR)
        if next_link is None:
            break
        previous_url = page.url
        try:
            next_link.click()
            page.wait_for_load_state()
            page.wait_for_selector(ITEM_SELECTOR)
        except Exception as error:
            # Usually a consent banner over the link: a fresh profile has agreed to nothing, so
            # the overlay is there and it eats the click (see the storage_state note in main()).
            # The pages already walked are still yours; losing them to the last click is not.
            print(
                f"Stopped paginating: {error} — keeping the {len(rows)} rows collected so far.",
                file=sys.stderr,
            )
            break
        time.sleep(DELAY_SECONDS)
        # A next arrow that never disappears would loop forever, so stop if the URL
        # did not move. Site paginates without changing it? Delete this, set MAX_PAGES.
        if page.url == previous_url:
            break
    return rows


def main():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=HEADLESS)
        page = browser.new_page()
        # Cookie banner or login wall? This browser starts with a clean profile that has
        # agreed to nothing, so plenty of sites — most UK and EU ones — cover the page with a
        # consent overlay that swallows every click, and a login-gated site never shows you
        # the content at all. Fix both the same way: record a session once with
        #     python3 -m playwright codegen --save-storage=auth.json "https://www.scrapethissite.com/pages/forms/?page_num=1"
        # dismiss the banner in the window that opens (and sign in, if you need to), close it,
        # then replace the new_page() line above with:
        #     page = browser.new_context(storage_state="auth.json").new_page()
        rows = scrape(page)
        browser.close()
    write_output(rows)
    print(f"Scraped {len(rows)} rows -> {OUTPUT_BASE}.csv and {OUTPUT_BASE}.json")


def write_output(rows):
    with open(f"{OUTPUT_BASE}.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=FIELDS)
        writer.writeheader()
        writer.writerows(rows)
    with open(f"{OUTPUT_BASE}.json", "w", encoding="utf-8") as f:
        json.dump(rows, f, indent=2, ensure_ascii=False)


if __name__ == "__main__":
    main()