Playwright vs Traditional Tools for Data Scientists

# static_scrape.py — the workhorse pattern
import requests
from bs4 import BeautifulSoup
import pandas as pd
import time

BASE_URL = "https://example.com/products"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (your research bot)",
}

def fetch_page(page_num: int):
    url = f"{BASE_URL}?page={page_num}"
    resp = requests.get(url, headers=HEADERS, timeout=10)
    resp.raise_for_status()
    return resp.text

def parse_page(html: str):
    soup = BeautifulSoup(html, "html.parser")
    rows = []
    for card in soup.select(".product-card"):
        rows.append({
            "name": card.select_one(".title").get_text(strip=True),
            "price": card.select_one(".price").get_text(strip=True),
            "rating": card.select_one(".rating")["data-value"],
        })
    return rows

if __name__ == "__main__":
    all_rows = []
    for page in range(1, 21):
        html = fetch_page(page)
        all_rows.extend(parse_page(html))
        time.sleep(1)   # be polite

    df = pd.DataFrame(all_rows)
    df.to_csv("products.csv", index=False)
    print(f"Saved {len(df)} rows")

# Why this wins for static pages:
# ✓ No browser process to start
# ✓ Can fetch 20 pages in 20 seconds
# ✓ Easy to debug (you can see the HTML)
# ✓ No Playwright/Selenium to install
#
# Do this FIRST on every scraping task.
# Only reach for Playwright if this returns empty.

The 3-step check before writing any scraper:

1. FETCH THE PAGE MANUALLY
   Open DevTools → Network tab → reload
   Look at the first request to the page
   Check "Response" tab.
   Is the data you want inside the HTML?

   YES → use requests + BeautifulSoup
   NO  → continue

2. LOOK FOR AN API
   Filter Network to "Fetch/XHR"
   Reload. Any JSON response with the data?
   Often the site loads via a clean JSON API
   you can call directly.

   YES → call the JSON API (still requests)
   NO  → continue

3. CHECK IF RENDERED BY JAVASCRIPT
   View page source (Ctrl+U)
   Is the data in the source?
   Is it missing but visible in the browser?

   NO → requests + BS4 works
   YES → you need Playwright

The rule:
95% of "scraping is hard" problems are
"the site is a React app now".
Check that before you install Playwright.

Remember: respect robots.txt, terms of service,
and rate limits. Data collection is not the
same as bypassing protections.

# playwright_scrape.py — for JS-rendered pages
from playwright.sync_api import sync_playwright
import pandas as pd

BASE_URL = "https://example.com/products"

def scrape_products():
    rows = []
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent="Mozilla/5.0 (research bot)",
            viewport={"width": 1280, "height": 800},
        )
        page = context.new_page()

        page.goto(BASE_URL, wait_until="networkidle")

        # Wait for the actual content
        page.wait_for_selector(".product-card")

        # Extract
        cards = page.query_selector_all(".product-card")
        for card in cards:
            rows.append({
                "name": card.query_selector(".title").inner_text(),
                "price": card.query_selector(".price").inner_text(),
                "rating": card.get_attribute("data-rating"),
            })

        context.close()
        browser.close()
    return pd.DataFrame(rows)

if __name__ == "__main__":
    df = scrape_products()
    df.to_csv("products.csv", index=False)
    print(f"Saved {len(df)} rows")

# Why Playwright works here when requests doesn't:
# - The browser actually runs the JavaScript
# - The DOM contains the rendered content
# - You wait for the real state (networkidle + selector)
#
# The three steps that matter:
# 1. wait_until="networkidle"  (or "domcontentloaded" if faster)
# 2. page.wait_for_selector()  (wait for the data to appear)
# 3. query_selector_all()      (extract)

# playwright_async.py — the faster version for many pages
import asyncio
from playwright.async_api import async_playwright
import pandas as pd

URLS = [f"https://example.com/products?page={i}"
        for i in range(1, 51)]

async def scrape_one(page, url):
    try:
        await page.goto(url, wait_until="domcontentloaded")
        await page.wait_for_selector(".product-card", timeout=10000)
        rows = []
        for card in await page.query_selector_all(".product-card"):
            name = await (await card.query_selector(".title")).inner_text()
            price = await (await card.query_selector(".price")).inner_text()
            rows.append({"name": name, "price": price})
        return rows
    except Exception as e:
        print(f"Failed {url}: {e}")
        return []

async def main():
    all_rows = []
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context()
        page = await context.new_page()

        # Concurrency in batches of 5
        for i in range(0, len(URLS), 5):
            batch = URLS[i:i+5]
            results = await asyncio.gather(
                *[scrape_one(page, u) for u in batch]
            )
            for r in results:
                all_rows.extend(r)

        await browser.close()

    df = pd.DataFrame(all_rows)
    df.to_csv("products.csv", index=False)
    print(f"Saved {len(df)} rows")

if __name__ == "__main__":
    asyncio.run(main())

# What you gain over sync:
# - 5x faster for the same number of pages
# - Same API, easy to reason about
# - Reuses one browser context

# What you don't get:
# - Magical bypass of login / captcha
# - Permission to scrape a site that forbids it

# scrapy_example.py — for crawling at scale
import scrapy

class ProductsSpider(scrapy.Spider):
    name = "products"
    start_urls = ["https://example.com/products"]

    custom_settings = {
        "DOWNLOAD_DELAY": 1,
        "CONCURRENT_REQUESTS": 8,
        "RETRY_TIMES": 3,
        "FEED_FORMAT": "csv",
        "FEED_URI": "products.csv",
    }

    def parse(self, response):
        for card in response.css(".product-card"):
            yield {
                "name": card.css(".title::text").get(),
                "price": card.css(".price::text").get(),
            }

        # Follow pagination
        next_page = response.css("a.next::attr(href)").get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)

# What Scrapy gives you that Playwright doesn't:
# ✓ Automatic scheduling of thousands of requests
# ✓ Concurrency out of the box
# ✓ Built-in retry, throttling, dedup
# ✓ Pipelines: cleaning, DB writing, exports
# ✓ Robots.txt respected by default

# When you actually need Scrapy:
# - 100,000+ pages
# - Multiple sites, same structure
# - Scheduled re-scrapes
# - You care about politeness and rate limits

# When you don't:
# - A few hundred pages → requests/Playwright is enough
# - One-off data collection for a project

# When you need both: JS + scale
#
# Scrapy handles scheduling; Playwright handles JS.
# Install the plugin: pip install scrapy-playwright

# settings.py
DOWNLOAD_HANDLERS = {
    "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
    "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"

# spider
import scrapy

class JsSiteSpider(scrapy.Spider):
    name = "js_site"
    start_urls = ["https://example.com/products"]

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(
                url,
                meta={
                    "playwright": True,
                    "playwright_page_methods": [
                        # wait for the rendered content
                        # (import PlaywrightPageMethod)
                    ],
                },
            )

    def parse(self, response):
        for card in response.css(".product-card"):
            yield {
                "name": card.css(".title::text").get(),
            }

# This is the professional setup for large-scale
# JS-heavy scraping.
#
# But note: Playwright inside Scrapy is slow.
# Each page uses a full browser context.
# Budget for it — 100k pages with JS is expensive.
#
# Rule:
# - Under 1,000 JS pages → plain Playwright is fine
# - 1,000-100,000 JS pages → Playwright + a queue (Celery/RQ)
# - 100,000+ JS pages → Scrapy + Playwright, or negotiate an API

Task-to-tool checklist:

STEP 1 — CHECK FOR AN API
☐ DevTools → Network → XHR
☐ Any JSON endpoint with the data?
   → Use requests, done.

STEP 2 — CHECK THE HTML SOURCE
☐ Ctrl+U on the page
☐ Is the data in the source?
   → YES: requests + BeautifulSoup
   → NO: continue

STEP 3 — HOW MUCH DATA?
☐ Under 1,000 pages?
   → Playwright (sync), simple
☐ 1,000-100,000 pages with JS?
   → Playwright (async) + a queue
☐ 100,000+ pages with JS?
   → Scrapy + Playwright plugin
☐ Static + 100,000+ pages?
   → Scrapy alone

STEP 4 — IS IT RECURRING?
☐ Once?
   → Any of the above
☐ Daily / weekly?
   → Schedule it (cron, Airflow, GitHub Actions)

STEP 5 — RESPECT THE SITE
☐ Robots.txt checked?
☐ Terms of Service checked?
☐ Rate limit defined and enforced?
☐ Consent for personal data collected?
☐ Contact email in User-Agent?

Tool comparison — fill in for your task:

                     requests+BS4   Playwright   Scrapy
Setup time           10 min         15 min       1 hour
Static pages         ✓ best         ✓ ok         ✓ ok
JS pages             ✗              ✓ best       ✓ with plugin
Scale (100k+)        ✗              slow         ✓ best
Scheduling           ✗              ✗            ✓ built-in
Interactive (login)  ~ cookies      ✓ best       ~ with plugin
Debug difficulty     low            medium       high
Speed (static)       fastest        medium       fast
Speed (JS)           n/a            fast         medium
Learning curve       low            medium       high

The default choice:
  requests + BS4 → start here
  Playwright     → when the page needs JS
  Scrapy         → when you need scale + scheduling

Don't install all three.
Install what the task actually requires.

Ethics and compliance — non-negotiable:

Before scraping any site:

☐ Have you read robots.txt?
   → If disallowed, stop. There's no "but".

☐ Have you read the ToS?
   → Some sites explicitly forbid scraping.

☐ Are you collecting personal data?
   → If yes, you need a lawful basis (GDPR / DPDP Act).
   → Anonymise when possible.
   → Never store more than you need.

☐ Are you rate-limiting?
   → 1 request per second is a reasonable default.
   → Use exponential backoff on 429s.

☐ Do you identify yourself?
   → User-Agent should say what you are.
   → Include a contact email if you're a researcher.

☐ Are you caching?
   → Don't refetch pages you've already fetched.
   → Saves bandwidth for the site too.

What will get you blocked or sued:
✗ Bypassing login
✗ Bypassing CAPTCHA
✗ Ignoring robots.txt
✗ Heavy concurrent requests without throttling
✗ Scraping personal data without a basis
✗ Reselling scraped data

The rule:
If you wouldn't want it done to you,
don't do it to the site.