import pandas as pd

# Works for plain HTML tables — no browser needed
# (requires lxml or html5lib installed)
tables = pd.read_html("https://example.com/page-with-table")
print(len(tables), "tables found")
df = tables[0]
print(df.head())

Before using Selenium:
☐ Is there an official API or download button?
☐ Does pd.read_html already work?
☐ Do the site's terms allow automated access?
☐ Does robots.txt disallow the pages I need?
☐ Is the data public and not personal information?

Only continue if the data needs a browser AND access is allowed.

# pip install selenium pandas
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

driver = webdriver.Chrome()   # Selenium Manager finds the driver
try:
    driver.get("https://quotes.toscrape.com/js/")  # practice site
    WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".quote")))
    rows = []
    for q in driver.find_elements(By.CSS_SELECTOR, ".quote"):
        rows.append({
            "text": q.find_element(By.CSS_SELECTOR, ".text").text,
            "author": q.find_element(By.CSS_SELECTOR, ".author").text,
        })
finally:
    driver.quit()

df = pd.DataFrame(rows)
print(df.head())

from selenium import webdriver

options = webdriver.ChromeOptions()
options.add_argument("--headless=new")
options.add_argument("--window-size=1366,768")
driver = webdriver.Chrome(options=options)
driver.get("https://quotes.toscrape.com/js/")
print(driver.title)
driver.quit()

import sqlite3
import pandas as pd
from datetime import date

df = pd.DataFrame(rows)          # rows from the Selenium step
df["author"] = df["author"].str.strip()
df = df.drop_duplicates()
df["collected_on"] = date.today().isoformat()

with sqlite3.connect("quotes.db") as con:
    df.to_sql("quotes", con, if_exists="append", index=False)

-- authors with the most quotes collected
SELECT author, COUNT(*) AS total_quotes
FROM quotes
GROUP BY author
ORDER BY total_quotes DESC
LIMIT 5;

Data Collection Job Plan
Business question: ______________________
Source page(s): ________________________
API available? Yes / No
Access allowed by terms? Yes / No (checked on ____)
Fields to collect: ______________________
Frequency: once / daily / weekly
Storage: CSV / SQLite / database table
Owner: ________

Run Log
Date       Rows   Errors   Duration   Notes
________   ____   ______   ________   ______________
________   ____   ______   ________   ______________
________   ____   ______   ________   ______________

If rows drop suddenly → check if the page layout changed.

Handover Note
Script: collect_quotes.py
How to run: python collect_quotes.py
Requirements: selenium, pandas
What breaks most often: CSS selectors after a site redesign
How to fix: inspect the page, update the selector, re-run
Where data goes: quotes.db, table quotes
Contact: ________