# Selenium — scrape an HTML table into a list of rows
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome()
driver.get("https://example.com/products")

# Wait until the table is present
WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.CSS_SELECTOR, "table.data"))
)

rows = []
for tr in driver.find_elements(By.CSS_SELECTOR, "table.data tbody tr"):
    cells = [td.text.strip() for td in tr.find_elements(By.TAG_NAME, "td")]
    if cells:
        rows.append(cells)

driver.quit()
print(f"Scraped {len(rows)} rows")
print(rows[:3])

# Headless Selenium + save directly to a Pandas DataFrame
import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By

opts = Options()
opts.add_argument("--headless=new")
opts.add_argument("--window-size=1920,1080")

driver = webdriver.Chrome(options=opts)
driver.get("https://example.com/listings")

data = []
for card in driver.find_elements(By.CSS_SELECTOR, ".listing-card"):
    data.append({
        "title":  card.find_element(By.CSS_SELECTOR, ".title").text,
        "price":  card.find_element(By.CSS_SELECTOR, ".price").text,
        "rating": card.find_element(By.CSS_SELECTOR, ".rating").text,
    })

driver.quit()

df = pd.DataFrame(data)
df["price"] = (
    df["price"].str.replace("₹", "").str.replace(",", "").astype(float)
)
df.to_csv("listings.csv", index=False)

print(df.head())
print(df["price"].describe())

# Pagination — collect every page of results
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

driver = webdriver.Chrome()
driver.get("https://example.com/search?q=laptop")

all_rows = []
page = 1

while True:
    WebDriverWait(driver, 10).until(
        EC.presence_of_all_elements_located(
            (By.CSS_SELECTOR, ".result-item")
        )
    )

    for item in driver.find_elements(By.CSS_SELECTOR, ".result-item"):
        all_rows.append({
            "page":  page,
            "name":  item.find_element(By.CSS_SELECTOR, ".name").text,
            "price": item.find_element(By.CSS_SELECTOR, ".price").text,
        })

    try:
        next_btn = driver.find_element(By.CSS_SELECTOR, "a.next")
        if not next_btn.is_enabled():
            break
        next_btn.click()
        page += 1
        time.sleep(1)   # small pause, be polite
    except Exception:
        break

driver.quit()
print(f"Collected {len(all_rows)} items across {page} pages")

# Project 1 — Dynamic price tracker
# README outline

Problem:
"Track competitor prices for a category of
products over time and visualise trends."

Data:
- Source: public product listing page
- Fields: name, price, rating, discount
- Frequency: daily (cron job)

Steps:
1. Use Selenium + headless Chrome to scrape
2. Store results with a scrape_date column
3. Compute week-over-week price changes
4. Chart top products by volatility

Results:
- 6-week trend chart
- Table of biggest price movers

Skills shown:
- Selenium, waits, pagination
- Pandas cleaning + aggregation
- Simple visualisations

# Project 2 — Job market analyser
# README outline

Problem:
"Which skills are most in demand for data
analyst roles in my city?"

Data:
- Source: a public job board
- Fields: title, company, skills, location
- Sample: 500+ listings

Steps:
1. Scrape job listings across pages
2. Extract skills from description text
3. Count frequency of each skill
4. Show top 20 skills in a bar chart
5. Highlight which of your own skills appear

Results:
- Top 20 demanded skills
- Gap analysis vs my own resume

Skills shown:
- Selenium + pagination + waits
- Text parsing with Pandas + regex
- Data storytelling

Perfect for: analysts targeting jobs now.

# A polite Selenium scraper skeleton
import time, random
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

opts = Options()
opts.add_argument("--headless=new")
opts.add_argument("--user-agent=DataAnalystBot/1.0")

driver = webdriver.Chrome(options=opts)

urls = [
    "https://example.com/page/1",
    "https://example.com/page/2",
    "https://example.com/page/3",
]

for url in urls:
    driver.get(url)
    # extract data here
    time.sleep(random.uniform(2.0, 4.0))   # polite pause

driver.quit()

# Check robots.txt before scraping
import urllib.robotparser as rp

SITE = "https://example.com"
bot = rp.RobotFileParser()
bot.set_url(f"{SITE}/robots.txt")
bot.read()

def allowed(path: str) -> bool:
    return bot.can_fetch("DataAnalystBot", f"{SITE}{path}")

for path in ["/products", "/api/data", "/private"]:
    print(path, "→", "allowed" if allowed(path) else "blocked")

# If robots.txt disallows a path, don't scrape it.
# Prefer an official API when one is available.

Daily Selenium routine (45-60 minutes):
20 min — Read one concept (e.g., waits)
20 min — Code along with a real page
10 min — Write your own notes
5  min — Review yesterday

Consistency beats intensity.
30 days × 45 min = a usable skill.

Recommended practice resources:
- Selenium official docs (Python bindings)
- A public "sandbox" site you can scrape
  (e.g., books.toscrape.com or quotes.toscrape.com)
- Pandas docs for DataFrame operations
- GitHub: search "selenium scraping examples"

Rule: pick ONE public site and build one project
on it. Depth beats breadth every time.