import pandas as pd
import requests

resp = requests.get("https://jsonplaceholder.typicode.com/users", timeout=10)
resp.raise_for_status()
users = pd.json_normalize(resp.json())          # flattens nested fields
users = users[["id", "name", "email", "address.city", "company.name"]]
users.columns = ["user_id", "name", "email", "city", "company"]
print(users.head())

import pandas as pd

df = pd.read_csv("orders.csv")
print(df.memory_usage(deep=True).sum() / 1e6, "MB")
# If this is close to your machine's RAM, use chunks, better dtypes,
# or move the work to a database or Spark.

import pandas as pd

totals = []
for chunk in pd.read_csv("events_large.csv",
                         usecols=["event_date", "event_type"],
                         dtype={"event_type": "category"},
                         parse_dates=["event_date"],
                         chunksize=500_000):
    totals.append(chunk.groupby(["event_date", "event_type"], observed=True).size())

daily = pd.concat(totals).groupby(level=[0, 1]).sum().rename("events").reset_index()
print(daily.head())

# pip install pyarrow
import pandas as pd

daily["month"] = daily["event_date"].dt.strftime("%Y-%m")
daily.to_parquet("out/daily_events", partition_cols=["month"], index=False)

# Read back only one partition later
nov = pd.read_parquet("out/daily_events", filters=[("month", "==", "2026-11")])
print(len(nov))

import pandas as pd

orders = pd.read_parquet("orders.parquet")
customers = pd.read_parquet("customers.parquet")

merged = orders.merge(customers, on="customer_id", how="left",
                      validate="many_to_one", indicator=True)
missing = merged["_merge"].eq("left_only").sum()
assert len(merged) == len(orders), "row count changed after merge"
assert missing == 0, f"{missing} orders have unknown customers"
merged = merged.drop(columns="_merge")

# transforms.py
import pandas as pd

def revenue_by_city(df: pd.DataFrame) -> pd.DataFrame:
    return (df.groupby("city", as_index=False)["amount"].sum()
              .sort_values("amount", ascending=False))

# test_transforms.py   (run: pytest -q)
from transforms import revenue_by_city

def test_revenue_by_city():
    df = pd.DataFrame({"city": ["Noida", "Delhi", "Noida"], "amount": [100, 50, 30]})
    out = revenue_by_city(df)
    assert out.iloc[0].to_dict() == {"city": "Noida", "amount": 130}

etl_orders/
├── extract.py       # read sources with dtypes, chunks
├── transform.py     # pure functions, no I/O
├── load.py          # write parquet / database, idempotent
├── checks.py        # row counts, nulls, ranges
├── run.py           # orchestrates the steps, logs timings
├── tests/
│   └── test_transform.py
└── README.md

pandas Pipeline Checklist
☐ dtypes set on read
☐ Only needed columns read
☐ Merges use validate= and row-count checks
☐ Transform logic in pure, tested functions
☐ Output written as Parquet or to a table with a key
☐ Reruns do not duplicate data
☐ Logging of row counts and duration
☐ README with how to run

Approach for DE take-home assignments
1. Restate the task and assumptions in the README
2. Profile the data: shape, dtypes, nulls, duplicates
3. Write transforms as small functions with tests
4. Add data quality checks and fail loudly
5. Output in a sensible format (Parquet/DB)
6. Explain how it would scale (Spark/warehouse)