Why Every Data Scientist Should Learn Pandas in 2026

import pandas as pd

customers = pd.read_csv("customers.csv")          # one row per customer
orders = pd.read_csv("orders.csv", parse_dates=["order_date"])
df = orders.merge(customers, on="customer_id", how="left",
                  validate="many_to_one", indicator=True)
print(df["_merge"].value_counts())                # any orders without a customer?
assert len(df) == len(orders), "merge changed the number of orders"

monthly = (df.assign(month=df["order_date"].dt.to_period("M"))
             .pivot_table(index="customer_id", columns="month",
                          values="amount", aggfunc="sum", fill_value=0))
print(monthly.iloc[:5, :4])

import pandas as pd

df = df.sort_values(["customer_id", "order_date"])
g = df.groupby("customer_id")
df["prev_amount"] = g["amount"].shift(1)
df["days_since_prev"] = g["order_date"].diff().dt.days
df["spend_last_30d"] = (
    df.set_index("order_date")
      .groupby("customer_id")["amount"]
      .rolling("30D", closed="left")       # excludes the current order
      .sum()
      .reset_index(level=0, drop=True)
      .values
)

# Average of previous orders only (no leakage from the current or future rows)
df["avg_prev_amount"] = (
    df.groupby("customer_id")["amount"]
      .transform(lambda s: s.shift(1).expanding().mean())
)
print(df[["customer_id", "order_date", "amount", "prev_amount", "avg_prev_amount"]].head(8))

import pandas as pd

pd.options.mode.copy_on_write = True
a = pd.read_csv("events.csv")
b = pd.read_csv("events.csv", dtype_backend="pyarrow")
print("default MB:", round(a.memory_usage(deep=True).sum() / 1e6, 1))
print("arrow MB:  ", round(b.memory_usage(deep=True).sum() / 1e6, 1))

features = (
    pd.read_csv("orders.csv", parse_dates=["order_date"])
      .query("amount > 0")
      .assign(month=lambda d: d["order_date"].dt.month,
              big_order=lambda d: d["amount"] >= 2000)
      .groupby("customer_id")
      .agg(orders=("order_id", "count"),
           total=("amount", "sum"),
           big_share=("big_order", "mean"))
      .reset_index()
)

import pandas as pd

def customer_features(orders: pd.DataFrame, as_of: pd.Timestamp) -> pd.DataFrame:
    past = orders[orders["order_date"] < as_of]          # only data before as_of
    out = (past.groupby("customer_id")
               .agg(orders=("order_id", "count"),
                    total=("amount", "sum"),
                    last_order=("order_date", "max"))
               .reset_index())
    out["days_since_last"] = (as_of - out["last_order"]).dt.days
    return out.drop(columns="last_order")

import pandas as pd
from features import customer_features

def test_features_ignore_future_orders():
    orders = pd.DataFrame({
        "customer_id": [1, 1, 1],
        "order_id": [10, 11, 12],
        "amount": [100, 200, 999],
        "order_date": pd.to_datetime(["2026-01-01", "2026-02-01", "2026-12-01"]),
    })
    f = customer_features(orders, pd.Timestamp("2026-06-01"))
    assert f.loc[0, "orders"] == 2
    assert f.loc[0, "total"] == 300

Pandas feature checklist
[ ] merge uses validate= and row counts are checked
[ ] Data sorted by entity and time before shift/rolling
[ ] Windows and averages exclude the current row
[ ] Features computed with an as_of date
[ ] Same function used for training and scoring
[ ] Tests cover leakage cases