Pandas vs Traditional Tools for DevOps Engineers

# Shell one-liners every DevOps engineer should know

# Top 10 IPs hitting your server
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head -10

# Count errors by hour
grep ERROR app.log | awk '{print $1" "$2}' | cut -d: -f1 | sort | uniq -c

# Find slow requests (over 2s) in nginx logs
awk '$NF > 2.0' access.log | head -50

# Errors from a specific service
grep '"service":"payments"' app.log | grep -i error | tail -100

# Distinct Kubernetes pod restarts
kubectl get pods -A -o json | jq -r '.items[] | select(.status.containerStatuses[].restartCount > 0) | .metadata.name'

# Watch error rate live
tail -f app.log | grep --line-buffered ERROR | pv -l -i 10 > /dev/null

When to stop using shell (the smell test):

✗ Your awk script has more than 3 comments
✗ You are maintaining awk across two terminals
✗ The pipeline spans more than 5 pipes
✗ You are grouping by 2+ fields
✗ You need to join two data sources
✗ You need to do arithmetic across rows
✗ You want the result as a chart or report
✗ Your teammate can't read the one-liner

In any of those cases, switch to Pandas.
The threshold is roughly: simple filter → shell.
Multi-step analysis → Python.

Shell is fastest for the first answer.
Pandas is faster for the tenth answer
and every answer after that.

# log_analysis.py — group errors by service and hour
import pandas as pd

df = pd.read_json("app.log", lines=True)
df["ts"] = pd.to_datetime(df["ts"])
df["hour"] = df["ts"].dt.floor("h")

errors = df[df["level"].str.upper() == "ERROR"]

report = (
    errors.groupby(["service", "hour"])
    .size()
    .reset_index(name="errors")
    .sort_values("errors", ascending=False)
)

print(report.head(20))
report.to_csv("errors_by_service_hour.csv", index=False)

# In awk this becomes three nested loops
# and a manual join on service name.
# In Pandas it is one groupby + reset_index.

# capacity_report.py — trends and forecasts
import pandas as pd

# Load Prometheus-style metrics export
df = pd.read_csv("cpu_metrics.csv", parse_dates=["ts"])

# Aggregate per instance per day
daily = (
    df.set_index("ts")
      .groupby("instance")["cpu_util"]
      .resample("1D").mean()
      .reset_index()
)

# 7-day rolling average for trend
daily["rolling_7d"] = (
    daily.groupby("instance")["cpu_util"]
         .transform(lambda s: s.rolling(7, min_periods=1).mean())
)

# Flag instances trending above 70%
at_risk = daily[daily["rolling_7d"] > 70]

# Export for the weekly capacity meeting
at_risk.to_csv("at_risk_instances.csv", index=False)

print(f"{len(at_risk)} instance-days above 70% CPU")

# Shell: top 5 endpoints by 500 error count

awk '$9 == 500 {print $7}' access.log \
  | sort \
  | uniq -c \
  | sort -rn \
  | head -5

# Output:
#  1243 /api/orders
#   892 /api/checkout
#   410 /api/search
#   287 /api/cart
#   198 /api/user

# Pros: instantaneous, no setup, runs anywhere
# Cons: brittle to log format changes,
#       no joins, no percentiles, no time windows

# Pandas: same answer, plus time windows and joins

import pandas as pd

# Load nginx combined log (last field is the path)
df = pd.read_csv(
    "access.log",
    sep=" ",
    header=None,
    names=["ip", "ident", "user", "ts", "tz", "req",
           "status", "size", "ref", "ua"],
    quotechar='"',
)

df["path"] = df["req"].str.split().str[1]
df["status"] = pd.to_numeric(df["status"], errors="coerce")

top_500s = (
    df[df["status"] == 500]
    .groupby("path")
    .size()
    .sort_values(ascending=False)
    .head(5)
)

print(top_500s)
top_500s.to_csv("top_500_paths.csv")

# Bonus, because it's one line each:
# Percentiles:  df["lat"].quantile([0.5, 0.95, 0.99])
# Time window:  df[df["ts"] > "2026-11-01"]
# Joins:        df.merge(instances, on="ip")

# error_digest.py — daily error digest
import pandas as pd
from datetime import datetime, timedelta

def digest(log_path="app.log", hours=24):
    df = pd.read_json(log_path, lines=True)
    df["ts"] = pd.to_datetime(df["ts"])

    cutoff = datetime.now() - timedelta(hours=hours)
    recent = df[df["ts"] >= cutoff]
    errors = recent[recent["level"].str.upper() == "ERROR"]

    summary = (
        errors.groupby(["service", "level"])
        .agg(count=("ts", "count"), first=("ts", "min"),
             last=("ts", "max"))
        .reset_index()
        .sort_values("count", ascending=False)
    )
    return summary

if __name__ == "__main__":
    print(digest().to_string(index=False))

# oncall_report.py — the weekly on-call summary
import pandas as pd

def report(alerts_csv="alerts.csv"):
    df = pd.read_csv(alerts_csv, parse_dates=["start", "end"])
    df["duration_min"] = (df["end"] - df["start"]).dt.total_seconds() / 60

    summary = (
        df.groupby(["service", "severity"])
        .agg(alerts=("start", "count"),
             total_min=("duration_min", "sum"),
             p95_min=("duration_min", lambda s: s.quantile(0.95)))
        .reset_index()
        .sort_values(["severity", "alerts"], ascending=[True, False])
    )
    return summary

if __name__ == "__main__":
    out = report()
    print(out.to_string(index=False))
    out.to_csv("weekly_oncall.csv", index=False)

# This replaces three shell scripts and a manual
# spreadsheet that every on-call rotation was fighting.

# shell_to_pandas.sh — pipe shell into Pandas

# 1. Shell filters to the interesting lines
grep -i "error\|timeout" /var/log/app/*.log \
  | tail -10000 \
  > /tmp/errors_recent.log

# 2. Pandas does the analysis
python3 - <<'PY'
import pandas as pd
df = pd.read_csv("/tmp/errors_recent.log", sep=" ",
                 header=None,
                 names=["date","time","service","level","msg"],
                 on_bad_lines="skip")
print(df.groupby("service").size().sort_values(ascending=False))
PY

# 3. Clean up
rm /tmp/errors_recent.log

# Why this split works:
# - Shell is good at "select lines from files"
# - Pandas is good at "what do the selected lines mean"
# - Neither tool is being abused