How Pandas Helps You Land a Cloud Engineer Job Faster

# pip install boto3 pandas
import boto3
import pandas as pd

ec2 = boto3.client("ec2", region_name="ap-south-1")
paginator = ec2.get_paginator("describe_instances")

rows = []
for page in paginator.paginate():
    for res in page["Reservations"]:
        for inst in res["Instances"]:
            tags = {t["Key"]: t["Value"] for t in inst.get("Tags", [])}
            rows.append({
                "instance_id": inst["InstanceId"],
                "type": inst["InstanceType"],
                "state": inst["State"]["Name"],
                "launched": inst["LaunchTime"],
                "owner": tags.get("owner"),
                "env": tags.get("env"),
            })

df = pd.DataFrame(rows)
print(df.groupby(["state", "type"]).size())

# Continue from the inventory DataFrame
untagged = df[df["owner"].isna() | df["env"].isna()]
print(f"{len(untagged)} instances missing owner or env tag")

# Stopped instances still keep their EBS volumes (and storage cost)
stopped = df[df["state"] == "stopped"].copy()
stopped["days_since_launch"] = (
    pd.Timestamp.now(tz="UTC") - stopped["launched"]
).dt.days

print(stopped.sort_values("days_since_launch", ascending=False)
             [["instance_id", "type", "owner", "days_since_launch"]])

untagged.to_csv("untagged_instances.csv", index=False)

import boto3
import pandas as pd

ce = boto3.client("ce")  # Cost Explorer
resp = ce.get_cost_and_usage(
    TimePeriod={"Start": "2026-09-01", "End": "2026-11-01"},  # End is exclusive
    Granularity="MONTHLY",
    Metrics=["UnblendedCost"],
    GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}],
)

rows = []
for period in resp["ResultsByTime"]:
    for g in period["Groups"]:
        rows.append({
            "month": period["TimePeriod"]["Start"],
            "service": g["Keys"][0],
            "cost": float(g["Metrics"]["UnblendedCost"]["Amount"]),
        })

df = pd.DataFrame(rows)
wide = df.pivot_table(index="service", columns="month",
                      values="cost", aggfunc="sum", fill_value=0)
wide["change"] = wide.iloc[:, -1] - wide.iloc[:, -2]
print(wide.sort_values("change", ascending=False).head(5).round(2))

import pandas as pd

# Legacy CUR CSV column names shown; CUR 2.0 uses snake_case
# names such as line_item_product_code and line_item_unblended_cost.
cols = ["lineItem/UsageStartDate", "lineItem/ProductCode",
        "lineItem/UnblendedCost", "resourceTags/user:team"]

cur = pd.read_csv("cur-2026-10.csv.gz", usecols=cols)
cur.columns = ["start", "service", "cost", "team"]
cur["start"] = pd.to_datetime(cur["start"], utc=True)
cur["team"] = cur["team"].fillna("untagged")

by_team = cur.groupby("team")["cost"].sum().sort_values(ascending=False)
print(by_team.round(2))

daily = cur.set_index("start").resample("D")["cost"].sum()
print(daily.tail(7).round(2))

# Parquet exports: pd.read_parquet(path) (needs pyarrow installed)

import pandas as pd

# First 13 positional fields of an ALB access log entry
names = ["type", "time", "elb", "client", "target",
         "req_time", "target_time", "resp_time",
         "elb_status", "target_status", "rx_bytes", "tx_bytes", "request"]

log = pd.read_csv("alb-sample.log.gz", sep=" ", quotechar='"',
                  header=None, usecols=range(13), names=names)
log["time"] = pd.to_datetime(log["time"])
# request looks like: GET https://host:443/api/orders?id=7 HTTP/1.1
log["path"] = (log["request"].str.split(" ").str[1]
               .str.replace(r"^https?://[^/]+", "", regex=True)
               .str.split("?").str[0])
log["is_5xx"] = log["elb_status"].between(500, 599)

summary = log.groupby("path").agg(requests=("is_5xx", "size"),
                                  errors=("is_5xx", "sum"))
summary["error_rate_%"] = (100 * summary["errors"] / summary["requests"]).round(2)
print(summary.sort_values("errors", ascending=False).head(10))

# target_time is -1 when the request never reached a target
ok = log[log["target_time"] >= 0]
print(ok.groupby("path")["target_time"].quantile(0.95).nlargest(5))

import gzip, json, glob
import pandas as pd

records = []
for f in glob.glob("cloudtrail/*.json.gz"):
    with gzip.open(f, "rt") as fh:
        records.extend(json.load(fh)["Records"])

ct = pd.json_normalize(records)
ct["eventTime"] = pd.to_datetime(ct["eventTime"])

# Failed API calls by caller and action
if "errorCode" in ct.columns:
    failed = ct[ct["errorCode"].notna()]
    print(failed.groupby(["userIdentity.arn", "eventName", "errorCode"])
                .size().sort_values(ascending=False).head(10))

# Who deleted or changed security groups?
sg = ct[ct["eventName"].isin(["DeleteSecurityGroup",
                               "AuthorizeSecurityGroupIngress"])]
print(sg[["eventTime", "userIdentity.arn", "eventName", "sourceIPAddress"]])

PROJECT 1: MONTHLY COST REPORT

Goal     : Explain last month's AWS cost change in one page
Inputs   : Cost Explorer get_cost_and_usage, grouped by SERVICE
Steps    :
  1. Fetch two months of MONTHLY cost by service
  2. Pivot service x month, add change and change %
  3. Flag services with change above a threshold
  4. Save a CSV and a simple bar chart (matplotlib)
  5. Schedule weekly with a cron job or EventBridge + Lambda
Show     : Sample chart, the table, and three plain-English findings
Skills   : boto3, pandas, IAM least privilege, cost awareness

PROJECT 2: IDLE RESOURCE FINDER (multi-region)

Checks:
  [ ] EBS volumes with State = available (unattached)
  [ ] EC2 instances stopped for a long time
  [ ] Elastic IPs with no association
  [ ] Old EBS snapshots past a retention period
  [ ] Resources missing owner / env tags

Core loop:
  regions = [r["RegionName"] for r in
             ec2.describe_regions()["Regions"]]
  for each region: boto3.client("ec2", region_name=region)
      -> describe_volumes(Filters=[{"Name": "status",
                                    "Values": ["available"]}])
      -> append rows with region column

Output: one DataFrame, grouped by region and resource type

README TEMPLATE

# AWS Cost & Waste Analyzer

## Problem
Teams struggle to explain cost changes and find idle resources.

## What it does
- Pulls cost by service via Cost Explorer
- Lists unattached EBS volumes across regions
- Produces a CSV report and a chart

## Permissions needed
Read-only: ce:GetCostAndUsage, ec2:Describe*

## How to run
pip install -r requirements.txt
python analyzer.py --months 2

## Sample findings (from my practice account)
1. ...
2. ...

## What I would add next
Slack alert, tag-based showback, Athena for large CUR data