ML Engineer Career in the Insurance Industry

# Fraud detection — feature engineering.
# The core of insurance ML work: turning raw claims into signals.

import pandas as pd

claims = pd.read_csv("claims_2026.csv")

# Base features from the claim itself
claims["claim_amount"] = claims["claim_amount"].astype(float)
claims["days_to_report"] = (
    pd.to_datetime(claims["report_date"]) -
    pd.to_datetime(claims["incident_date"])
).dt.days

# Historical behaviour — the strongest fraud signal
claims["customer_claim_count_12m"] = (
    claims.groupby("customer_id")["claim_id"]
    .transform(lambda s: s.expanding().count() - 1)
)

claims["customer_avg_amount_12m"] = (
    claims.groupby("customer_id")["claim_amount"]
    .transform(lambda s: s.expanding().mean().shift())
)

# Deviation from the customer's typical claim
claims["amount_vs_history"] = (
    claims["claim_amount"] / claims["customer_avg_amount_12m"]
)

# Channel and timing signals
claims["is_weekend"] = pd.to_datetime(claims["incident_date"]).dt.dayofweek >= 5
claims["is_night_incident"] = pd.to_datetime(claims["incident_time"]).dt.hour.between(1, 4)

# Provider / garage risk (fraud clusters around a few providers)
claims["provider_risk_score"] = (
    claims.groupby("provider_id")["was_flagged_fraud"]
    .transform("mean")
)

# What this teaches:
# - Fraud signals come from history, not just the claim
# - Deviation from a customer's baseline is often the strongest signal
# - Timing and channel patterns matter
# - Provider-level features reveal fraud clusters
#
# What a hiring manager wants to hear:
# "Tell me about a feature you engineered that made a real
#  difference to the model." This is the kind of answer that
#  gets you hired.

# Claims triage — training a model with class imbalance.
# Claims triage is heavily imbalanced: most claims are simple.

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, roc_auc_score
from sklearn.preprocessing import StandardScaler
import lightgbm as lgb

df = pd.read_parquet("claims_features.parquet")

# Target: is this claim complex (1) or simple (0)?
X = df.drop(columns=["claim_id", "is_complex"])
y = df["is_complex"]

print(f"Complex rate: {y.mean():.2%}")  # e.g., 4% of claims

# Split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# LightGBM handles class imbalance well via scale_pos_weight
scale_pos_weight = (y_train == 0).sum() / (y_train == 1).sum()

model = lgb.LGBMClassifier(
    n_estimators=400,
    learning_rate=0.05,
    num_leaves=63,
    scale_pos_weight=scale_pos_weight,
    random_state=42,
)

model.fit(
    X_train, y_train,
    eval_set=[(X_test, y_test)],
    eval_metric="auc",
    callbacks=[lgb.early_stopping(50)],
)

# Evaluate
y_prob = model.predict_proba(X_test)[:, 1]
print(f"Test AUC: {roc_auc_score(y_test, y_prob):.4f}")
print(classification_report(y_test, (y_prob > 0.5).astype(int)))

# What this teaches:
# - Class imbalance is the default in insurance, not the exception
# - scale_pos_weight is often more effective than resampling
# - AUC is more useful than accuracy in this domain
# - Early stopping prevents overfitting without manual epochs

# Why it matters:
# The model does not decide who gets paid. It decides which
# claims should go to a senior adjuster first. That triage
# decision is worth real money in a claims department.

# Explainability — SHAP for regulators.
# Insurance ML has a legal constraint most ML roles do not:
# every decision may need to be explained to a regulator.

import shap
import pandas as pd
import lightgbm as lgb

model = lgb.Booster(model_file="claims_triage.txt")
df = pd.read_parquet("claims_features.parquet")
X = df.drop(columns=["claim_id", "is_complex"])

# Compute SHAP values for a batch of claims
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X)

# Global explanation — which features matter most across all claims?
shap.summary_plot(shap_values, X, plot_type="bar")

# Local explanation — why did THIS claim get flagged as complex?
sample = X.iloc[42]
sample_shap = explainer.shap_values(sample.to_frame().T)[0]

top_features = sorted(
    zip(X.columns, sample_shap),
    key=lambda x: abs(x[1]),
    reverse=True,
)[:5]

print("Top contributing features for this claim:")
for name, value in top_features:
    print(f"  {name}: {value:+.3f}")

# Sample output:
#   days_to_report: +0.412       ← long delay increased complexity
#   claim_amount: +0.301         ← high amount increased complexity
#   provider_risk_score: +0.182  ← risky provider
#   customer_claim_count_12m: -0.045
#   is_weekend: +0.031

# What this teaches:
# - SHAP explains individual predictions, not just aggregate performance
# - Regulators want to know WHY, not just WHAT
# - A model that is accurate but unexplainable is unusable in insurance

# Why it matters:
# In insurance, "the model said so" is not a valid answer.
# A claims decision must be defensible — to a customer,
# to an auditor, and to a regulator. Explainability is
# a first-class requirement, not a nice-to-have.
#
# This is the skill that separates insurance ML engineers
# from generic ML engineers.