Why Fintech Companies Are Hiring More Data Engineers in 2026

from collections import defaultdict, deque
from datetime import datetime, timedelta

WINDOW = timedelta(minutes=5)
recent = defaultdict(deque)

def on_event(account: str, ts: datetime) -> int:
    """Return number of transactions for this account in the last 5 minutes."""
    q = recent[account]
    q.append(ts)
    while q and ts - q[0] > WINDOW:
        q.popleft()
    return len(q)

t0 = datetime(2026, 11, 5, 10, 0)
for m in [0, 1, 2, 3, 9]:
    print(m, on_event("acc-1", t0 + timedelta(minutes=m)))

# pip install kafka-python
import json
from kafka import KafkaConsumer

consumer = KafkaConsumer(
    "transactions",
    bootstrap_servers="localhost:9092",
    group_id="txn-aggregator",
    value_deserializer=lambda b: json.loads(b.decode()),
    auto_offset_reset="earliest",
)

for msg in consumer:
    txn = msg.value
    if txn["status"] == "FAILED":
        print("failed txn:", txn["txn_id"], txn.get("error_code"))

-- Transactions in our ledger but missing from the partner file
SELECT l.txn_id, l.amount
FROM ledger l
LEFT JOIN partner_settlement p ON p.txn_id = l.txn_id
WHERE l.txn_date = '2026-11-05'
  AND l.status = 'SUCCESS'
  AND p.txn_id IS NULL;

-- Amount mismatches between the two
SELECT l.txn_id, l.amount AS ledger_amt, p.amount AS partner_amt
FROM ledger l
JOIN partner_settlement p ON p.txn_id = l.txn_id
WHERE l.amount <> p.amount;

import pandas as pd

ledger = pd.DataFrame({"txn_id": [1, 2, 3, 4], "amount": [100, 250, 75, 300]})
partner = pd.DataFrame({"txn_id": [1, 2, 4], "amount": [100, 240, 300]})

m = ledger.merge(partner, on="txn_id", how="outer",
                 suffixes=("_ledger", "_partner"), indicator=True)
missing = m[m["_merge"] != "both"]
mismatch = m[(m["_merge"] == "both") & (m["amount_ledger"] != m["amount_partner"])]
print("missing:\n", missing[["txn_id", "_merge"]])
print("mismatch:\n", mismatch[["txn_id", "amount_ledger", "amount_partner"]])

import pandas as pd

def check(df: pd.DataFrame) -> list:
    issues = []
    if df["txn_id"].duplicated().any():
        issues.append("duplicate txn_id")
    ok = df["status"] == "SUCCESS"
    if (df.loc[ok, "amount"] <= 0).any():
        issues.append("non-positive amount on successful txn")
    if df["txn_time"].isna().any():
        issues.append("missing txn_time")
    return issues

df = pd.DataFrame({
    "txn_id": [1, 2, 2], "status": ["SUCCESS", "SUCCESS", "FAILED"],
    "amount": [100, -5, 50], "txn_time": ["10:00", None, "10:02"],
})
print(check(df) or "all checks passed")

import hashlib
import pandas as pd

SALT = "load-from-secret-store"   # never hardcode in real code

def mask_account(acc: str) -> str:
    return "X" * (len(acc) - 4) + acc[-4:]

def pseudonymise(value: str) -> str:
    return hashlib.sha256((SALT + value).encode()).hexdigest()[:12]

df = pd.DataFrame({"account": ["123456789012"], "phone": ["9876543210"]})
safe = pd.DataFrame({
    "account_masked": df["account"].map(mask_account),
    "user_key": df["phone"].map(pseudonymise),
})
print(safe)

Project: Payments data pipeline (dummy data)

1. Generate transactions with Python (faker + random)
2. Load raw files into a local warehouse (DuckDB or Postgres)
3. Build cleaned and daily summary tables with SQL
4. Add quality checks (duplicates, amounts, freshness)
5. Reconcile against a generated partner file
6. Mask account numbers in the reporting view
7. Schedule with cron or Airflow

raw.transactions   (as received, append-only)
stg.transactions   (typed, deduplicated)
mart.daily_summary (date, partner, status, count, amount)
mart.recon_issues  (txn_id, issue_type, detected_at)
rpt.transactions_masked (view for analysts)

# Payments Data Pipeline

## Problem
Reliable daily payment reporting with reconciliation.

## Architecture
Generate -> raw -> staging -> marts -> masked views

## Quality checks
List each check and what happens on failure.

## Reconciliation results
Sample of issues found in dummy data.

## How to run