Uncodemy vs Simplilearn Big Data Course Comparison

# PySpark — word count, the canonical Big Data starter.
# Both Uncodemy and Simplilearn teach this — the difference is
# whether a mentor reviews your code and explains the DAG.

from pyspark.sql import SparkSession
from pyspark.sql.functions import explode, split, lower, col

# 1. Create a Spark session
spark = (
    SparkSession.builder
    .appName("WordCount")
    .master("local[*]")
    .getOrCreate()
)

# 2. Load text (local file, HDFS, or S3)
df = spark.read.text("data/input.txt")

# 3. Transform — split into words, lower-case, count
words = (
    df.select(explode(split(lower(col("value")), r"\W+")).alias("word"))
      .filter(col("word") != "")
)

counts = words.groupBy("word").count().orderBy(col("count").desc())

# 4. Show top 20
counts.show(20, truncate=False)

# 5. Save output
counts.write.mode("overwrite").parquet("data/word_counts")

spark.stop()

# What this teaches:
# - SparkSession is your entry point to the cluster
# - DataFrames are lazy — nothing runs until .show() or .write()
# - explode() flattens array columns
# - The DAG (execution plan) is what runs on the cluster
#
# What a good mentor adds:
# - Why is local[*] not "the cluster"?
# - What happens if you .show() on a 10TB dataset?
# - Why parquet over CSV?
# - How would you run this on EMR?
#
# Uncodemy's mentor sessions spend 20 minutes on these.
# With Simplilearn, you may need to self-study them.

# Kafka — producer and consumer
# A hands-on module that separates "I watched a video" from
# "I actually ran a stream".

from kafka import KafkaProducer, KafkaConsumer
import json
import time

# ── PRODUCER ──
producer = KafkaProducer(
    bootstrap_servers="localhost:9092",
    value_serializer=lambda v: json.dumps(v).encode("utf-8"),
)

# Send 5 events
for i in range(5):
    event = {
        "user_id": f"user_{i}",
        "action": "page_view",
        "page": "/products",
        "ts": time.time(),
    }
    producer.send("user-events", value=event)
    print(f"Sent: {event}")

producer.flush()
producer.close()

# ── CONSUMER ──
consumer = KafkaConsumer(
    "user-events",
    bootstrap_servers="localhost:9092",
    value_deserializer=lambda m: json.loads(m.decode("utf-8")),
    auto_offset_reset="earliest",
    enable_auto_commit=True,
    group_id="analytics-group",
)

for message in consumer:
    event = message.value
    print(f"Received: {event}  [offset={message.offset}]")
    if message.offset >= 4:
        break

consumer.close()

# What this teaches:
# - Kafka is a distributed log, not a queue
# - Producers write; consumers read by offset
# - Consumer groups enable parallel processing
# - Serialization matters (JSON vs Avro vs Protobuf)
#
# What a good mentor adds:
# - Why enable_auto_commit=True is dangerous in production?
# - How do partitions affect ordering?
# - How would you scale this to 1M events/sec?
#
# These are the interview questions. Which course prepares you for them?

-- Spark SQL — real analytics query
-- The layer most Big Data teams actually use day-to-day.

-- Load raw events from S3 (or HDFS)
CREATE OR REPLACE TEMPORARY VIEW events
USING parquet
OPTIONS (path "s3://my-bucket/events/2026/09/*.parquet");

-- Revenue by day and category
SELECT
  date_trunc('day', event_time) AS day,
  product_category,
  COUNT(DISTINCT user_id) AS unique_users,
  SUM(revenue) AS revenue,
  SUM(revenue) / COUNT(DISTINCT user_id) AS revenue_per_user
FROM events
WHERE event_type = 'purchase'
  AND event_time >= '2026-09-01'
  AND event_time <  '2026-10-01'
GROUP BY 1, 2
ORDER BY 1 DESC, 4 DESC;

-- Top 5 products per category (window functions in Spark SQL)
WITH ranked AS (
  SELECT
    product_category,
    product_id,
    SUM(revenue) AS revenue,
    ROW_NUMBER() OVER (
      PARTITION BY product_category
      ORDER BY SUM(revenue) DESC
    ) AS rank_in_category
  FROM events
  WHERE event_type = 'purchase'
    AND event_time >= '2026-09-01'
  GROUP BY 1, 2
)
SELECT *
FROM ranked
WHERE rank_in_category <= 5
ORDER BY product_category, rank_in_category;

-- What this teaches:
-- - Spark SQL is ANSI SQL + Spark-specific extensions
-- - Parquet + partitioning = fast analytics
-- - Window functions work the same as in any SQL
-- - This is closer to real Big Data work than word count
--
-- The interview question: "walk me through a Spark SQL query
-- you would write for a 2TB events table." Both courses teach
-- the syntax — you need practice to answer the design part.