{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Traffic Jam Warnings from a Data Stream\n",
    "\n",
    "traffic_stream.py\n",
    "\n",
    "Capstone project: warning a city's traffic control room fifteen minutes\n",
    "before a road jams, from a non-stop stream of sensor readings.\n",
    "\n",
    "CBSE Class XII, Artificial Intelligence (843), Part C - Capstone Project.\n",
    "SDG 11 (Sustainable Cities and Communities).\n",
    "\n",
    "Data Science Methodology, 10 steps, with BIG DATA ideas: the file is never\n",
    "loaded whole. It is read in chunks, summarised map-and-reduce style, cleaned\n",
    "chunk by chunk, and learned from in two big-data ways: an online model\n",
    "(partial_fit) that learns from every chunk in turn, and a Random Forest\n",
    "trained on a 10% sample. Both are compared with a simple rule.\n",
    "\n",
    "Libraries: pandas, numpy, matplotlib, scikit-learn, joblib."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Business Understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"=\" * 64)\n",
    "print(\"TRAFFIC JAM EARLY WARNING\")\n",
    "print(\"=\" * 64)\n",
    "print(\"Problem : by the time the control room sees a jam, it has already formed\")\n",
    "print(\"Goal    : warn 15 minutes ahead, so signals can be retimed and traffic\")\n",
    "print(\"          diverted while it still helps\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Analytic Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Analytic approach : predictive - will this road jam in 15 minutes? (yes/no)\")\n",
    "print(\"Jam means         : average speed below 20 km/h\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 - Data Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import joblib\n",
    "\n",
    "FILE = \"traffic_sensors.csv\"\n",
    "CHUNK = 20000                      # rows held in memory at a time\n",
    "TEST_FROM = \"2025-07-28\"           # the last week is kept for testing\n",
    "FEATURES = [\"speed_kmph\", \"speed_drop_15min\", \"vehicles\", \"occupancy_pct\", \"rush_hour\", \"weekend\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4 - Data Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 4: Data Collection ---\")\n",
    "print(\"File size :\", round(os.path.getsize(FILE) / 1e6, 1), \"MB\")\n",
    "rows = 0\n",
    "chunks = 0\n",
    "for chunk in pd.read_csv(FILE, chunksize=CHUNK):\n",
    "    rows = rows + len(chunk)\n",
    "    chunks = chunks + 1\n",
    "print(\"Readings  :\", rows)\n",
    "print(\"Read in\", chunks, \"chunks of at most\", CHUNK, \"rows - never all at once\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5 - Data Understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MAP: summarise each chunk on its own. REDUCE: add the summaries together.\n",
    "print(\"--- Step 5: Data Understanding ---\")\n",
    "speed_total = None\n",
    "error_codes = 0\n",
    "blank_counts = 0\n",
    "jam_total = None\n",
    "for chunk in pd.read_csv(FILE, chunksize=CHUNK):\n",
    "    bad = (chunk[\"speed_kmph\"] < 0) | (chunk[\"speed_kmph\"] > 150)\n",
    "    error_codes = error_codes + int(bad.sum())\n",
    "    blank_counts = blank_counts + int(chunk[\"vehicles\"].isnull().sum())\n",
    "    good = chunk[~bad].copy()\n",
    "    t = pd.to_datetime(good[\"time\"])\n",
    "    good[\"hour\"] = t.dt.hour\n",
    "    good[\"day\"] = np.where(t.dt.dayofweek >= 5, \"Weekend\", \"Weekday\")\n",
    "    part = good.groupby([\"day\", \"hour\"])[\"speed_kmph\"].agg([\"sum\", \"count\"])           # map\n",
    "    speed_total = part if speed_total is None else speed_total.add(part, fill_value=0)  # reduce\n",
    "    jams = good.groupby([\"sensor\", \"hour\"])[\"jam_in_15_min\"].agg([\"sum\", \"count\"])\n",
    "    jam_total = jams if jam_total is None else jam_total.add(jams, fill_value=0)\n",
    "avg_speed = (speed_total[\"sum\"] / speed_total[\"count\"]).unstack(0)\n",
    "jam_share = (jam_total[\"sum\"] / jam_total[\"count\"]).unstack(1)\n",
    "overall = jam_total[\"sum\"].sum() / jam_total[\"count\"].sum()\n",
    "print(\"Speed error codes (-1 or 255) :\", error_codes)\n",
    "print(\"Blank vehicle counts          :\", blank_counts)\n",
    "print(\"Readings followed by a jam    :\", round(100 * overall, 1), \"%\")\n",
    "print(\"Slowest weekday hour          :\", str(int(avg_speed[\"Weekday\"].idxmin())) + \":00, average\",\n",
    "      round(avg_speed[\"Weekday\"].min(), 1), \"km/h\")\n",
    "print(\"Slowest weekend hour          :\", str(int(avg_speed[\"Weekend\"].idxmin())) + \":00, average\",\n",
    "      round(avg_speed[\"Weekend\"].min(), 1), \"km/h\")\n",
    "print(\"Sensor with the most jams     :\", jam_share.mean(axis=1).idxmax())\n",
    "print()\n",
    "\n",
    "# Chart 1: average speed through the day.\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.plot(avg_speed.index, avg_speed[\"Weekday\"], marker=\"o\", label=\"Weekday\", color=\"#e15759\")\n",
    "plt.plot(avg_speed.index, avg_speed[\"Weekend\"], marker=\"o\", label=\"Weekend\", color=\"#4e79a7\")\n",
    "plt.axhline(20, color=\"gray\", linestyle=\"--\", label=\"Jam below 20 km/h\")\n",
    "plt.title(\"Average speed through the day, all sensors\")\n",
    "plt.xlabel(\"Hour of the day\")\n",
    "plt.ylabel(\"km/h\")\n",
    "plt.xticks(range(0, 24, 2))\n",
    "plt.legend()\n",
    "plt.grid(True, linestyle=\"--\", alpha=0.4)\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart1_hourly_speed.png\")\n",
    "plt.close()\n",
    "\n",
    "# Chart 2: where and when jams follow.\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.imshow(jam_share.values, cmap=\"Reds\", aspect=\"auto\")\n",
    "plt.colorbar(label=\"Share of readings followed by a jam\")\n",
    "plt.yticks(range(len(jam_share)), jam_share.index)\n",
    "plt.xticks(range(0, 24, 2))\n",
    "plt.xlabel(\"Hour of the day\")\n",
    "plt.ylabel(\"Sensor\")\n",
    "plt.title(\"Where and when jams happen\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart2_jam_heatmap.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6 - Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Each chunk is cleaned on its own. A chunk's first rows need the readings\n",
    "# just before them (for the 15-minute change), so the last three readings of\n",
    "# every sensor are CARRIED from one chunk to the next.\n",
    "print(\"--- Step 6: Data Preparation ---\")\n",
    "\n",
    "\n",
    "def prepare(chunk, carry):\n",
    "    chunk = chunk.copy()\n",
    "    chunk[\"new\"] = True\n",
    "    both = pd.concat([carry, chunk])\n",
    "    before = len(both)\n",
    "    both = both.drop_duplicates(subset=[\"time\", \"sensor\"])\n",
    "    duplicates = before - len(both)\n",
    "    bad = (both[\"speed_kmph\"] < 0) | (both[\"speed_kmph\"] > 150)\n",
    "    both.loc[bad, \"speed_kmph\"] = np.nan\n",
    "    both = both.sort_values([\"sensor\", \"time\"], kind=\"stable\")\n",
    "    # An error code or a blank count is replaced by that sensor's previous reading.\n",
    "    both[\"speed_kmph\"] = both.groupby(\"sensor\")[\"speed_kmph\"].ffill()\n",
    "    both[\"vehicles\"] = both.groupby(\"sensor\")[\"vehicles\"].ffill()\n",
    "    both[\"speed_drop_15min\"] = both.groupby(\"sensor\")[\"speed_kmph\"].shift(3) - both[\"speed_kmph\"]\n",
    "    t = pd.to_datetime(both[\"time\"])\n",
    "    both[\"weekend\"] = (t.dt.dayofweek >= 5).astype(int)\n",
    "    both[\"rush_hour\"] = (t.dt.hour.isin([8, 9, 10, 17, 18, 19, 20]) & (both[\"weekend\"] == 0)).astype(int)\n",
    "    new_carry = both.groupby(\"sensor\").tail(3).drop(columns=\"speed_drop_15min\").assign(new=False)\n",
    "    ready = both[both[\"new\"] & both[\"speed_drop_15min\"].notna()].sort_values([\"time\", \"sensor\"])\n",
    "    return ready, new_carry, duplicates\n",
    "\n",
    "\n",
    "def stream(file):\n",
    "    \"\"\"Yield one cleaned chunk at a time.\"\"\"\n",
    "    carry = pd.DataFrame()\n",
    "    for chunk in pd.read_csv(file, chunksize=CHUNK):\n",
    "        ready, carry, duplicates = prepare(chunk, carry)\n",
    "        yield ready, duplicates\n",
    "\n",
    "\n",
    "scaler = StandardScaler()\n",
    "counts = np.zeros(2)\n",
    "sample = []\n",
    "duplicates_removed = 0\n",
    "cleaned_rows = 0\n",
    "for ready, dups in stream(FILE):\n",
    "    duplicates_removed = duplicates_removed + dups\n",
    "    cleaned_rows = cleaned_rows + len(ready)\n",
    "    train = ready[ready[\"time\"] < TEST_FROM]\n",
    "    if len(train) > 0:\n",
    "        scaler.partial_fit(train[FEATURES])                       # learns means bit by bit\n",
    "        counts = counts + np.bincount(train[\"jam_in_15_min\"], minlength=2)\n",
    "        sample.append(train.sample(frac=0.1, random_state=0))    # a 10% sample for the forest\n",
    "print(\"Readings sent twice, removed   :\", duplicates_removed)\n",
    "print(\"Error codes and blanks         : replaced by the sensor's previous reading\")\n",
    "print(\"Clean readings                 :\", cleaned_rows)\n",
    "print(\"Training readings (3 weeks)    :\", int(counts.sum()))\n",
    "print(\"  of which followed by a jam   :\", int(counts[1]))\n",
    "weight = counts[0] / counts[1]\n",
    "print(\"Weight given to a jam reading  :\", round(weight, 1), \"(for the online model)\")\n",
    "print()\n",
    "\n",
    "# Chart 3: one day at the sensor with the most error codes, before and after cleaning.\n",
    "raw = pd.read_csv(FILE, nrows=CHUNK)\n",
    "clean0, _, _ = prepare(raw, pd.DataFrame())\n",
    "faulty = raw[(raw[\"speed_kmph\"] < 0) | (raw[\"speed_kmph\"] > 150)]\n",
    "bad_sensor = faulty[\"sensor\"].value_counts().index[0] if len(faulty) > 0 else raw[\"sensor\"].iloc[0]\n",
    "day = faulty[faulty[\"sensor\"] == bad_sensor][\"time\"].iloc[0][:10] if len(faulty) > 0 else raw[\"time\"].iloc[0][:10]\n",
    "r = raw[(raw[\"sensor\"] == bad_sensor) & raw[\"time\"].str.startswith(day)].drop_duplicates(subset=\"time\")\n",
    "c = clean0[(clean0[\"sensor\"] == bad_sensor) & clean0[\"time\"].str.startswith(day)]\n",
    "e = r[(r[\"speed_kmph\"] < 0) | (r[\"speed_kmph\"] > 150)]\n",
    "plt.figure(figsize=(9, 4))\n",
    "plt.plot(pd.to_datetime(r[\"time\"]), r[\"speed_kmph\"], color=\"#bab0ac\", linewidth=3, label=\"As received\")\n",
    "plt.plot(pd.to_datetime(c[\"time\"]), c[\"speed_kmph\"], color=\"#4e79a7\", label=\"Cleaned\")\n",
    "plt.scatter(pd.to_datetime(e[\"time\"]), e[\"speed_kmph\"], color=\"#e15759\", zorder=3, label=\"Error code\")\n",
    "plt.title(\"Sensor \" + bad_sensor + \", \" + day + \": error codes replaced by the previous reading\")\n",
    "plt.ylabel(\"Speed, km/h\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart3_cleaning.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7 - Modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 7: Modelling ---\")\n",
    "# average=True keeps a running average of the weights, so the online model\n",
    "# does not swing with whichever chunk it saw last.\n",
    "online = SGDClassifier(loss=\"log_loss\", class_weight={0: 1, 1: weight}, average=True, random_state=0)\n",
    "for epoch in range(3):                                            # three passes over the stream\n",
    "    for ready, _ in stream(FILE):\n",
    "        train = ready[ready[\"time\"] < TEST_FROM]\n",
    "        if len(train) > 0:\n",
    "            online.partial_fit(scaler.transform(train[FEATURES]), train[\"jam_in_15_min\"], classes=[0, 1])\n",
    "print(\"Online model    : logistic regression learned with partial_fit, chunk by chunk, 3 passes\")\n",
    "small = pd.concat(sample)\n",
    "forest = RandomForestClassifier(n_estimators=200, min_samples_leaf=5, class_weight=\"balanced\", random_state=0)\n",
    "forest.fit(small[FEATURES], small[\"jam_in_15_min\"])\n",
    "print(\"Random Forest   : trained on a 10% sample of\", len(small), \"readings held in memory\")\n",
    "print(\"Rule            : warn if the road is already jammed now\")\n",
    "print(\"Online model weights:\")\n",
    "for f, w in zip(FEATURES, online.coef_[0]):\n",
    "    print(\"  \", f.ljust(18), round(w, 2))\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8 - Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 8: Evaluation ---\")\n",
    "names = [\"Rule\", \"Online model\", \"Random Forest\"]\n",
    "tally = {n: np.zeros((2, 2), dtype=int) for n in names}          # [true][predicted]\n",
    "onset = {n: np.zeros(3, dtype=int) for n in names}               # new jams warned, new jams, false alarms\n",
    "by_hour = np.zeros((24, 2))                                       # jams caught, jams total\n",
    "test_rows = 0\n",
    "for ready, _ in stream(FILE):\n",
    "    test = ready[ready[\"time\"] >= TEST_FROM]\n",
    "    if len(test) == 0:\n",
    "        continue\n",
    "    test_rows = test_rows + len(test)\n",
    "    y = test[\"jam_in_15_min\"].values\n",
    "    clear = (test[\"speed_kmph\"] >= 20).values                     # not jammed yet\n",
    "    preds = {\"Rule\": (test[\"speed_kmph\"] < 20).astype(int).values,\n",
    "             \"Online model\": online.predict(scaler.transform(test[FEATURES])),\n",
    "             \"Random Forest\": forest.predict(test[FEATURES])}\n",
    "    for n in names:\n",
    "        for a in (0, 1):\n",
    "            for b in (0, 1):\n",
    "                tally[n][a, b] += int(((y == a) & (preds[n] == b)).sum())\n",
    "        onset[n] += [int((clear & (y == 1) & (preds[n] == 1)).sum()), int((clear & (y == 1)).sum()),\n",
    "                     int((clear & (y == 0) & (preds[n] == 1)).sum())]\n",
    "    hours = pd.to_datetime(test[\"time\"]).dt.hour.values\n",
    "    for h in range(24):\n",
    "        jam = (hours == h) & (y == 1) & clear\n",
    "        by_hour[h, 0] += (preds[\"Random Forest\"][jam] == 1).sum()\n",
    "        by_hour[h, 1] += jam.sum()\n",
    "print(\"Test readings (last week) :\", test_rows)\n",
    "scores = {}\n",
    "for n in names:\n",
    "    tn, fp, fn, tp = tally[n].ravel()\n",
    "    p = tp / (tp + fp)\n",
    "    r = tp / (tp + fn)\n",
    "    scores[n] = [p, r, 2 * p * r / (p + r), onset[n][0] / onset[n][1]]\n",
    "    print(n.ljust(14), \" TP\", tp, \" FN\", fn, \" FP\", fp, \" TN\", tn)\n",
    "print()\n",
    "print(\"Model           precision  recall    F1\")\n",
    "for n in names:\n",
    "    print(f\"{n:<15} {scores[n][0]:9.3f} {scores[n][1]:7.3f} {scores[n][2]:6.3f}\")\n",
    "for n in names:\n",
    "    print(\"Precision, recall, F1 -\", n, \":\", round(scores[n][0], 3), \",\", round(scores[n][1], 3), \",\", round(scores[n][2], 3))\n",
    "print()\n",
    "print(\"A warning matters most BEFORE a jam starts. Readings where the road\")\n",
    "print(\"was still clear but jammed 15 minutes later:\", onset[\"Rule\"][1])\n",
    "for n in names:\n",
    "    print(n + \", new jams warned of :\", onset[n][0], \"(\" + str(round(100 * scores[n][3])) + \"%)\")\n",
    "    print(n + \", false alarms on clear roads :\", onset[n][2])\n",
    "best = max([\"Online model\", \"Random Forest\"], key=lambda n: scores[n][2])\n",
    "print(\"Best F1 :\", best)\n",
    "print()\n",
    "rush = [8, 9, 10, 17, 18, 19, 20]\n",
    "calm = [h for h in range(24) if h not in rush]\n",
    "for label, hrs in [(\"in rush hours \", rush), (\"at other times\", calm)]:\n",
    "    if by_hour[hrs, 1].sum() > 0:\n",
    "        print(\"Random Forest, new jams warned of\", label, \":\", round(by_hour[hrs, 0].sum() / by_hour[hrs, 1].sum(), 3))\n",
    "    else:\n",
    "        print(\"Random Forest, new jams warned of\", label, \": no new jams at these times\")\n",
    "print()\n",
    "\n",
    "# Chart 4: the three approaches compared.\n",
    "x = np.arange(3)\n",
    "plt.figure(figsize=(9, 4))\n",
    "labels = [(\"Precision\", \"#4e79a7\"), (\"Recall\", \"#e15759\"), (\"F1\", \"#59a14f\"), (\"New jams warned of\", \"#f28e2b\")]\n",
    "for k, (label, colour) in enumerate(labels):\n",
    "    plt.bar(x + (k - 1.5) * 0.2, [scores[n][k] for n in names], width=0.2, label=label, color=colour)\n",
    "plt.xticks(x, names)\n",
    "plt.ylim(0, 1)\n",
    "plt.title(\"Jam warnings on the test week\")\n",
    "plt.legend(loc=\"upper left\", fontsize=8)\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart4_models.png\")\n",
    "plt.close()\n",
    "\n",
    "# Chart 5: which new jams the forest warns of, by hour.\n",
    "plt.figure(figsize=(9, 4))\n",
    "share = by_hour[:, 0] / np.maximum(by_hour[:, 1], 1)\n",
    "plt.bar(range(24), share, color=[\"#e15759\" if h in rush else \"#9c9c9c\" for h in range(24)])\n",
    "plt.title(\"Share of new jams warned of in advance, by hour (red = weekday rush hours)\")\n",
    "plt.xlabel(\"Hour of the day\")\n",
    "plt.ylabel(\"Share warned of\")\n",
    "plt.xticks(range(0, 24, 2))\n",
    "plt.ylim(0, 1)\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart5_recall_by_hour.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9 - Deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The saved model reads the stream as it arrives, and warns on the fly.\n",
    "print(\"--- Step 9: Deployment ---\")\n",
    "joblib.dump(forest, \"jam_model.pkl\")\n",
    "model = joblib.load(\"jam_model.pkl\")\n",
    "evening = None\n",
    "for ready, _ in stream(FILE):\n",
    "    part = ready[(ready[\"sensor\"] == \"J05\") & (ready[\"time\"] >= \"2025-07-30 15:00\") & (ready[\"time\"] <= \"2025-07-30 22:00\")]\n",
    "    if len(part) > 0:\n",
    "        evening = part if evening is None else pd.concat([evening, part])\n",
    "alerts = []\n",
    "for i in range(len(evening)):\n",
    "    reading = evening.iloc[[i]]                                   # one reading arrives\n",
    "    if model.predict(reading[FEATURES])[0] == 1:\n",
    "        alerts.append(reading[\"time\"].iloc[0])\n",
    "print(\"Sensor J05, 30 July, 3 pm to 10 pm :\", len(evening), \"readings streamed\")\n",
    "print(\"Warnings issued                    :\", len(alerts))\n",
    "print(\"First warning                      :\", alerts[0] if alerts else \"none\")\n",
    "jam_times = evening[evening[\"speed_kmph\"] < 20][\"time\"]\n",
    "print(\"Road first jammed                  :\", jam_times.iloc[0] if len(jam_times) else \"never\")\n",
    "print()\n",
    "\n",
    "# Chart 6: the evening, as the control room would have seen it.\n",
    "t = pd.to_datetime(evening[\"time\"])\n",
    "plt.figure(figsize=(9, 4))\n",
    "plt.plot(t, evening[\"speed_kmph\"], color=\"#4e79a7\", label=\"Speed\")\n",
    "plt.axhline(20, color=\"gray\", linestyle=\"--\", label=\"Jam below 20 km/h\")\n",
    "warned = evening[\"time\"].isin(alerts)\n",
    "plt.scatter(t[warned], evening[\"speed_kmph\"][warned], color=\"#f28e2b\", zorder=3, label=\"Warning issued\")\n",
    "plt.title(\"Sensor J05, 30 July: warnings issued from the stream\")\n",
    "plt.ylabel(\"km/h\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart6_stream_evening.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10 - Feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Two festival shopping days, on a weekend: crowds when the model expects a\n",
    "# quiet road. Day 1 is added to the training sample, and day 2 checked again.\n",
    "print(\"--- Step 10: Feedback ---\")\n",
    "fest, _ = next(stream(\"festival_days.csv\"))\n",
    "day1 = fest[fest[\"time\"] < \"2025-10-19\"]\n",
    "day2 = fest[fest[\"time\"] >= \"2025-10-19\"]\n",
    "\n",
    "\n",
    "def check(m, d):\n",
    "    p = m.predict(d[FEATURES])\n",
    "    y = d[\"jam_in_15_min\"].values\n",
    "    clear = (d[\"speed_kmph\"] >= 20).values\n",
    "    return ((p == 1) & (y == 1)).sum() / (y == 1).sum(), ((p == 1) & (y == 1) & clear).sum() / ((y == 1) & clear).sum()\n",
    "\n",
    "\n",
    "print(\"Festival day 1, new jams warned of :\", round(check(model, day1)[1], 3))\n",
    "r_before, n_before = check(model, day2)\n",
    "print(\"Festival day 2 before retraining, recall             :\", round(r_before, 3))\n",
    "print(\"Festival day 2 before retraining, new jams warned of :\", round(n_before, 3))\n",
    "small2 = pd.concat([small, day1])\n",
    "model2 = RandomForestClassifier(n_estimators=200, min_samples_leaf=5, class_weight=\"balanced\", random_state=0)\n",
    "model2.fit(small2[FEATURES], small2[\"jam_in_15_min\"])\n",
    "r_after, n_after = check(model2, day2)\n",
    "print(\"Festival day 2 after adding day 1, recall             :\", round(r_after, 3))\n",
    "print(\"Festival day 2 after adding day 1, new jams warned of :\", round(n_after, 3))\n",
    "\n",
    "# Chart 7: festival days against an ordinary weekend, and the retraining.\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.subplot(1, 2, 1)\n",
    "fh = pd.to_datetime(fest[\"time\"]).dt.hour\n",
    "plt.plot(fest.groupby(fh)[\"speed_kmph\"].mean(), color=\"#f28e2b\", label=\"Festival weekend\")\n",
    "plt.plot(avg_speed[\"Weekend\"], color=\"#4e79a7\", label=\"Ordinary weekend\")\n",
    "plt.axhline(20, color=\"gray\", linestyle=\"--\")\n",
    "plt.title(\"Average speed by hour\")\n",
    "plt.xlabel(\"Hour\")\n",
    "plt.ylabel(\"km/h\")\n",
    "plt.legend()\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.bar([\"Before retraining\", \"After adding day 1\"], [n_before, n_after], color=[\"#9c9c9c\", \"#59a14f\"])\n",
    "plt.ylim(0, 1)\n",
    "plt.title(\"Festival day 2: new jams warned of\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart7_festival.png\")\n",
    "plt.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}