{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Districts with Similar Health Needs\n",
    "\n",
    "district_health.py\n",
    "\n",
    "Capstone project: grouping districts with similar health needs, so that\n",
    "each group gets the programme it needs instead of one plan for all.\n",
    "\n",
    "CBSE Class XII, Artificial Intelligence (843), Part C - Capstone Project.\n",
    "SDG 3 (Good Health and Well-being) and SDG 10 (Reduced Inequalities).\n",
    "\n",
    "Data Science Methodology, 10 steps. UNSUPERVISED LEARNING: the districts\n",
    "have no labels. K-means clustering finds the groups; the number of groups\n",
    "is chosen with the elbow method and the silhouette score, and the result\n",
    "is checked for stability. Each group is then described by its weakest\n",
    "indicators.\n",
    "\n",
    "Libraries: pandas, numpy, matplotlib, scikit-learn, joblib."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 - Business Understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"=\" * 64)\n",
    "print(\"DISTRICTS WITH SIMILAR HEALTH NEEDS\")\n",
    "print(\"=\" * 64)\n",
    "print(\"Problem : one state-wide plan treats every district the same,\")\n",
    "print(\"          though their health needs are very different\")\n",
    "print(\"Goal    : find groups of districts with similar needs, and what\")\n",
    "print(\"          each group needs most\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 - Analytic Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Analytic approach : descriptive - clustering (unsupervised learning)\")\n",
    "print(\"There are no labels: nobody has said which district belongs where\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 - Data Requirements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import silhouette_score, silhouette_samples, adjusted_rand_score\n",
    "import joblib\n",
    "\n",
    "INDICATORS = [\"institutional_births_pct\", \"full_immunisation_pct\", \"stunted_children_pct\", \"anaemic_women_pct\",\n",
    "              \"improved_sanitation_pct\", \"clean_fuel_pct\", \"women_literacy_pct\", \"health_insurance_pct\"]\n",
    "SHORT = [\"Births in\\nhospital\", \"Full\\nimmunisation\", \"Stunted\\nchildren\", \"Anaemic\\nwomen\",\n",
    "         \"Sanitation\", \"Clean\\nfuel\", \"Women's\\nliteracy\", \"Health\\ninsurance\"]\n",
    "LOWER_IS_BETTER = [\"stunted_children_pct\", \"anaemic_women_pct\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4 - Data Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"districts.csv\")\n",
    "print(\"Districts collected :\", len(data))\n",
    "print(\"Indicators          :\", len(INDICATORS))\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5 - Data Understanding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 5: Data Understanding ---\")\n",
    "print(\"Blank cells in each indicator:\")\n",
    "print(data[INDICATORS].isnull().sum().to_string())\n",
    "print(\"Values above 100 percent :\", int((data[INDICATORS] > 100).sum().sum()))\n",
    "print()\n",
    "\n",
    "# Chart 1: how the indicators move together.\n",
    "corr = data[INDICATORS].corr()\n",
    "plt.figure(figsize=(8, 6.5))\n",
    "plt.imshow(corr, cmap=\"RdBu\", vmin=-1, vmax=1)\n",
    "for a in range(len(INDICATORS)):\n",
    "    for b in range(len(INDICATORS)):\n",
    "        plt.text(b, a, round(corr.iloc[a, b], 2), ha=\"center\", va=\"center\", fontsize=8)\n",
    "plt.xticks(range(len(INDICATORS)), SHORT, fontsize=8)\n",
    "plt.yticks(range(len(INDICATORS)), SHORT, fontsize=8)\n",
    "plt.colorbar(label=\"Correlation\")\n",
    "plt.title(\"How the indicators move together\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart1_correlation.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6 - Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 6: Data Preparation ---\")\n",
    "before = len(data)\n",
    "data = data.drop_duplicates(subset=\"district\").reset_index(drop=True)\n",
    "print(\"District entered twice, removed :\", before - len(data))\n",
    "typo = data[INDICATORS] > 100\n",
    "print(\"Values typed without the decimal point, fixed :\", int(typo.sum().sum()))\n",
    "for col in INDICATORS:\n",
    "    data.loc[data[col] > 100, col] = data[col] / 10        # 912 -> 91.2\n",
    "medians = data[INDICATORS].median()\n",
    "print(\"Blank cells filled with the indicator's median :\", int(data[INDICATORS].isnull().sum().sum()))\n",
    "data[INDICATORS] = data[INDICATORS].fillna(medians)\n",
    "print()\n",
    "print(\"Spread (standard deviation) of each indicator:\")\n",
    "print(data[INDICATORS].std().round(1).to_string())\n",
    "# K-means measures distance. Without scaling, the indicators that vary most\n",
    "# (clean fuel, insurance) would decide the groups on their own.\n",
    "scaler = StandardScaler()\n",
    "X = scaler.fit_transform(data[INDICATORS])\n",
    "print(\"Every indicator rescaled to mean 0 and standard deviation 1\")\n",
    "print()\n",
    "\n",
    "# Chart 2: the spread of every indicator.\n",
    "plt.figure(figsize=(10, 4.5))\n",
    "plt.boxplot([data[c] for c in INDICATORS])\n",
    "plt.xticks(range(1, len(INDICATORS) + 1), SHORT, fontsize=8)\n",
    "plt.ylabel(\"Percent\")\n",
    "plt.title(\"The eight indicators across all districts\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart2_spread.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7 - Modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 7: Modelling ---\")\n",
    "print(\"k   inertia  silhouette\")\n",
    "inertias = []\n",
    "silhouettes = []\n",
    "ks = range(2, 9)\n",
    "for k in ks:\n",
    "    km = KMeans(n_clusters=k, n_init=10, random_state=0).fit(X)\n",
    "    inertias.append(km.inertia_)\n",
    "    silhouettes.append(silhouette_score(X, km.labels_))\n",
    "    print(k, \" \", round(km.inertia_, 1), \"  \", round(silhouettes[-1], 4))\n",
    "# When two values of k have almost the same silhouette, the larger one is\n",
    "# kept: more groups mean more targeted programmes, at no cost in fit.\n",
    "best = max(silhouettes)\n",
    "K = max(k for k, s in zip(ks, silhouettes) if s >= best - 0.01)\n",
    "print(\"Highest silhouette :\", round(best, 4))\n",
    "print(\"Number of groups chosen :\", K)\n",
    "print(\"(the most groups with a silhouette within 0.01 of the highest)\")\n",
    "model = KMeans(n_clusters=K, n_init=10, random_state=0).fit(X)\n",
    "print()\n",
    "\n",
    "# Chart 3: the elbow and the silhouette.\n",
    "plt.figure(figsize=(10, 4))\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(list(ks), inertias, marker=\"o\", color=\"#4e79a7\")\n",
    "plt.axvline(K, color=\"gray\", linestyle=\"--\")\n",
    "plt.title(\"Elbow method: inertia\")\n",
    "plt.xlabel(\"Number of groups (k)\")\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(list(ks), silhouettes, marker=\"o\", color=\"#e15759\")\n",
    "plt.axvline(K, color=\"gray\", linestyle=\"--\")\n",
    "plt.title(\"Silhouette score\")\n",
    "plt.xlabel(\"Number of groups (k)\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart3_elbow_silhouette.png\")\n",
    "plt.close()\n",
    "\n",
    "# Name the groups in order of need: Group 1 has the greatest need.\n",
    "sign = np.array([-1 if c in LOWER_IS_BETTER else 1 for c in INDICATORS])\n",
    "wellbeing = (model.cluster_centers_ * sign).mean(axis=1)    # high = better off\n",
    "order = np.argsort(wellbeing)\n",
    "group_of = {old: new + 1 for new, old in enumerate(order)}\n",
    "data[\"group\"] = [group_of[g] for g in model.labels_]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8 - Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 8: Evaluation ---\")\n",
    "print(\"Silhouette score :\", round(silhouette_score(X, model.labels_), 3))\n",
    "same = []\n",
    "for seed in range(1, 11):\n",
    "    other = KMeans(n_clusters=K, n_init=10, random_state=seed).fit(X)\n",
    "    same.append(adjusted_rand_score(model.labels_, other.labels_))\n",
    "print(\"Lowest agreement with 10 runs from other starting points (adjusted Rand index) :\", round(min(same), 3))\n",
    "print(\"Districts in each group:\")\n",
    "print(data[\"group\"].value_counts().sort_index().to_string())\n",
    "print(\"Group sizes :\", \", \".join(str(n) for n in data[\"group\"].value_counts().sort_index()))\n",
    "print()\n",
    "profile = data.groupby(\"group\")[INDICATORS].mean().round(1)\n",
    "print(\"Average of each indicator in each group:\")\n",
    "print(profile.T.to_string())\n",
    "print()\n",
    "state = data[INDICATORS].mean()\n",
    "need = {}\n",
    "for g in profile.index:\n",
    "    gap = (profile.loc[g] - state) / data[INDICATORS].std() * sign   # negative = worse than average\n",
    "    weakest = gap.sort_values().index[:2]\n",
    "    need[g] = [w for w in weakest if gap[w] < 0]\n",
    "    names = [SHORT[INDICATORS.index(w)].replace(\"\\n\", \" \").lower() for w in need[g]]\n",
    "    print(\"Group\", g, \"- weakest compared with the state :\", \", \".join(names) if names else \"none - above average on all\")\n",
    "sil = silhouette_samples(X, model.labels_)\n",
    "data[\"silhouette\"] = sil\n",
    "print(\"Districts that fit their group poorly (silhouette below 0) :\", int((sil < 0).sum()))\n",
    "print()\n",
    "\n",
    "# Chart 4: each group's profile.\n",
    "z = (profile - state) / data[INDICATORS].std() * sign\n",
    "plt.figure(figsize=(10, 3.2 + 0.4 * K))\n",
    "plt.imshow(z.values, cmap=\"RdYlGn\", vmin=-2, vmax=2, aspect=\"auto\")\n",
    "for a in range(K):\n",
    "    for b in range(len(INDICATORS)):\n",
    "        plt.text(b, a, profile.iloc[a, b], ha=\"center\", va=\"center\", fontsize=9)\n",
    "plt.xticks(range(len(INDICATORS)), SHORT, fontsize=8)\n",
    "plt.yticks(range(K), [\"Group \" + str(g) for g in profile.index])\n",
    "plt.colorbar(label=\"Better (green) or worse (red) than the state average\")\n",
    "plt.title(\"What each group looks like (numbers are group averages, %)\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart4_profiles.png\")\n",
    "plt.close()\n",
    "\n",
    "# Chart 5: the groups on two of the indicators.\n",
    "colours = [\"#e15759\", \"#f28e2b\", \"#edc948\", \"#59a14f\", \"#4e79a7\", \"#b07aa1\", \"#9c755f\", \"#bab0ac\"]\n",
    "plt.figure(figsize=(7.5, 5))\n",
    "for g in range(1, K + 1):\n",
    "    part = data[data[\"group\"] == g]\n",
    "    plt.scatter(part[\"clean_fuel_pct\"], part[\"stunted_children_pct\"], s=18, color=colours[g - 1], label=\"Group \" + str(g))\n",
    "plt.xlabel(\"Households using clean cooking fuel, %\")\n",
    "plt.ylabel(\"Stunted children, %\")\n",
    "plt.title(\"The districts on two of the eight indicators\")\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart5_scatter.png\")\n",
    "plt.close()\n",
    "\n",
    "# Chart 6: how well each district fits its group.\n",
    "plt.figure(figsize=(8, 4.5))\n",
    "y0 = 0\n",
    "for g in range(1, K + 1):\n",
    "    s = np.sort(data.loc[data[\"group\"] == g, \"silhouette\"].values)\n",
    "    plt.barh(range(y0, y0 + len(s)), s, height=1.0, color=colours[g - 1], label=\"Group \" + str(g))\n",
    "    y0 = y0 + len(s) + 5\n",
    "plt.axvline(0, color=\"black\", linewidth=0.8)\n",
    "plt.title(\"Silhouette of every district, by group\")\n",
    "plt.xlabel(\"Silhouette (near 1 = fits its group well, below 0 = closer to another group)\")\n",
    "plt.yticks([])\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart6_silhouette.png\")\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9 - Deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 9: Deployment ---\")\n",
    "joblib.dump({\"medians\": medians, \"scaler\": scaler, \"model\": model, \"group_of\": group_of}, \"district_groups.pkl\")\n",
    "saved = joblib.load(\"district_groups.pkl\")\n",
    "\n",
    "\n",
    "def group_for(values):\n",
    "    row = pd.DataFrame([values], columns=INDICATORS).fillna(saved[\"medians\"])\n",
    "    label = saved[\"model\"].predict(saved[\"scaler\"].transform(row))[0]\n",
    "    return saved[\"group_of\"][label]\n",
    "\n",
    "\n",
    "new_district = [78, 64, 36, 60, 64, 33, 66, None]\n",
    "g = group_for(new_district)\n",
    "print(\"A newly formed district, insurance figure not yet known : Group\", g)\n",
    "print(\"Programme focus for Group\", g, \":\", \", \".join(SHORT[INDICATORS.index(w)].replace(\"\\n\", \" \").lower() for w in need[g]))\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10 - Feedback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"--- Step 10: Feedback ---\")\n",
    "nxt = pd.read_csv(\"districts_next_survey.csv\")\n",
    "nxt[\"group\"] = [group_for(list(r)) for r in nxt[INDICATORS].values]\n",
    "both = data[[\"district\", \"group\"]].merge(nxt[[\"district\", \"group\"]], on=\"district\", suffixes=(\"_before\", \"_after\"))\n",
    "moved = both[both[\"group_before\"] != both[\"group_after\"]]\n",
    "print(\"Districts in the next survey        :\", len(nxt))\n",
    "print(\"Districts that changed group        :\", len(moved))\n",
    "print(\"  moved to a better-off group       :\", int((moved[\"group_after\"] > moved[\"group_before\"]).sum()))\n",
    "print(\"  moved to a group in greater need  :\", int((moved[\"group_after\"] < moved[\"group_before\"]).sum()))\n",
    "print(\"Districts in Group 1 before         :\", int((both[\"group_before\"] == 1).sum()))\n",
    "print(\"Districts in Group 1 after          :\", int((both[\"group_after\"] == 1).sum()))\n",
    "print(\"Districts that jumped two or more groups:\")\n",
    "jump = moved[moved[\"group_after\"] - moved[\"group_before\"] >= 2]\n",
    "for _, r in jump.iterrows():\n",
    "    print(\"  \", r[\"district\"], \": Group\", r[\"group_before\"], \"-> Group\", r[\"group_after\"])\n",
    "\n",
    "# Chart 7: where each district went.\n",
    "table = pd.crosstab(both[\"group_before\"], both[\"group_after\"]).reindex(index=range(1, K + 1), columns=range(1, K + 1), fill_value=0)\n",
    "plt.figure(figsize=(6, 5))\n",
    "plt.imshow(table.values, cmap=\"Blues\")\n",
    "for a in range(K):\n",
    "    for b in range(K):\n",
    "        plt.text(b, a, table.iloc[a, b], ha=\"center\", va=\"center\",\n",
    "                 color=\"white\" if table.iloc[a, b] > table.values.max() / 2 else \"black\")\n",
    "plt.xticks(range(K), [\"Group \" + str(g) for g in range(1, K + 1)])\n",
    "plt.yticks(range(K), [\"Group \" + str(g) for g in range(1, K + 1)])\n",
    "plt.xlabel(\"Group in the next survey\")\n",
    "plt.ylabel(\"Group before\")\n",
    "plt.title(\"Feedback: which group each district is in now\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"chart7_next_survey.png\")\n",
    "plt.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}