# ---------------------------------------------------------------------------
# meal_story.py
#
# Capstone project: a data story about the Mid-Day Meal Scheme - how
# enrolment, attendance and dropout in one state's primary schools changed
# after the scheme began in 1995 and cooked meals arrived in 2001.
#
# CBSE Class XII, Artificial Intelligence (843), Part C - Capstone Project.
# This is the curriculum's own sample data-storytelling problem.
# SDG 4 (Quality Education) and SDG 2 (Zero Hunger).
#
# Data Science Methodology, 10 steps. The analytics is DESCRIPTIVE (what
# happened) and DIAGNOSTIC (why it may have happened). Two models of the
# dropout trend are compared with leave-one-out cross validation:
#   A. one straight line for all the years
#   B. a line allowed to change slope in 2001 and again in 2012
#      (segmented regression)
# The output of the project is a data story, structured on Freytag's pyramid.
#
# Libraries: pandas, numpy, matplotlib, scikit-learn.
# ---------------------------------------------------------------------------

# %% Step 1 - Business Understanding
print("=" * 64)
print("THE MID-DAY MEAL STORY")
print("=" * 64)
print("Question : did dropout fall after the Mid-Day Meal Scheme began,")
print("           and is the change large enough to tell a story about?")
print()

# %% Step 2 - Analytic Approach
print("Analytic approach : descriptive (what happened) and diagnostic (why)")
print()

# %% Step 3 - Data Requirements
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import LeaveOneOut, cross_val_score

pd.set_option("display.width", 130)
pd.set_option("display.max_columns", 20)
required = ["year", "enrolment_lakh", "girls_share_pct", "attendance_pct", "dropout_pct", "meal_coverage_pct"]
print("Columns required :", required)

# %% Step 4 - Data Collection
data = pd.read_csv("school_data.csv")
print("Rows collected   :", len(data))
print("Years covered    :", data["year"].min(), "to", data["year"].max())
print()

# %% Step 5 - Data Understanding
print("--- Step 5: Data Understanding ---")
print("Missing values:")
print(data.isnull().sum().to_string())
print()
print(data.describe().round(1))
print()

events = {1995: "Scheme begins\n(dry rations)", 2001: "Cooked meals\n(court order)",
          2010: "RTE Act", 2020: "COVID-19\nclosures"}

# Chart 1: dropout over the years, with the events marked.
plt.figure(figsize=(9, 4.5))
plt.plot(data["year"], data["dropout_pct"], marker="o", color="#e15759", markersize=4)
for y in events:
    plt.axvline(y, color="#999999", linestyle="--", linewidth=0.8)
    plt.text(y + 0.3, 37, events[y], fontsize=8, va="top")
plt.title("Primary school dropout rate, 1990-2024")
plt.xlabel("Year")
plt.ylabel("Dropout (%)")
plt.grid(True, linestyle="--", alpha=0.3)
plt.tight_layout()
plt.savefig("chart1_dropout.png")
plt.close()

# Chart 2: enrolment and the share of girls.
plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
plt.plot(data["year"], data["enrolment_lakh"], color="#4e79a7")
plt.title("Enrolment (lakh children)")
plt.xlabel("Year")
plt.grid(True, linestyle="--", alpha=0.3)
plt.subplot(1, 2, 2)
plt.plot(data["year"], data["girls_share_pct"], color="#b07aa1")
plt.title("Girls as % of enrolment")
plt.xlabel("Year")
plt.grid(True, linestyle="--", alpha=0.3)
plt.tight_layout()
plt.savefig("chart2_enrolment.png")
plt.close()

# %% Step 6 - Data Preparation
print("--- Step 6: Data Preparation ---")
before = len(data)
data = data.drop_duplicates()
print("Duplicate rows removed       :", before - len(data))

# Attendance is missing for 1998 (records lost) and 2020 (schools closed).
# 1998 is filled by linear interpolation between 1997 and 1999. 2020 is
# left empty on purpose: there was no attendance to record, and inventing
# a value would hide the most important fact about that year.
data["attendance_pct"] = pd.to_numeric(data["attendance_pct"])
data.loc[data["year"] == 1998, "attendance_pct"] = np.nan
filled = data["attendance_pct"].interpolate()
data.loc[data["year"] == 1998, "attendance_pct"] = filled[data["year"] == 1998]
print("Attendance 1998 (interpolated) :", round(data.loc[data["year"] == 1998, "attendance_pct"].values[0], 1))
print("Attendance 2020                 : left missing - schools were closed")

# Feature engineering: the period each year belongs to.
period = []
for y in data["year"]:
    if y < 1995:
        period.append("Before the scheme")
    elif y < 2001:
        period.append("Dry rations")
    else:
        period.append("Cooked meals")
data["period"] = period
# Chart 1 shows the fall steepening in 2001 and flattening around 2012, so
# two "years since" features let a straight-line model bend at both points.
data["years_since_2001"] = np.maximum(0, data["year"] - 2001)
data["years_since_2012"] = np.maximum(0, data["year"] - 2012)
print()

summary = data.groupby("period", sort=False)[["dropout_pct", "attendance_pct", "enrolment_lakh"]].mean().round(1)
print("Average of each period:")
print(summary)
print()
print("Average dropout before the scheme :", summary.loc["Before the scheme", "dropout_pct"], "%")
print("Average dropout with cooked meals :", summary.loc["Cooked meals", "dropout_pct"], "%")
print()

corr_cov = data["meal_coverage_pct"].corr(data["dropout_pct"])
corr_att = data["meal_coverage_pct"].corr(data["attendance_pct"])
print("Correlation, meal coverage and dropout    :", round(corr_cov, 2))
print("Correlation, meal coverage and attendance :", round(corr_att, 2))
print()

# Chart 3: meal coverage against dropout, coloured by period.
colours = {"Before the scheme": "#9c9c9c", "Dry rations": "#f28e2b", "Cooked meals": "#59a14f"}
plt.figure(figsize=(7, 5))
for p in colours:
    part = data[data["period"] == p]
    plt.scatter(part["meal_coverage_pct"], part["dropout_pct"], color=colours[p], label=p, s=40)
plt.title("Meal coverage and dropout, one dot per year")
plt.xlabel("Schools serving meals (%)")
plt.ylabel("Dropout (%)")
plt.legend()
plt.grid(True, linestyle="--", alpha=0.3)
plt.tight_layout()
plt.savefig("chart3_coverage_dropout.png")
plt.close()

# Chart 4: attendance, with the 2020 gap visible.
plt.figure(figsize=(9, 4))
plt.plot(data["year"], data["attendance_pct"], marker="o", markersize=4, color="#4e79a7")
plt.axvline(2001, color="#999999", linestyle="--", linewidth=0.8)
plt.title("Average attendance (2020 missing: schools closed)")
plt.xlabel("Year")
plt.ylabel("Attendance (%)")
plt.grid(True, linestyle="--", alpha=0.3)
plt.tight_layout()
plt.savefig("chart4_attendance.png")
plt.close()

# %% Step 7 - Modelling
print("--- Step 7: Modelling ---")
# The trend is modelled on 1990-2019. 2020-2024 are left out of the fit:
# COVID-19 was an outside shock that no school-meal model should explain.
trend = data[data["year"] <= 2019]
y = trend["dropout_pct"]
X_a = trend[["year"]]
X_b = trend[["year", "years_since_2001", "years_since_2012"]]
model_a = LinearRegression().fit(X_a, y)
model_b = LinearRegression().fit(X_b, y)
print("Model A slope (all years)            :", round(model_a.coef_[0], 2), "points a year")
slope_before = model_b.coef_[0]
slope_meals = model_b.coef_[0] + model_b.coef_[1]
slope_plateau = slope_meals + model_b.coef_[2]
print("Model B slope before 2001            :", round(slope_before, 2), "points a year")
print("Model B slope 2001 to 2012           :", round(slope_meals, 2), "points a year")
print("Model B slope after 2012             :", round(slope_plateau, 2), "points a year")
print()

# %% Step 8 - Evaluation
print("--- Step 8: Evaluation ---")
# Leave-one-out cross validation: each year in turn is held out, the model
# is fitted on the other years, and the held-out year is predicted.
loo = LeaveOneOut()
err_a = np.sqrt(-cross_val_score(LinearRegression(), X_a, y, cv=loo, scoring="neg_mean_squared_error"))
err_b = np.sqrt(-cross_val_score(LinearRegression(), X_b, y, cv=loo, scoring="neg_mean_squared_error"))
rmse_a = np.sqrt((err_a ** 2).mean())
rmse_b = np.sqrt((err_b ** 2).mean())
print("Years used for the trend          :", len(trend))
print("Leave-one-out RMSE, Model A       :", round(rmse_a, 2), "points")
print("Leave-one-out RMSE, Model B       :", round(rmse_b, 2), "points")
print("R squared, Model A                :", round(model_a.score(X_a, y), 3))
print("R squared, Model B                :", round(model_b.score(X_b, y), 3))
print()

# Chart 5: the two trend models over the data.
plt.figure(figsize=(9, 4.5))
plt.scatter(trend["year"], y, color="#e15759", s=20, label="Actual dropout")
plt.plot(trend["year"], model_a.predict(X_a), color="#9c9c9c", label="Model A: one line")
plt.plot(trend["year"], model_b.predict(X_b), color="#59a14f", linewidth=2, label="Model B: slope changes in 2001 and 2012")
plt.title("Two models of the dropout trend, 1990-2019")
plt.xlabel("Year")
plt.ylabel("Dropout (%)")
plt.legend()
plt.grid(True, linestyle="--", alpha=0.3)
plt.tight_layout()
plt.savefig("chart5_models.png")
plt.close()

# Chart 6: leave-one-out error for each year.
plt.figure(figsize=(9, 4))
plt.bar(trend["year"] - 0.2, err_a, width=0.4, color="#9c9c9c", label="Model A")
plt.bar(trend["year"] + 0.2, err_b, width=0.4, color="#59a14f", label="Model B")
plt.title("Leave-one-out error for each year")
plt.xlabel("Year held out")
plt.ylabel("Error (percentage points)")
plt.legend()
plt.tight_layout()
plt.savefig("chart6_loo_errors.png")
plt.close()

# %% Step 9 - Deployment
print("--- Step 9: Deployment: the data story ---")
d_1990 = data.loc[data["year"] == 1990, "dropout_pct"].values[0]
d_2001 = data.loc[data["year"] == 2001, "dropout_pct"].values[0]
d_2012 = data.loc[data["year"] == 2012, "dropout_pct"].values[0]
d_2021 = data.loc[data["year"] == 2021, "dropout_pct"].values[0]
d_2024 = data.loc[data["year"] == 2024, "dropout_pct"].values[0]
story = [
    ["Exposition", "1990: " + str(d_1990) + "% of children left primary school early."],
    ["Rising action", "1995 dry rations; 2001 cooked meals in every school."],
    ["Climax", "Dropout fell from " + str(d_2001) + "% (2001) to " + str(d_2012) + "% (2012)."],
    ["Falling action", "A plateau after 2012; COVID pushed it to " + str(d_2021) + "% in 2021."],
    ["Resolution", "Back to " + str(d_2024) + "% by 2024 - meals helped; the last mile needs more."],
]
for stage, beat in story:
    print(stage.ljust(15), ":", beat)
print()

# Chart 7: Freytag's pyramid, carrying this story's beats.
xs = [0, 1, 2, 3, 4]
ys = [0, 1, 2, 1, 0.3]
plt.figure(figsize=(10, 4.8))
plt.plot(xs, ys, color="#4e79a7", marker="o", linewidth=2)
for i in range(5):
    plt.text(xs[i], ys[i] + 0.12, story[i][0], ha="center", fontsize=10, fontweight="bold")
    plt.text(xs[i], ys[i] - 0.25, story[i][1].replace("; ", ";\n").replace(" from", "\nfrom").replace(" - ", "\n"),
             ha="center", fontsize=7.5, va="top")
plt.ylim(-1.1, 2.5)
plt.xlim(-0.6, 4.6)
plt.axis("off")
plt.title("The story on Freytag's pyramid")
plt.tight_layout()
plt.savefig("chart7_freytag.png")
plt.close()

# Chart 8: the headline chart of the story.
plt.figure(figsize=(9, 4.8))
plt.fill_between(data["year"], data["dropout_pct"], color="#e15759", alpha=0.25)
plt.plot(data["year"], data["dropout_pct"], color="#e15759", linewidth=2)
plt.axvspan(2001, 2012, color="#59a14f", alpha=0.12)
plt.text(2006.5, 35, "Cooked meals:\nthe steepest fall", ha="center", fontsize=10, color="#2e7d32")
plt.annotate("COVID-19", xy=(2021, d_2021), xytext=(2014, 26), arrowprops=dict(arrowstyle="->"))
plt.title("Since cooked mid-day meals began, dropout has fallen by more than half")
plt.xlabel("Year")
plt.ylabel("Dropout (%)")
plt.ylim(0, 42)
plt.tight_layout()
plt.savefig("chart8_headline.png")
plt.close()

# %% Step 10 - Feedback
print("--- Step 10: Feedback ---")
# The story makes a claim that new data can test: dropout should stay near
# the post-2012 plateau. Model B's plateau-era average is compared with the
# years after COVID as they arrive.
plateau = data[(data["year"] >= 2012) & (data["year"] <= 2019)]["dropout_pct"].mean()
latest = data[data["year"] >= 2023]["dropout_pct"].mean()
print("Average dropout 2012-2019 (plateau) :", round(plateau, 1), "%")
print("Average dropout 2023-2024 (latest)  :", round(latest, 1), "%")
if abs(latest - plateau) <= 1.0:
    print("Feedback: the latest years are back on the plateau - the story's ending holds.")
else:
    print("Feedback: the latest years differ from the plateau - the story's ending must be revised.")