A Grievance Desk that Reads

# ---------------------------------------------------------------------------
# grievance_desk.py
#
# Capstone project: a grievance desk that reads each citizen's complaint and
# sends it to the right department, so it is not left waiting in a pile.
#
# CBSE Class XII, Artificial Intelligence (843), Part C - Capstone Project.
# SDG 16 (Peace, Justice and Strong Institutions).
#
# Data Science Methodology, 10 steps. NATURAL LANGUAGE PROCESSING: the text
# is cleaned, turned into numbers with TF-IDF, and classified into six
# departments. Naive Bayes is compared with Logistic Regression, with
# stratified 5-fold cross validation. Grievances the model is unsure about
# go to a human officer.
#
# Libraries: pandas, numpy, matplotlib, scikit-learn, joblib.
# ---------------------------------------------------------------------------

# %% Step 1 - Business Understanding
print("=" * 64)
print("THE GRIEVANCE DESK")
print("=" * 64)
print("Problem : grievances wait for days before someone reads them and")
print("          forwards them to the right department")
print("Goal    : read each grievance and route it at once; send the")
print("          unclear ones to a human officer")
print()

# %% Step 2 - Analytic Approach
print("Analytic approach : predictive - text classification into 6 departments")
print()

# %% Step 3 - Data Requirements
import string
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score
import joblib

# %% Step 4 - Data Collection
data = pd.read_csv("grievances.csv", keep_default_na=False)
print("Grievances collected :", len(data))
print()

# %% Step 5 - Data Understanding
print("--- Step 5: Data Understanding ---")
print("Department names as typed:")
print(data["department"].value_counts().to_string())
print()

# %% Step 6 - Data Preparation
print("--- Step 6: Data Preparation ---")
before = len(data)
data = data.drop_duplicates(subset="id")
print("Grievances submitted twice, removed :", before - len(data))
blank = data["grievance"].str.strip() == ""
print("Blank grievances removed            :", int(blank.sum()))
data = data[~blank].copy()
data["department"] = data["department"].str.strip().str.title()
print("Department names after fixing case  :", data["department"].nunique())

STOP = {"a", "an", "the", "in", "on", "at", "of", "to", "is", "are", "and", "for", "from", "near",
        "my", "our", "this", "it", "by", "has", "was", "very", "please", "kindly", "since",
        "so", "no", "not", "we", "nobody", "again", "every", "last", "take", "action", "help",
        "soon", "solve", "urgent", "request", "quick", "listening", "be", "days", "week", "month"}


def clean(text):
    text = text.lower()
    for mark in string.punctuation:
        text = text.replace(mark, " ")
    words = [w for w in text.split() if w not in STOP and not w.isdigit()]
    return " ".join(words)


data["clean"] = data["grievance"].apply(clean)
print("Example before cleaning :", data["grievance"].iloc[2])
print("Example after cleaning  :", data["clean"].iloc[2])
data["words"] = data["clean"].str.split().str.len()
print("Average words per grievance after cleaning :", round(data["words"].mean(), 1))
print()

depts = sorted(data["department"].unique())
counts = data["department"].value_counts().reindex(depts)
print("Grievances per department:")
print(counts.to_string())
print()

# Chart 1: grievances per department.
plt.figure(figsize=(8, 4))
plt.bar(depts, counts.values, color="#4e79a7")
plt.title("Grievances received by each department")
plt.ylabel("Grievances")
plt.xticks(rotation=20)
plt.tight_layout()
plt.savefig("chart1_departments.png")
plt.close()

# Chart 2: how long the grievances are.
plt.figure(figsize=(7, 4))
plt.hist(data["words"], bins=range(1, data["words"].max() + 2), color="#59a14f", edgecolor="white")
plt.title("Most grievances are short: words left after cleaning")
plt.xlabel("Words")
plt.ylabel("Grievances")
plt.tight_layout()
plt.savefig("chart2_lengths.png")
plt.close()

# Chart 3: the commonest words in each department.
plt.figure(figsize=(11, 6))
for k, d in enumerate(depts):
    words = " ".join(data[data["department"] == d]["clean"]).split()
    top = pd.Series(words).value_counts().head(6)
    plt.subplot(2, 3, k + 1)
    plt.barh(top.index[::-1], top.values[::-1], color="#f28e2b")
    plt.title(d)
plt.suptitle("The commonest words in each department's grievances")
plt.tight_layout()
plt.savefig("chart3_top_words.png")
plt.close()

# %% Step 7 - Modelling
print("--- Step 7: Modelling ---")
X_train, X_test, y_train, y_test = train_test_split(
    data["clean"], data["department"], test_size=0.2, stratify=data["department"], random_state=4)
print("Training grievances :", len(X_train))
print("Testing grievances  :", len(X_test))
bayes = make_pipeline(TfidfVectorizer(), MultinomialNB())
logistic = make_pipeline(TfidfVectorizer(), LogisticRegression(max_iter=2000))
bayes.fit(X_train, y_train)
logistic.fit(X_train, y_train)
vocab = bayes.named_steps["tfidfvectorizer"].get_feature_names_out()
print("Vocabulary learned (distinct words) :", len(vocab))
example = bayes.named_steps["tfidfvectorizer"].transform([X_train.iloc[0]])
print("TF-IDF example :", X_train.iloc[0])
for j in example.nonzero()[1]:
    print("   ", vocab[j], ":", round(example[0, j], 3))
print()

# %% Step 8 - Evaluation
print("--- Step 8: Evaluation ---")
pred_b = bayes.predict(X_test)
pred_l = logistic.predict(X_test)
print("Accuracy, Naive Bayes         :", round(accuracy_score(y_test, pred_b), 4))
print("Accuracy, Logistic Regression :", round(accuracy_score(y_test, pred_l), 4))
print("Macro F1, Naive Bayes         :", round(f1_score(y_test, pred_b, average="macro"), 4))
print("Macro F1, Logistic Regression :", round(f1_score(y_test, pred_l, average="macro"), 4))
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)
cv_b = cross_val_score(make_pipeline(TfidfVectorizer(), MultinomialNB()), data["clean"], data["department"],
                       cv=folds, scoring="f1_macro")
cv_l = cross_val_score(make_pipeline(TfidfVectorizer(), LogisticRegression(max_iter=2000)), data["clean"],
                       data["department"], cv=folds, scoring="f1_macro")
print("Macro F1 in each fold, Naive Bayes         :", np.round(cv_b, 3))
print("Macro F1 in each fold, Logistic Regression :", np.round(cv_l, 3))
print("Mean CV macro F1, Naive Bayes         :", round(cv_b.mean(), 4))
print("Mean CV macro F1, Logistic Regression :", round(cv_l.mean(), 4))
best_name = "Logistic Regression" if cv_l.mean() >= cv_b.mean() else "Naive Bayes"
best = logistic if best_name == "Logistic Regression" else bayes
pred = pred_l if best_name == "Logistic Regression" else pred_b
print("Chosen model :", best_name)
print()

prec = precision_score(y_test, pred, labels=depts, average=None)
rec = recall_score(y_test, pred, labels=depts, average=None)
print("Department      precision  recall")
for d, p, r in zip(depts, prec, rec):
    print(f"{d:<15} {p:9.3f} {r:7.3f}")
print()
wrong = [i for i in range(len(y_test)) if pred[i] != y_test.iloc[i]]
print("Grievances sent to the wrong department (test set) :", len(wrong))
print("The first eight:")
for i in wrong[:8]:
    print("  ", repr(data.loc[y_test.index[i], "grievance"]), "| true:", y_test.iloc[i], "| sent to:", pred[i])
print()

# Chart 4: the confusion matrix.
cm = confusion_matrix(y_test, pred, labels=depts)
plt.figure(figsize=(7, 6))
plt.imshow(cm, cmap="Blues")
for a in range(len(depts)):
    for b in range(len(depts)):
        if cm[a, b] > 0:
            plt.text(b, a, cm[a, b], ha="center", va="center",
                     color="white" if cm[a, b] > cm.max() / 2 else "black")
plt.xticks(range(len(depts)), depts, rotation=30)
plt.yticks(range(len(depts)), depts)
plt.xlabel("Sent to")
plt.ylabel("True department")
plt.title(best_name + ": confusion matrix (test set)")
plt.tight_layout()
plt.savefig("chart4_confusion.png")
plt.close()

# Chart 5: precision and recall for each department.
x = np.arange(len(depts))
plt.figure(figsize=(8.5, 4))
plt.bar(x - 0.2, prec, width=0.4, label="Precision", color="#4e79a7")
plt.bar(x + 0.2, rec, width=0.4, label="Recall", color="#e15759")
plt.xticks(x, depts, rotation=20)
plt.ylim(0.8, 1.01)
plt.title("Precision and recall for each department")
plt.legend(loc="lower right")
plt.tight_layout()
plt.savefig("chart5_per_department.png")
plt.close()

# Chart 6: macro F1 in each fold.
x = np.arange(5)
plt.figure(figsize=(7.5, 4))
plt.bar(x - 0.2, cv_b, width=0.4, label="Naive Bayes", color="#9c9c9c")
plt.bar(x + 0.2, cv_l, width=0.4, label="Logistic Regression", color="#76b7b2")
plt.xticks(x, ["Fold 1", "Fold 2", "Fold 3", "Fold 4", "Fold 5"])
plt.ylim(0.8, 1.0)
plt.ylabel("Macro F1")
plt.title("Stratified 5-fold cross validation")
plt.legend(loc="lower right")
plt.tight_layout()
plt.savefig("chart6_kfold.png")
plt.close()

# How sure is the model? Its highest probability for each test grievance.
sure = best.predict_proba(X_test).max(axis=1)
right = pred == y_test.values
print("Average confidence when right :", round(sure[right].mean(), 3))
if (~right).sum() > 0:
    print("Average confidence when wrong :", round(sure[~right].mean(), 3))
else:
    print("Average confidence when wrong : none were wrong")
THRESHOLD = 0.5
print("Test grievances below", THRESHOLD, "confidence :", int((sure < THRESHOLD).sum()))
print("Wrong routings caught by the threshold :", int((~right & (sure < THRESHOLD)).sum()), "of", int((~right).sum()))
auto_test = sure >= THRESHOLD
print("Accuracy of the automatic routings :", round(right[auto_test].mean(), 4))
print()

# Chart 7: confidence of right and wrong routings.
plt.figure(figsize=(7.5, 4))
plt.hist([sure[right], sure[~right]], bins=np.linspace(0, 1, 21), stacked=True,
         color=["#59a14f", "#e15759"], label=["Routed correctly", "Routed wrongly"])
plt.axvline(THRESHOLD, color="black", linestyle="--", label="Send to an officer below this")
plt.title("How sure the model was about each test grievance")
plt.xlabel("Highest probability")
plt.ylabel("Grievances")
plt.legend(loc="upper left")
plt.tight_layout()
plt.savefig("chart7_confidence.png")
plt.close()

# %% Step 9 - Deployment
print("--- Step 9: Deployment ---")
joblib.dump(best, "grievance_model.pkl")
model = joblib.load("grievance_model.pkl")


def route(text):
    cleaned = clean(text)
    probs = model.predict_proba([cleaned])[0]
    top = probs.argmax()
    if probs[top] < THRESHOLD:
        return "Human officer", probs[top]
    return model.classes_[top], probs[top]


for text in ["The street light outside my house is not working for a week",
             "Garbage is lying on the road and the drain is blocked",
             "My pension has not come for two months"]:
    dept, p = route(text)
    print(repr(text), "->", dept, "(confidence", str(round(p, 2)) + ")")
print()

# %% Step 10 - Feedback
print("--- Step 10: Feedback ---")
new = pd.read_csv("new_month.csv", keep_default_na=False)
print("Grievances in the new month :", len(new))
sent = [route(t)[0] for t in new["grievance"]]
new["sent_to"] = sent
known = new["department"].isin(model.classes_)
auto = new["sent_to"] != "Human officer"
print("Routed automatically        :", int(auto.sum()))
print("Sent to a human officer     :", int((~auto).sum()))
print("Automatic routings correct  :", int((new["sent_to"] == new["department"])[auto].sum()), "of", int(auto.sum()))
print("Departments never seen in training :", sorted(new.loc[~known, "department"].unique()))
print("Of these grievances, sent to an officer :", int((~auto & ~known).sum()), "of", int((~known).sum()))
for i in new.index[~known]:
    print("  ", repr(new.loc[i, "grievance"]), "->", new.loc[i, "sent_to"])

# Chart 8: what happened to the new month's grievances.
outcome = pd.Series(np.where(~auto, "Sent to an officer",
                    np.where(new["sent_to"] == new["department"], "Routed correctly", "Routed wrongly")))
outcome = outcome.value_counts().reindex(["Routed correctly", "Routed wrongly", "Sent to an officer"], fill_value=0)
plt.figure(figsize=(7, 4))
plt.bar(outcome.index, outcome.values, color=["#59a14f", "#e15759", "#edc948"])
for k, v in enumerate(outcome.values):
    plt.text(k, v + 2, str(v), ha="center")
plt.title("Feedback: the new month's grievances")
plt.ylabel("Grievances")
plt.tight_layout()
plt.savefig("chart8_new_month.png")
plt.close()

================================================================
THE GRIEVANCE DESK
================================================================
Problem : grievances wait for days before someone reads them and
          forwards them to the right department
Goal    : read each grievance and route it at once; send the
          unclear ones to a human officer

Analytic approach : predictive - text classification into 6 departments

Grievances collected : 1752

--- Step 5: Data Understanding ---
Department names as typed:
department
Water Supply    364
Sanitation      341
Electricity     339
Roads           311
Health          206
Education       177
roads             4
ELECTRICITY       3
SANITATION        2
WATER SUPPLY      2
education         1
water supply      1
health            1

--- Step 6: Data Preparation ---
Grievances submitted twice, removed : 12
Blank grievances removed            : 9
Department names after fixing case  : 6
Example before cleaning : Water tanker did not come to nehru park area. please help
Example after cleaning  : water tanker did come nehru park area
Average words per grievance after cleaning : 5.2

Grievances per department:
department
Education       177
Electricity     336
Health          205
Roads           310
Sanitation      337
Water Supply    366

--- Step 7: Modelling ---
Training grievances : 1384
Testing grievances  : 347
Vocabulary learned (distinct words) : 404
TF-IDF example : low water pressure old city
    city : 0.416
    low : 0.529
    old : 0.416
    pressure : 0.536
    water : 0.295

--- Step 8: Evaluation ---
Accuracy, Naive Bayes         : 0.9164
Accuracy, Logistic Regression : 0.9222
Macro F1, Naive Bayes         : 0.9152
Macro F1, Logistic Regression : 0.9216
Macro F1 in each fold, Naive Bayes         : [0.878 0.92  0.935 0.908 0.912]
Macro F1 in each fold, Logistic Regression : [0.881 0.912 0.927 0.934 0.909]
Mean CV macro F1, Naive Bayes         : 0.9106
Mean CV macro F1, Logistic Regression : 0.9126
Chosen model : Logistic Regression

Department      precision  recall
Education           0.943   0.917
Electricity         0.917   0.985
Health              0.973   0.878
Roads               0.883   0.855
Sanitation          0.889   0.941
Water Supply        0.958   0.932

Grievances sent to the wrong department (test set) : 27
The first eight:
   'street light not working and road is dark and broken in block c. kindly take action' | true: Roads | sent to: Electricity
   'Pothole filled with sewer paani in sector 4. we request quick action!!' | true: Sanitation | sent to: Roads
   'hospital has no electricity for three days. nobody is listening' | true: Health | sent to: Electricity
   'Naali watr flooding the sadak in our colony. nobody is listening' | true: Roads | sent to: Sanitation
   'road dug for paani pipeline and left open in ward 12. nobody is listening' | true: Water Supply | sent to: Roads
   'Officials not responding about the problem in our colony. kindly take action' | true: Water Supply | sent to: Electricity
   'dengue mosquitoes from blocked drain in block c. kindly take action' | true: Health | sent to: Sanitation
   'dirty water in the school tap in shanti vihar. nobody is listening' | true: Education | sent to: Water Supply

Average confidence when right : 0.799
Average confidence when wrong : 0.447
Test grievances below 0.5 confidence : 33
Wrong routings caught by the threshold : 17 of 27
Accuracy of the automatic routings : 0.9682

--- Step 9: Deployment ---
'The street light outside my house is not working for a week' -> Electricity (confidence 0.73)
'Garbage is lying on the road and the drain is blocked' -> Sanitation (confidence 0.83)
'My pension has not come for two months' -> Human officer (confidence 0.42)

--- Step 10: Feedback ---
Grievances in the new month : 293
Routed automatically        : 266
Sent to a human officer     : 27
Automatic routings correct  : 258 of 266
Departments never seen in training : ['Cyber Crime']
Of these grievances, sent to an officer : 6 of 6
   'money taken from my bank account by an online fraud call' -> Human officer
   'fake website took my payment' -> Human officer
   'online shopping fraud, money not returned' -> Human officer
   'someone asked my otp on phone and took money' -> Human officer
   'my upi was used by a stranger' -> Human officer
   'fake job offer message took registration money' -> Human officer