Grouping Homes by Water Use

# ---------------------------------------------------------------------------
# water_groups.py
#
# AI project on Statistical Data: finding groups of households by how they
# use water, so that a housing society can give each group the right advice.
#
# CBSE Class X, Artificial Intelligence (417), Part C project work.
# SDG 6 (Clean Water and Sanitation) and SDG 11 (Sustainable Cities).
#
# This is UNSUPERVISED learning. Nobody has told the program which flats are
# careful and which are wasteful - there is no label column at all. The
# model, k-means clustering, finds the groups by itself:
#   1. pick k flats at random as the first group centres
#   2. put every flat in the group whose centre is nearest
#   3. move each centre to the middle (the mean) of its group
#   4. repeat steps 2 and 3 until nothing changes
#
# Libraries: pandas (the data), numpy (the arithmetic), matplotlib (charts).
# ---------------------------------------------------------------------------

# %% Stage 1 - Problem scoping
print("=" * 60)
print("GROUPING HOMES BY WATER USE")
print("=" * 60)
print("Who   : the housing society's water committee")
print("What  : one notice for everyone is ignored by everyone")
print("Where : a society with a water meter in every flat")
print("Why   : advice that fits each group saves more water")
print()

# %% Stage 2 - Data acquisition
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

pd.set_option("display.width", 120)
pd.set_option("display.max_columns", 20)

data = pd.read_csv("water_use.csv")

print("--- Stage 2: Data acquisition ---")
print("Rows read    :", len(data))
print("Columns read :", len(data.columns))
print()
print("The first 10 rows:")
print(data.head(10))
print()

# %% Stage 3 - Data exploration
print("--- Stage 3: Data exploration ---")
features = ["litres_per_person", "garden_minutes", "car_washes_week"]

missing = data["litres_per_person"].isnull().sum()
data = data.dropna(subset=["litres_per_person"])
print("Rows with no meter reading     :", missing, "(removed)")

# Water used cannot be negative; a negative number means the meter was reset.
negative = (data["litres_per_person"] < 0).sum()
data = data[data["litres_per_person"] >= 0]
print("Rows with negative litres      :", negative, "(removed)")
print("Rows left                      :", len(data))
print()

litres = data["litres_per_person"]
print("Litres per person a day - mean   :", round(np.mean(litres), 1))
print("Litres per person a day - median :", np.median(litres))
over = (litres > 135).sum()
print("Flats above 135 litres a person  :", over, "of", len(data))
print()

# Chart 1: how much water a person uses in a day, across the society.
plt.figure(figsize=(7, 4))
plt.hist(litres, bins=25, color="#4e9fd1")
plt.axvline(135, color="#e15759", linestyle="--", label="135 litres: the city norm")
plt.title("Water used by one person in a day")
plt.xlabel("Litres per person per day")
plt.ylabel("Number of flats")
plt.legend()
plt.tight_layout()
plt.savefig("chart1_litres.png")
plt.close()

# Chart 2: litres against garden watering, before any grouping.
plt.figure(figsize=(7, 5))
plt.scatter(data["litres_per_person"], data["garden_minutes"], color="#999999", alpha=0.7, s=18)
plt.title("Every flat, before grouping")
plt.xlabel("Litres per person per day")
plt.ylabel("Garden watering (minutes a day)")
plt.grid(True, linestyle="--", alpha=0.4)
plt.tight_layout()
plt.savefig("chart2_before.png")
plt.close()

# %% Stage 4 - Modelling
print("--- Stage 4: Modelling ---")

# Rescale every feature to run from 0 to 1, so litres (in hundreds) do not
# outweigh car washes (under 5).
low = data[features].min()
high = data[features].max()
scaled = (data[features] - low) / (high - low)

k = 3
starts = 5

# k-means can get stuck if two of its random first centres land in the same
# group. So it is run from 5 different random starts, and the grouping with
# the smallest SPREAD (the total distance from every flat to its own centre)
# is kept. Orange's k-Means widget does the same thing ("re-runs").
best_spread = None
same_as_best = 0
for start in range(starts):
    # Step 1: k flats chosen at random are the first centres.
    centres_try = scaled.sample(k, random_state=start).reset_index(drop=True)
    for step in range(20):
        # Step 2: distance from every flat to each centre; join the nearest.
        distance = pd.DataFrame()
        for c in range(k):
            distance[c] = ((scaled - centres_try.iloc[c]) ** 2).sum(axis=1)
        group_try = distance.idxmin(axis=1)
        # Step 3: move each centre to the mean of the flats in its group.
        centres_try = scaled.groupby(group_try).mean().reset_index(drop=True)
    spread = round(distance.min(axis=1).sum(), 4)
    print("Start", start + 1, "- spread :", spread)
    if best_spread is None or spread < best_spread:
        best_spread = spread
        group = group_try
        centres = centres_try
        same_as_best = 1
    elif spread == best_spread:
        same_as_best = same_as_best + 1
print("Starts that found the best grouping :", same_as_best, "of", starts)
print()

# Name the groups by how much water they use, smallest first.
order = data.groupby(group)["litres_per_person"].mean().sort_values().index
names = ["Careful", "Typical", "Heavy"]
data["group"] = group.replace({order[0]: names[0], order[1]: names[1], order[2]: names[2]})
print()

summary = data.groupby("group")[features].mean().round(1).reindex(names)
summary["flats"] = data["group"].value_counts().reindex(names)
print("What each group looks like (averages):")
print(summary)
print()

colours = ["#59a14f", "#4e9fd1", "#e15759"]

# Chart 3: the same scatter, now coloured by the group k-means found.
plt.figure(figsize=(7, 5))
for i in range(k):
    rows = data[data["group"] == names[i]]
    plt.scatter(rows["litres_per_person"], rows["garden_minutes"],
                color=colours[i], alpha=0.75, s=18, label=names[i])
plt.title("The three groups k-means found")
plt.xlabel("Litres per person per day")
plt.ylabel("Garden watering (minutes a day)")
plt.legend()
plt.grid(True, linestyle="--", alpha=0.4)
plt.tight_layout()
plt.savefig("chart3_groups.png")
plt.close()

# %% Stage 5 - Evaluation
print("--- Stage 5: Evaluation ---")

# There is no label, so there is no accuracy. Two other checks are used.

# There is no label, so there is no accuracy. Instead the program checks
# how tight the groups are for different values of k. Spread falls as k
# grows, because more centres means every flat has one closer by. The k
# after which it stops falling quickly is a good choice - the "elbow".
spreads = []
for kk in range(1, 6):
    best_kk = None
    for start in range(starts):
        c_kk = scaled.sample(kk, random_state=start).reset_index(drop=True)
        for step in range(20):
            d = pd.DataFrame()
            for c in range(kk):
                d[c] = ((scaled - c_kk.iloc[c]) ** 2).sum(axis=1)
            g = d.idxmin(axis=1)
            c_kk = scaled.groupby(g).mean().reset_index(drop=True)
        s_kk = d.min(axis=1).sum()
        if best_kk is None or s_kk < best_kk:
            best_kk = s_kk
    spreads.append(best_kk)
for kk in range(1, 6):
    print("Spread with k =", kk, ":", round(spreads[kk - 1], 2))
print()
print("Drop in spread from k = 2 to k = 3 :", round(spreads[1] - spreads[2], 2))
print("Drop in spread from k = 3 to k = 4 :", round(spreads[2] - spreads[3], 2))
print()

# Chart 4: the elbow chart.
plt.figure(figsize=(7, 4))
plt.plot(range(1, 6), spreads, marker="o", color="#4e9fd1")
plt.title("Choosing k: how tight the groups are")
plt.xlabel("k (number of groups)")
plt.xticks(range(1, 6))
plt.ylabel("Spread (total distance to centres)")
plt.grid(True, linestyle="--", alpha=0.4)
plt.tight_layout()
plt.savefig("chart4_elbow.png")
plt.close()

# Chart 5: how many flats in each group.
plt.figure(figsize=(6, 4))
plt.bar(names, summary["flats"], color=colours)
plt.title("Flats in each group")
plt.ylabel("Number of flats")
plt.tight_layout()
plt.savefig("chart5_group_sizes.png")
plt.close()

# %% Stage 6 - Deployment
print("--- Stage 6: Deployment ---")
tips = {
    "Careful": "Thank you note - keep it up",
    "Typical": "Fix dripping taps; use a bucket, not a shower",
    "Heavy":   "Water the garden at dawn; wash the car with a bucket",
}
new_flats = pd.DataFrame({
    "litres_per_person": [95, 140, 230],
    "garden_minutes":    [0, 12, 35],
    "car_washes_week":   [0, 1, 4],
})
new_scaled = (new_flats - low) / (high - low)
# Give each centre its group's name, so the nearest centre IS the answer.
centres = centres.rename(index={order[0]: names[0], order[1]: names[1], order[2]: names[2]})
for i in range(len(new_flats)):
    d = ((centres - new_scaled.iloc[i]) ** 2).sum(axis=1)
    g = d.idxmin()
    f = new_flats.iloc[i]
    print(int(f["litres_per_person"]), "litres,", int(f["garden_minutes"]), "garden min,",
          int(f["car_washes_week"]), "car washes ->", g, "->", tips[g])

============================================================
GROUPING HOMES BY WATER USE
============================================================
Who   : the housing society's water committee
What  : one notice for everyone is ignored by everyone
Where : a society with a water meter in every flat
Why   : advice that fits each group saves more water

--- Stage 2: Data acquisition ---
Rows read    : 300
Columns read : 4

The first 10 rows:
  flat_no  litres_per_person  garden_minutes  car_washes_week
0    F101               91.0               9                2
1    F102              196.0              39                4
2    F103              234.0              36                4
3    F104              146.0              10                1
4    F105              136.0               9                2
5    F106              131.0               9                1
6    F107              103.0               4                0
7    F108               91.0               4                0
8    F109               90.0               4                0
9    F110              210.0              35                4

--- Stage 3: Data exploration ---
Rows with no meter reading     : 3 (removed)
Rows with negative litres      : 2 (removed)
Rows left                      : 295

Litres per person a day - mean   : 142.6
Litres per person a day - median : 133.0
Flats above 135 litres a person  : 145 of 295

--- Stage 4: Modelling ---
Start 1 - spread : 9.8589
Start 2 - spread : 9.8589
Start 3 - spread : 13.9302
Start 4 - spread : 9.8589
Start 5 - spread : 9.8589
Starts that found the best grouping : 4 of 5


What each group looks like (averages):
         litres_per_person  garden_minutes  car_washes_week  flats
group                                                             
Careful               95.7             4.3              0.3    102
Typical              138.9            11.9              1.4    123
Heavy                217.4            33.4              3.2     70

--- Stage 5: Evaluation ---
Spread with k = 1 : 50.63
Spread with k = 2 : 15.8
Spread with k = 3 : 9.86
Spread with k = 4 : 8.04
Spread with k = 5 : 6.48

Drop in spread from k = 2 to k = 3 : 5.94
Drop in spread from k = 3 to k = 4 : 1.82

--- Stage 6: Deployment ---
95 litres, 0 garden min, 0 car washes -> Careful -> Thank you note - keep it up
140 litres, 12 garden min, 1 car washes -> Typical -> Fix dripping taps; use a bucket, not a shower
230 litres, 35 garden min, 4 car washes -> Heavy -> Water the garden at dawn; wash the car with a bucket