Which Crop for This Field?

# ---------------------------------------------------------------------------
# crop_recommendation.py
#
# AI project on Statistical Data: recommending a crop for a field from its
# soil test and its weather.
#
# CBSE Class X, Artificial Intelligence (417), Part C project work.
# SDG 2 (Zero Hunger) and SDG 15 (Life on Land).
#
# The program follows the six stages of the AI Project Cycle. The model is
# "k nearest neighbours": to recommend a crop for a new field, it finds the
# k fields in the training data whose soil and weather are most like it, and
# recommends the crop that most of those neighbours grew well.
#
# Libraries: pandas (the data), numpy (the arithmetic), matplotlib (charts).
# ---------------------------------------------------------------------------

# %% Stage 1 - Problem scoping
print("=" * 60)
print("CROP RECOMMENDATION FROM A SOIL TEST")
print("=" * 60)
print("Who   : small farmers deciding what to sow")
print("What  : the wrong crop for the soil means a poor harvest")
print("Where : farms that have a soil test but no expert to read it")
print("Why   : the right crop gives more food from the same land")
print()

# %% Stage 2 - Data acquisition
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

pd.set_option("display.width", 120)
pd.set_option("display.max_columns", 20)

data = pd.read_csv("soil_samples.csv")

print("--- Stage 2: Data acquisition ---")
print("Rows read    :", len(data))
print("Columns read :", len(data.columns))
print()
print("The first 10 rows:")
print(data.head(10))
print()

# %% Stage 3 - Data exploration
print("--- Stage 3: Data exploration ---")

features = ["nitrogen", "phosphorus", "potassium", "ph", "rainfall_mm", "temperature_c"]
label = "crop"

# The same sample typed in twice would count twice, so repeats are removed.
before = len(data)
data = data.drop_duplicates()
print("Repeated rows removed        :", before - len(data))

# A soil test with a reading missing cannot be compared with other fields.
missing = data[features].isnull().any(axis=1).sum()
data = data.dropna(subset=features)
print("Rows with a reading missing  :", missing, "(removed)")

# "rice", " RICE" and "Rice" must all be one crop.
data[label] = data[label].str.strip().str.title()
print("Rows left                    :", len(data))
print()

crops = sorted(data[label].unique())
print("Fields of each crop:")
print(data[label].value_counts().sort_index())
print()

rain = data["rainfall_mm"]
print("Rainfall - mean   :", round(np.mean(rain), 1), "mm")
print("Rainfall - median :", round(np.median(rain), 1), "mm")
print()

averages_by_crop = data.groupby(label)[features].mean().round(1)
print("The average field for each crop:")
print(averages_by_crop)
print()

colours = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd"]

# Chart 1: average rainfall for each crop.
plt.figure(figsize=(7, 4))
plt.bar(crops, averages_by_crop["rainfall_mm"], color=colours)
plt.title("Average rainfall where each crop grew well")
plt.xlabel("Crop")
plt.ylabel("Rainfall (mm a month)")
plt.tight_layout()
plt.savefig("chart1_rainfall.png")
plt.close()

# Chart 2: rainfall against temperature, one colour per crop.
plt.figure(figsize=(7, 5))
for i in range(len(crops)):
    rows = data[data[label] == crops[i]]
    plt.scatter(rows["rainfall_mm"], rows["temperature_c"],
                color=colours[i], label=crops[i], alpha=0.7, s=18)
plt.title("Rainfall and temperature of every field")
plt.xlabel("Rainfall (mm a month)")
plt.ylabel("Temperature (deg C)")
plt.legend()
plt.grid(True, linestyle="--", alpha=0.4)
plt.tight_layout()
plt.savefig("chart2_rain_temperature.png")
plt.close()

# Chart 3: the three nutrients, side by side for each crop.
x = np.arange(len(crops))
plt.figure(figsize=(8, 4.5))
plt.bar(x - 0.27, averages_by_crop["nitrogen"], width=0.27, label="Nitrogen (N)", color="#4e79a7")
plt.bar(x, averages_by_crop["phosphorus"], width=0.27, label="Phosphorus (P)", color="#f28e2b")
plt.bar(x + 0.27, averages_by_crop["potassium"], width=0.27, label="Potassium (K)", color="#59a14f")
plt.xticks(x, crops)
plt.title("Average N, P and K in the soil for each crop")
plt.ylabel("kg per hectare")
plt.legend()
plt.tight_layout()
plt.savefig("chart3_npk.png")
plt.close()

# %% Stage 4 - Modelling
print("--- Stage 4: Modelling ---")

# Train-test split: shuffle (the same way every run), then 80% / 20%.
data = data.sample(frac=1, random_state=3)
cut = int(len(data) * 0.8)
train = data[:cut]
test = data[cut:]
print("Training rows :", len(train))
print("Testing rows  :", len(test))

# Rainfall runs into hundreds and pH stays under 10. Each feature is rescaled
# to run from 0 to 1 (using the training data only), so that no feature wins
# just because its numbers are bigger.
low = train[features].min()
high = train[features].max()
train_scaled = (train[features] - low) / (high - low)
test_scaled = (test[features] - low) / (high - low)

k = 5
print("Neighbours asked (k) :", k)
print()

predictions = []
for i in range(len(test)):
    row = test_scaled.iloc[i]
    # Distance from this field to EVERY training field.
    distance = ((train_scaled - row) ** 2).sum(axis=1)
    # The k closest fields, and the crop most of them grew.
    nearest = distance.nsmallest(k).index
    predictions.append(train.loc[nearest, label].mode()[0])

# %% Stage 5 - Evaluation
print("--- Stage 5: Evaluation ---")
actual = list(test[label])

correct = 0
for i in range(len(actual)):
    if predictions[i] == actual[i]:
        correct = correct + 1
accuracy = correct / len(actual) * 100
print("Correct recommendations :", correct, "of", len(actual))
print("Accuracy                :", round(accuracy, 1), "%")
print()

matrix = pd.crosstab(pd.Series(actual, name="Actual"),
                     pd.Series(predictions, name="Predicted"))
matrix = matrix.reindex(index=crops, columns=crops, fill_value=0)
print("Confusion matrix:")
print(matrix)
print()

print("Crop        Precision   Recall")
for c in crops:
    tp = matrix.loc[c, c]
    if matrix[c].sum() > 0:
        precision = tp / matrix[c].sum() * 100
    else:
        precision = 0
    if matrix.loc[c].sum() > 0:
        recall = tp / matrix.loc[c].sum() * 100
    else:
        recall = 0
    print(c.ljust(10), str(round(precision, 1)).rjust(9), "%", str(round(recall, 1)).rjust(7), "%")
print()

# Chart 4: the confusion matrix.
plt.figure(figsize=(6.5, 5.5))
plt.imshow(matrix.values, cmap="Greens")
for r in range(len(crops)):
    for c in range(len(crops)):
        value = matrix.values[r][c]
        if value > matrix.values.max() / 2:
            ink = "white"
        else:
            ink = "black"
        plt.text(c, r, value, ha="center", va="center", fontsize=13, color=ink)
plt.xticks(range(len(crops)), crops)
plt.yticks(range(len(crops)), crops)
plt.xlabel("Recommended crop")
plt.ylabel("Crop that grew well")
plt.title("Confusion matrix (testing data)")
plt.tight_layout()
plt.savefig("chart4_confusion_matrix.png")
plt.close()

# How much does the choice of k matter? Try every k from 1 to 15.
k_values = list(range(1, 16))
k_accuracy = []
for kk in k_values:
    right = 0
    for i in range(len(test)):
        distance = ((train_scaled - test_scaled.iloc[i]) ** 2).sum(axis=1)
        nearest = distance.nsmallest(kk).index
        if train.loc[nearest, label].mode()[0] == actual[i]:
            right = right + 1
    k_accuracy.append(right / len(test) * 100)

print("Accuracy for k = 1 :", round(k_accuracy[0], 1), "%")
print("Accuracy for k = 5 :", round(k_accuracy[4], 1), "%")
print("Accuracy for k = 15 :", round(k_accuracy[14], 1), "%")
print()

# Chart 5: accuracy for each value of k.
plt.figure(figsize=(7, 4))
plt.plot(k_values, k_accuracy, marker="o", color="#2ca02c")
plt.title("How many neighbours should the model ask?")
plt.xlabel("k (number of neighbours)")
plt.ylabel("Accuracy on testing data (%)")
plt.xticks(k_values)
plt.ylim(80, 100)
plt.grid(True, linestyle="--", alpha=0.4)
plt.tight_layout()
plt.savefig("chart5_choosing_k.png")
plt.close()

# %% Stage 6 - Deployment
print("--- Stage 6: Deployment ---")
print("Three new soil test cards:")

new_fields = pd.DataFrame({
    "nitrogen":      [85, 38, 120],
    "phosphorus":    [45, 70, 44],
    "potassium":     [41, 79, 19],
    "ph":            [6.2, 7.4, 7.1],
    "rainfall_mm":   [240, 78, 82],
    "temperature_c": [25, 19, 24],
})
new_scaled = (new_fields - low) / (high - low)

for i in range(len(new_fields)):
    distance = ((train_scaled - new_scaled.iloc[i]) ** 2).sum(axis=1)
    nearest = distance.nsmallest(k).index
    votes = train.loc[nearest, label]
    best = votes.mode()[0]
    agree = (votes == best).sum()
    f = new_fields.iloc[i]
    print("N", int(f["nitrogen"]), " P", int(f["phosphorus"]), " K", int(f["potassium"]),
          " pH", f["ph"], " rain", f["rainfall_mm"], "mm  ->", best,
          "(" + str(agree), "of", k, "neighbours agree)")

============================================================
CROP RECOMMENDATION FROM A SOIL TEST
============================================================
Who   : small farmers deciding what to sow
What  : the wrong crop for the soil means a poor harvest
Where : farms that have a soil test but no expert to read it
Why   : the right crop gives more food from the same land

--- Stage 2: Data acquisition ---
Rows read    : 503
Columns read : 8

The first 10 rows:
  sample_no  nitrogen  phosphorus  potassium   ph  rainfall_mm  temperature_c      crop
0     S1001        89          63         43  6.8         69.9           18.4     Wheat
1     S1002        57          65         83  7.2         76.8           16.4  Chickpea
2     S1003        23          67         79  7.1         71.3           19.4  Chickpea
3     S1004        44          59         78  7.5         79.4           19.4  Chickpea
4     S1005       111          49         39  6.9         60.7           14.4     Wheat
5     S1006        95          48         46  6.9         72.7           18.5     Wheat
6     S1006        95          48         46  6.9         72.7           18.5     Wheat
7     S1007       107          35         42  6.9         81.0           19.1     Wheat
8     S1008        78          41         38  7.8         54.2           17.3     Wheat
9     S1009       100          49         42  6.6        194.4           23.7      Rice

--- Stage 3: Data exploration ---
Repeated rows removed        : 3
Rows with a reading missing  : 4 (removed)
Rows left                    : 496

Fields of each crop:
crop
Chickpea     97
Cotton       99
Maize       101
Rice         99
Wheat       100
Name: count, dtype: int64

Rainfall - mean   : 110.6 mm
Rainfall - median : 83.5 mm

The average field for each crop:
          nitrogen  phosphorus  potassium   ph  rainfall_mm  temperature_c
crop                                                                      
Chickpea      38.6        68.0       80.1  7.1         79.6           19.0
Cotton       117.8        45.4       20.1  7.0         79.4           23.8
Maize         77.2        48.4       20.4  6.2         83.0           22.5
Rice          80.2        46.9       40.7  6.3        235.2           24.2
Wheat         99.0        49.8       40.2  6.9         76.0           16.9

--- Stage 4: Modelling ---
Training rows : 396
Testing rows  : 100
Neighbours asked (k) : 5

--- Stage 5: Evaluation ---
Correct recommendations : 97 of 100
Accuracy                : 97.0 %

Confusion matrix:
Predicted  Chickpea  Cotton  Maize  Rice  Wheat
Actual                                         
Chickpea         19       0      0     0      0
Cotton            0      23      1     0      0
Maize             0       2     13     0      0
Rice              0       0      0    27      0
Wheat             0       0      0     0     15

Crop        Precision   Recall
Chickpea       100.0 %   100.0 %
Cotton          92.0 %    95.8 %
Maize           92.9 %    86.7 %
Rice           100.0 %   100.0 %
Wheat          100.0 %   100.0 %

Accuracy for k = 1 : 96.0 %
Accuracy for k = 5 : 97.0 %
Accuracy for k = 15 : 97.0 %

--- Stage 6: Deployment ---
Three new soil test cards:
N 85  P 45  K 41  pH 6.2  rain 240.0 mm  -> Rice (5 of 5 neighbours agree)
N 38  P 70  K 79  pH 7.4  rain 78.0 mm  -> Chickpea (5 of 5 neighbours agree)
N 120  P 44  K 19  pH 7.1  rain 82.0 mm  -> Cotton (5 of 5 neighbours agree)