# ---------------------------------------------------------------------------
# bus_feedback.py
#
# AI project on Natural Language Processing: sentiment analysis of feedback
# that passengers send about a city's buses - is each message positive or
# negative?
#
# CBSE Class X, Artificial Intelligence (417), Part C project work.
# SDG 11 (Sustainable Cities and Communities - target 11.2, transport).
#
# The steps are the ones Unit 6 teaches:
#   1. Text normalisation - lower case, remove punctuation and numbers,
#      split into tokens, remove stop words, and stem each word
#   2. Bag of Words - count the words in each message
#   3. A model that decides the sentiment from those words
#
# Two models are compared:
#   - LEXICON (rule based): two lists of words written by a person - good
#     words and bad words. This is how Orange's Sentiment Analysis widget works.
#   - LEARNED WORD SCORES (learning based): from the training messages, the
#     program works out how positive or negative each word tends to be.
#
# Libraries: pandas (the messages), matplotlib (charts), numpy.
# ---------------------------------------------------------------------------

# %% Stage 1 - Problem scoping
print("=" * 60)
print("WHAT DO PASSENGERS FEEL?")
print("=" * 60)
print("Who   : the city transport office, and daily bus passengers")
print("What  : hundreds of feedback messages a week, read by nobody")
print("Where : a city bus service with an online feedback form")
print("Why   : complaints read quickly get fixed quickly")
print()

# %% Stage 2 - Data acquisition
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

pd.set_option("display.width", 140)
pd.set_option("display.max_colwidth", 90)

data = pd.read_csv("bus_feedback.csv")

print("--- Stage 2: Data acquisition ---")
print("Messages read :", len(data))
print()
print("The first 5 messages:")
print(data.head(5).to_string(index=False))
print()

# %% Stage 3 - Data exploration
print("--- Stage 3: Data exploration ---")

before = len(data)
# The same form submitted twice has the same ID and the same message. Two
# different passengers may write the same words - those are both kept.
data = data.drop_duplicates()
print("Messages sent twice           :", before - len(data), "(removed)")
empty = data["message"].isnull().sum()
data = data.dropna(subset=["message"])
print("Empty forms                   :", empty, "(removed)")
print("Messages left                 :", len(data))
print(data["sentiment"].value_counts().to_string())
print()

# Stop words: very common words that carry no feeling on their own.
# "not" and "never" are NOT on the list - they change the meaning.
stop_words = ["the", "a", "an", "and", "was", "were", "is", "are", "it", "to", "at",
              "of", "for", "our", "we", "i", "his", "this", "today", "also", "but",
              "very", "too", "again", "all", "every", "route", "bus", "trip",
              "morning", "evening", "please", "thanks", "an", "did", "on"]

# Text normalisation, one message at a time.
tokens_list = []
for text in data["message"]:
    text = text.lower()                                  # 1. common case
    clean = ""
    for ch in text:                                      # 2. keep letters only
        if ch.isalpha():
            clean = clean + ch
        else:
            clean = clean + " "
    tokens = []
    for word in clean.split():                           # 3. tokenise
        if word in stop_words:                           # 4. remove stop words
            continue
        for ending in ["ing", "ed", "s"]:                # 5. stem: cut endings
            if word.endswith(ending) and len(word) > 4:
                word = word[: -len(ending)]
                break
        tokens.append(word)
    tokens_list.append(tokens)
data["tokens"] = tokens_list

example = data.iloc[0]
print("Text normalisation of one message:")
print("  Original :", example["message"])
print("  Tokens   :", example["tokens"])
print()

# Bag of Words for the first three messages, on a few words.
vocabulary = []
for tokens in data["tokens"]:
    for word in tokens:
        if word not in vocabulary:
            vocabulary.append(word)
print("Words in the vocabulary :", len(vocabulary))
print()
show = ["driver", "polite", "rude", "late", "clean", "not", "never"]
bow = pd.DataFrame(0, index=["msg 1", "msg 2", "msg 3"], columns=show)
for m in range(3):
    for word in data.iloc[m]["tokens"]:
        if word in show:
            bow.loc["msg " + str(m + 1), word] = bow.loc["msg " + str(m + 1), word] + 1
print("Bag of Words (part of it) for the first three messages:")
print(bow)
print()

# Chart 1: the most common words in positive and in negative messages.
plt.figure(figsize=(9, 4.2))
for k, feeling in enumerate(["positive", "negative"]):
    counts = {}
    for tokens in data[data["sentiment"] == feeling]["tokens"]:
        for word in tokens:
            counts[word] = counts.get(word, 0) + 1
    top = pd.Series(counts).sort_values(ascending=False).head(10)
    plt.subplot(1, 2, k + 1)
    plt.barh(top.index[::-1], top.values[::-1], color=["#59a14f", "#e15759"][k])
    plt.title("Most common words: " + feeling)
    plt.xlabel("Times used")
plt.tight_layout()
plt.savefig("chart1_common_words.png")
plt.close()

# %% Stage 4 - Modelling
print("--- Stage 4: Modelling ---")
data = data.sample(frac=1, random_state=12)
cut = int(len(data) * 0.8)
train = data[:cut]
test = data[cut:]
print("Training messages :", len(train))
print("Testing messages  :", len(test))
print()

# Model A - LEXICON. Two lists written by a person (in stemmed form).
good_words = ["polite", "helpful", "clean", "comfortable", "smooth", "friendly", "great",
              "safely", "pleasant", "properly", "right"]
bad_words = ["late", "rude", "overcrowd", "crowd", "dirty", "broken", "uncomfortable",
             "terrible", "fast", "skipp", "shout", "wait", "complaint", "broke"]

# Model B - LEARNED WORD SCORES. For every word: how much more often it
# appears in positive training messages than in negative ones, on a scale
# from -1 (only ever negative) to +1 (only ever positive).
pos_count = {}
neg_count = {}
for i in range(len(train)):
    for word in train.iloc[i]["tokens"]:
        if train.iloc[i]["sentiment"] == "positive":
            pos_count[word] = pos_count.get(word, 0) + 1
        else:
            neg_count[word] = neg_count.get(word, 0) + 1
score = {}
for word in vocabulary:
    p = pos_count.get(word, 0)
    n = neg_count.get(word, 0)
    score[word] = (p - n) / (p + n + 1)

scores = pd.Series(score).sort_values()
print("Most negative words learned :", list(scores.index[:6]))
print("Most positive words learned :", list(scores.index[-6:][::-1]))
for word in ["not", "never", "complaint", "late"]:
    print("Learned score for '" + word + "'".ljust(12 - len(word)), ":", round(score.get(word, 0), 2))
print()

# Chart 2: the learned word scores.
pick = pd.concat([scores.head(8), scores.tail(8)])
plt.figure(figsize=(7, 5))
colours = []
for v in pick.values:
    if v > 0:
        colours.append("#59a14f")
    else:
        colours.append("#e15759")
plt.barh(pick.index, pick.values, color=colours)
plt.axvline(0, color="#999999")
plt.title("What the model learned about each word")
plt.xlabel("Word score (-1 negative ... +1 positive)")
plt.tight_layout()
plt.savefig("chart2_word_scores.png")
plt.close()

lexicon = []
learned = []
for tokens in test["tokens"]:
    total_a = 0
    total_b = 0
    for word in tokens:
        if word in good_words:
            total_a = total_a + 1
        if word in bad_words:
            total_a = total_a - 1
        total_b = total_b + score.get(word, 0)     # a word never seen scores 0
    if total_a >= 0:
        lexicon.append("positive")
    else:
        lexicon.append("negative")
    if total_b >= 0:
        learned.append("positive")
    else:
        learned.append("negative")

# %% Stage 5 - Evaluation
print("--- Stage 5: Evaluation ---")
actual = list(test["sentiment"])
results = []
for name, predicted in [["Lexicon", lexicon], ["Learned", learned]]:
    right = 0
    caught = 0
    negatives = 0
    for i in range(len(actual)):
        if predicted[i] == actual[i]:
            right = right + 1
        if actual[i] == "negative":
            negatives = negatives + 1
            if predicted[i] == "negative":
                caught = caught + 1
    results.append([name, right / len(actual) * 100, caught / negatives * 100])
    print(name.ljust(8), "accuracy                 :", round(right / len(actual) * 100, 1), "%")
    print(name.ljust(8), "complaints caught (recall):", round(caught / negatives * 100, 1), "%")
print()

# Messages the lexicon got wrong but the learned model got right.
print("Where the word lists went wrong:")
shown = 0
for i in range(len(actual)):
    if lexicon[i] != actual[i] and learned[i] == actual[i] and shown < 4:
        print("  [" + actual[i] + "]", test.iloc[i]["message"])
        shown = shown + 1
print()

matrix = pd.crosstab(pd.Series(actual, name="Actual"), pd.Series(learned, name="Predicted"))
matrix = matrix.reindex(index=["positive", "negative"], columns=["positive", "negative"], fill_value=0)
print("Confusion matrix of the learned model:")
print(matrix)
print()

# Chart 3: confusion matrix.
plt.figure(figsize=(5, 4.3))
plt.imshow(matrix.values, cmap="Greens")
for r in range(2):
    for c in range(2):
        value = matrix.values[r][c]
        if value > matrix.values.max() / 2:
            ink = "white"
        else:
            ink = "black"
        plt.text(c, r, value, ha="center", va="center", fontsize=15, color=ink)
plt.xticks([0, 1], ["positive", "negative"])
plt.yticks([0, 1], ["positive", "negative"])
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Learned model (testing messages)")
plt.tight_layout()
plt.savefig("chart3_matrix.png")
plt.close()

# Chart 4: the two models compared.
x = np.arange(2)
plt.figure(figsize=(7, 4.2))
plt.bar(x - 0.2, [results[0][1], results[1][1]], width=0.4, label="Accuracy", color="#4e79a7")
plt.bar(x + 0.2, [results[0][2], results[1][2]], width=0.4, label="Complaints caught", color="#e15759")
plt.xticks(x, ["Lexicon (word lists)", "Learned word scores"])
plt.ylim(0, 100)
plt.ylabel("Percent")
plt.title("Rule based and learning based sentiment")
plt.legend()
plt.tight_layout()
plt.savefig("chart4_models.png")
plt.close()

# %% Stage 6 - Deployment
print("--- Stage 6: Deployment ---")
new_messages = [
    "Bus 45 was never late this week, driver very polite!",
    "Waited 30 minutes and the bus was not clean.",
    "No complaints at all, smooth ride :)",
    "The conductor shouted at students. Terrible.",
]
for text in new_messages:
    clean = ""
    for ch in text.lower():
        if ch.isalpha():
            clean = clean + ch
        else:
            clean = clean + " "
    total = 0
    for word in clean.split():
        if word in stop_words:
            continue
        for ending in ["ing", "ed", "s"]:
            if word.endswith(ending) and len(word) > 4:
                word = word[: -len(ending)]
                break
        total = total + score.get(word, 0)
    if total >= 0:
        feeling = "POSITIVE"
    else:
        feeling = "NEGATIVE - send to the complaints desk"
    print(text.ljust(52), "score", str(round(total, 2)).rjust(6), "->", feeling)