# Program 13: Bag of Words
# The three documents used in the CBSE AI handbook.

import pandas as pd

documents = ["Aman and Avni are stressed",
             "Aman went to a therapist",
             "Avni went to download a health chatbot"]

# Step 1: text normalisation - lower case, split into tokens.
# (As in the handbook, stop words are kept so the table matches it.)
tokens = []
for d in documents:
    tokens.append(d.lower().split())
for i in range(len(tokens)):
    print("Document", i + 1, ":", tokens[i])
print()

# Step 2: create the dictionary - every different word, in the order met.
dictionary = []
for t in tokens:
    for word in t:
        if word not in dictionary:
            dictionary.append(word)
print("Dictionary (" + str(len(dictionary)) + " words):")
print(dictionary)
print()

# Step 3: document vectors - how many times each dictionary word occurs.
vectors = []
for t in tokens:
    row = []
    for word in dictionary:
        row.append(t.count(word))
    vectors.append(row)

table = pd.DataFrame(vectors, columns=dictionary, index=["Doc 1", "Doc 2", "Doc 3"])
pd.set_option("display.width", 160)
pd.set_option("display.max_columns", None)
print("Document vector table:")
print(table)
print()

# Step 4: how often each word occurs in the whole corpus.
print("Occurrences in the whole corpus:")
print(table.sum().sort_values(ascending=False).to_string())