# Program 12: Text normalisation - the steps from the CBSE AI handbook
#   1. Sentence segmentation   2. Tokenisation
#   3. Removing stop words, special characters and numbers
#   4. Converting to a common case   5. Stemming

corpus = "Raj and Vijay are best friends. They play together with other friends! Raj likes to play football but Vijay prefers to play online games. Vijay wants to be an online gamer in 2030."

print("Corpus:")
print(corpus)
print()

# 1. Sentence segmentation: split wherever a sentence ends.
text = corpus.replace("!", ".").replace("?", ".")
sentences = []
for s in text.split("."):
    if s.strip() != "":
        sentences.append(s.strip())
print("1. Sentences:")
for s in sentences:
    print("  ", s)
print()

# 2. Tokenisation: split each sentence into words (tokens).
tokens = []
for s in sentences:
    for word in s.split():
        tokens.append(word)
print("2. Tokens (" + str(len(tokens)) + "):")
print(tokens)
print()

# 3. Remove stop words, special characters and numbers.
stop_words = ["and", "are", "the", "a", "an", "to", "is", "in", "they", "with",
              "but", "be", "of", "other"]
kept = []
for word in tokens:
    if word.lower() in stop_words or not word.isalpha():
        continue
    kept.append(word)
print("3. After removing stop words and numbers (" + str(len(kept)) + "):")
print(kept)
print()

# 4. Convert to a common case (lower case).
lower = []
for word in kept:
    lower.append(word.lower())
print("4. Lower case:")
print(lower)
print()

# 5. Stemming: remove the affixes from the end of each word.
affixes = ["ing", "ed", "er", "es", "s"]
stems = []
for word in lower:
    for a in affixes:
        if word.endswith(a) and len(word) > len(a) + 2:
            word = word[:-len(a)]
            break
    stems.append(word)
print("5. After stemming:")
print(stems)
print()

# The handbook's own stemming table, worked by the same rule.
print("Word        Stem")
for word in ["healed", "healing", "healer", "studies", "studying"]:
    stem = word
    for a in affixes:
        if word.endswith(a) and len(word) > len(a) + 2:
            stem = word[:-len(a)]
            break
    print(word.ljust(11), stem)