text = "The Quick Brown Fox Jumps Over The LAZY Dog"
lower = text.lower()
print(lower)
# the quick brown fox jumps over the lazy dog

from bs4 import BeautifulSoup

html_text = "<p>The movie was <strong>absolutely</strong> fantastic!</p>"
clean = BeautifulSoup(html_text, "html.parser").get_text()
print(clean)  # The movie was absolutely fantastic!

import re

text = "Hello, World! This is NLP: amazing... isn't it? #NLP @python"

# Remove everything that isn't a letter or space
clean = re.sub(r'[^a-zA-Z\s]', '', text)
print(clean)
# Hello World This is NLP amazing isnt it  NLP python

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

# Download NLTK data (first time only)
nltk.download('stopwords', quiet=True)
nltk.download('punkt', quiet=True)

stop_words = set(stopwords.words('english'))
print(f"Number of English stopwords: {len(stop_words)}")
print(f"Sample: {list(stop_words)[:10]}")
# Number of English stopwords: 179
# Sample: ['i', 'me', 'my', 'myself', 'we', 'our', ...]

text = "The movie was absolutely the best film I have ever seen in my life"
tokens = word_tokenize(text.lower())
filtered = [w for w in tokens if w not in stop_words and w.isalpha()]
print("Original tokens:", tokens)
print("After stopword removal:", filtered)
# Original: ['the', 'movie', 'was', 'absolutely', 'the', 'best', 'film', 'i', 'have', ...]
# Filtered: ['movie', 'absolutely', 'best', 'film', 'ever', 'seen', 'life']

from nltk.stem import PorterStemmer

stemmer = PorterStemmer()

words = ["running", "runner", "runs", "studies", "studying", "studied",
         "easily", "fairness", "happiness", "university"]

for word in words:
    stem = stemmer.stem(word)
    print(f"  {word:15s} → {stem}")

# running         → run
# runner          → runner
# runs            → run
# studies         → studi        ← over-stemmed! Not a real word
# studying        → studi        ← both map to same broken form
# studied         → studi
# easily          → easili       ← over-stemmed
# fairness        → fair
# happiness       → happi        ← over-stemmed
# university      → univers

import spacy

# Load English model: python -m spacy download en_core_web_sm
nlp = spacy.load("en_core_web_sm")

text = "The geese were flying and the cats ran quickly through the forests"
doc = nlp(text)

for token in doc:
    print(f"  {token.text:15s} → lemma: {token.lemma_:15s}  POS: {token.pos_}")

# The             → lemma: the              POS: DET
# geese           → lemma: goose            POS: NOUN  ← irregular plural
# were            → lemma: be               POS: AUX
# flying          → lemma: fly              POS: VERB
# cats            → lemma: cat              POS: NOUN
# ran             → lemma: run              POS: VERB  ← irregular past tense
# quickly         → lemma: quickly          POS: ADV
# through         → lemma: through          POS: ADP
# forests         → lemma: forest           POS: NOUN

import re
import nltk
import spacy
from bs4 import BeautifulSoup
from nltk.corpus import stopwords

nltk.download('stopwords', quiet=True)
nltk.download('punkt', quiet=True)

nlp = spacy.load("en_core_web_sm")
stop_words = set(stopwords.words('english'))

def preprocess_text(text, remove_stopwords=True, use_lemmatization=True):
    """
    Full NLP preprocessing pipeline.

    Returns: list of cleaned tokens
    """
    # 1. Remove HTML tags
    text = BeautifulSoup(text, "html.parser").get_text()

    # 2. Lowercase
    text = text.lower()

    # 3. Remove URLs
    text = re.sub(r'http\S+|www\S+', '', text)

    # 4. Remove special characters (keep letters and spaces)
    text = re.sub(r'[^a-z\s]', '', text)

    # 5. Remove extra whitespace
    text = re.sub(r'\s+', ' ', text).strip()

    # 6. Lemmatize (using spaCy) OR just tokenize
    if use_lemmatization:
        doc = nlp(text)
        tokens = [token.lemma_ for token in doc if token.is_alpha]
    else:
        tokens = text.split()

    # 7. Remove stopwords (optional — skip for BERT!)
    if remove_stopwords:
        tokens = [t for t in tokens if t not in stop_words]

    return tokens


# Test the pipeline
examples = [
    "<p>The movie was absolutely FANTASTIC! Best film I've <b>ever</b> seen.</p>",
    "Running faster than the fastest runners in the world is running.",
    "Check out https://example.com for more info!! 123 special @chars#"
]

for text in examples:
    result = preprocess_text(text)
    print(f"Input:  {text[:60]}...")
    print(f"Output: {result}")
    print()

from nltk.tokenize import word_tokenize

texts = [
    "I can't believe it's not butter!",       # contractions
    "Dr. Smith visited the U.S.A. yesterday.", # abbreviations
    "The price is $9.99 + tax.",               # symbols and numbers
    "The email is user@example.com.",           # email addresses
    "She said 'hello world' to me."            # quotes
]

for text in texts:
    tokens = word_tokenize(text)
    print(f"Input:  {text}")
    print(f"Tokens: {tokens}")
    print()

# Input:  I can't believe it's not butter!
# Tokens: ['I', 'ca', "n't", 'believe', 'it', "'s", 'not', 'butter', '!']
#                   ↑ splits contractions intelligently!

# Input:  Dr. Smith visited the U.S.A. yesterday.
# Tokens: ['Dr.', 'Smith', 'visited', 'the', 'U.S.A.', 'yesterday', '.']
#          ↑ keeps abbreviations with their periods!

from nltk.tokenize import sent_tokenize

text = """
Dr. Smith gave a presentation on A.I. research.
The audience was amazed. "Will this work?" asked Prof. Johnson.
It remains to be seen. However, the early results are promising.
"""

sentences = sent_tokenize(text.strip())
for i, sent in enumerate(sentences, 1):
    print(f"Sentence {i}: {sent}")

# Sentence 1: Dr. Smith gave a presentation on A.I. research.
#   ↑ Correctly treats "Dr." and "A.I." as abbreviations, not sentence endings!
# Sentence 2: The audience was amazed.
# Sentence 3: "Will this work?" asked Prof. Johnson.
# Sentence 4: It remains to be seen.
# Sentence 5: However, the early results are promising.

text = "Hello, NLP!"

# Character tokenization: each character is a token
char_tokens = list(text)
print(char_tokens)
# ['H', 'e', 'l', 'l', 'o', ',', ' ', 'N', 'L', 'P', '!']

# Encoding: map characters to integers
vocab = sorted(set(char_tokens))
char_to_id = {c: i for i, c in enumerate(vocab)}
ids = [char_to_id[c] for c in char_tokens]
print(f"Vocabulary: {vocab}")
print(f"Token IDs: {ids}")

# Pros:
# - No OOV (out-of-vocabulary) words — every character is known
# - Small fixed vocabulary (~100 characters for English)
# Cons:
# - Extremely long sequences — "The cat" = 7 characters vs 2 words
# - Each character carries little semantic information
# - Harder for model to learn word-level patterns from characters

# BPE algorithm intuition (simplified)
# Start with character-level vocabulary of training corpus
# Corpus: "low low lower lowest new newer"

corpus_frequencies = {
    ('l','o','w','</w>'):   5,   # "low" appears 5 times
    ('l','o','w','e','r','</w>'): 2,   # "lower"
    ('l','o','w','e','s','t','</w>'): 1,  # "lowest"
    ('n','e','w','</w>'):   6,   # "new"
    ('n','e','w','e','r','</w>'): 3,   # "newer"
}

# Step 1: Find most frequent pair → ('e', 'r') with freq 2+3=5
# Merge: now 'er' is a token

# Step 2: Find most frequent pair → ('l', 'o') with freq 5+2+1=8
# Merge: now 'lo' is a token

# ... after many merges, the vocabulary contains:
# 'low', 'er', 'est', 'new', 'lo', 'w', etc.

# Real BPE with the `tokenizers` library (Hugging Face)
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace

# Train a small BPE tokenizer
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Whitespace()
trainer = BpeTrainer(vocab_size=1000, special_tokens=["[UNK]", "[CLS]", "[SEP]"])

# In practice, train on a text file:
# tokenizer.train(files=["corpus.txt"], trainer=trainer)

# GPT-2 uses BPE with vocab_size=50,257
# Example output for an unseen word:
print("Tokenization examples (BPE-like):")
print("  'unhappiness' → ['un', 'happiness']")
print("  'tokenization' → ['token', 'ization']")
print("  'COVID-19' → ['CO', 'VID', '-', '19']")
print("  'supercalifragilistic' → ['super', 'cal', 'if', 'rag', 'ilis', 'tic']")

from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

examples = [
    "tokenization is fascinating",
    "Let's do some NLP!",
    "unhappiness and disappointment",
    "COVID-19 vaccination rates are increasing",
    "antidisestablishmentarianism"
]

for text in examples:
    tokens = tokenizer.tokenize(text)
    ids    = tokenizer.encode(text)
    print(f"Text:   {text}")
    print(f"Tokens: {tokens}")
    print(f"IDs:    {ids[:8]}{'...' if len(ids)>8 else ''}")
    print()

# Text:   tokenization is fascinating
# Tokens: ['token', '##ization', 'is', 'fascinating']
#                    ↑ ## means "continuation of previous token"

# Text:   Let's do some NLP!
# Tokens: ['let', "'", 's', 'do', 'some', 'nl', '##p', '!']
#                                              ↑ NLP splits to nl + ##p

# Text:   antidisestablishmentarianism
# Tokens: ['anti', '##dis', '##establishment', '##arian', '##ism']
#   ↑ Long rare word split into 5 known subwords, all in BERT vocabulary!

import sentencepiece as spm

# SentencePiece is used by: T5, ALBERT, XLM-R, mT5, and many multilingual models
# It treats the input as a stream of characters, including spaces (represented as ▁)

# Load a pre-trained SentencePiece model (e.g., from T5)
# sp = spm.SentencePieceProcessor(model_file='t5_tokenizer.model')
# tokens = sp.encode("Hello world!", out_type=str)
# → ['▁Hello', '▁world', '!']
# The ▁ symbol marks the beginning of a word (replaces the space)

# Advantage: same model works for English, Chinese, Arabic, Finnish
print("SentencePiece examples:")
print("  English: 'Hello world' → ['▁Hello', '▁world']")
print("  Chinese: '你好世界' → ['▁你好', '世界'] (no spaces needed)")
print("  German compound: 'Donaudampfschiff' → ['▁Don', 'au', 'dam', 'pf', 'schiff']")

from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd

# Small corpus: 4 documents
corpus = [
    "The cat sat on the mat",
    "The cat sat on the hat",
    "The cat in the hat",
    "The dog sat on the mat"
]

# CountVectorizer builds vocabulary from corpus and counts occurrences
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)

# Show vocabulary
print("Vocabulary:", vectorizer.vocabulary_)
# {'cat': 0, 'dog': 2, 'hat': 3, 'in': 4, 'mat': 5, 'on': 6, 'sat': 7, 'the': 8}

# Document-term matrix
df = pd.DataFrame(
    X.toarray(),
    columns=vectorizer.get_feature_names_out(),
    index=[f"Doc {i+1}" for i in range(len(corpus))]
)
print("\nDocument-Term Matrix:")
print(df.to_string())

#        cat  dog  hat  in  mat  on  sat  the
# Doc 1    1    0    0   0    1   1    1    2
# Doc 2    1    0    1   0    0   1    1    2
# Doc 3    1    0    1   1    0   0    0    2
# Doc 4    0    1    0   0    1   1    1    2

# Notice: "the" appears twice in every document — it's noise!
# "cat" vs "dog" actually distinguishes the documents

import numpy as np
import math

# Manual TF-IDF calculation on a small corpus
corpus = [
    "data science machine learning python",         # Doc 0
    "machine learning deep learning neural network", # Doc 1
    "python programming web development flask",      # Doc 2
    "neural network deep learning computer vision",  # Doc 3
]

# Tokenize (simple split)
tokenized = [doc.split() for doc in corpus]
N = len(corpus)  # 4 documents

# Build vocabulary
vocab = sorted(set(word for doc in tokenized for word in doc))
print(f"Vocabulary ({len(vocab)} words): {vocab}")

# Compute TF for each document
def tf(word, doc_tokens):
    return doc_tokens.count(word) / len(doc_tokens)

# Compute IDF for each word
def idf(word, all_docs):
    df = sum(1 for doc in all_docs if word in doc)
    return math.log(N / df) + 1  # sklearn uses this formula

# Compute TF-IDF matrix
tfidf_matrix = np.zeros((N, len(vocab)))
for i, doc_tokens in enumerate(tokenized):
    for j, word in enumerate(vocab):
        tfidf_matrix[i, j] = tf(word, doc_tokens) * idf(word, tokenized)

# Show which words have highest TF-IDF in each document
for i, doc in enumerate(corpus):
    scores = {vocab[j]: tfidf_matrix[i, j] for j in range(len(vocab))}
    top3 = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:3]
    print(f"\nDoc {i}: '{doc[:40]}...'")
    print(f"  Top TF-IDF words: {top3}")

from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

# sklearn's TfidfVectorizer handles everything automatically
corpus = [
    "The quick brown fox jumps over the lazy dog",
    "The lazy dog sleeps all day long",
    "The quick brown fox is very quick and very clever",
    "Brown foxes are often quick and clever animals"
]

tfidf = TfidfVectorizer(
    max_features=15,     # keep only top 15 features (by TF-IDF)
    stop_words='english' # built-in stopword removal
)

X = tfidf.fit_transform(corpus)

# Show as dataframe
df = pd.DataFrame(
    X.toarray().round(3),
    columns=tfidf.get_feature_names_out(),
    index=[f"Doc {i+1}" for i in range(len(corpus))]
)
print(df.to_string())

# Notice: "the" and "the" are removed (stopwords)
# "quick" has high TF-IDF in Doc 1 and Doc 3 (appears multiple times, less common in Doc 4)
# "clever" has high TF-IDF in Doc 3 and Doc 4 (rare across corpus)

from sklearn.feature_extraction.text import TfidfVectorizer

# Train on a corpus that doesn't contain "cryptocurrency"
train_corpus = [
    "bitcoin price has risen sharply",
    "ethereum blockchain technology is advancing",
    "digital assets are becoming mainstream"
]

test_doc = ["cryptocurrency and bitcoin prices are volatile and unpredictable"]

vectorizer = TfidfVectorizer()
vectorizer.fit(train_corpus)

# Transform the test document
test_vec = vectorizer.transform(test_doc)
feature_names = vectorizer.get_feature_names_out()

# Check which words from test_doc are in vocabulary
test_words = test_doc[0].split()
known     = [w for w in test_words if w in vectorizer.vocabulary_]
oov_words = [w for w in test_words if w not in vectorizer.vocabulary_]

print(f"Training vocabulary size: {len(feature_names)}")
print(f"Test words:    {test_words}")
print(f"Known words:   {known}")
print(f"OOV words:     {oov_words}")    # ['cryptocurrency', 'volatile', 'unpredictable']
print(f"Non-zero features in test vec: {test_vec.nnz}")
# OOV words are SILENTLY IGNORED — their information is simply lost!

from sklearn.feature_extraction.text import TfidfVectorizer

# max_features controls vocabulary size
for max_features in [1000, 5000, 10000, 50000, None]:
    vec = TfidfVectorizer(max_features=max_features)
    # Smaller vocab → more OOV, less memory, faster training
    # Larger vocab → less OOV, more memory, slower training
    label = str(max_features) if max_features else 'all'
    print(f"max_features={label:>6s}: keeps top {label} words by TF-IDF score")

# Typical vocabulary sizes:
print("\nTypical NLP vocabulary sizes:")
print("  Bag of Words / TF-IDF:  10,000–100,000 words")
print("  BERT WordPiece:         30,522 subword tokens")
print("  GPT-2 BPE:              50,257 subword tokens")
print("  GPT-4 (estimated BPE):  ~100,000 tokens")
print("  SentencePiece (T5):     32,000 tokens")

from sklearn.feature_extraction.text import CountVectorizer
import numpy as np

vec = CountVectorizer()
sentences = [
    "The dog bit the man",
    "The man bit the dog"
]

X = vec.fit_transform(sentences).toarray()
print("Vocabulary:", vec.vocabulary_)
print("Doc 1:", X[0])
print("Doc 2:", X[1])
print("Identical?", np.array_equal(X[0], X[1]))
# Identical? True  ← SAME VECTOR for opposite-meaning sentences!
# BoW literally cannot distinguish subject from object.

from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer

docs = [
    "I love cars and automobiles",  # "cars" and "automobiles" are synonyms
    "I love cars",
    "I love automobiles",
    "I love bananas"               # completely different topic
]

vec = TfidfVectorizer()
X = vec.fit_transform(docs).toarray()

# Cosine similarity between first document and others
sim = cosine_similarity(X[0:1], X[1:])
print("Similarity between 'cars and automobiles' and:")
print(f"  'I love cars':        {sim[0][0]:.3f}")
print(f"  'I love automobiles': {sim[0][1]:.3f}")
print(f"  'I love bananas':     {sim[0][2]:.3f}")

# 'cars' and 'automobiles' are in completely different vocabulary dimensions
# so 'I love cars and automobiles' is NOT highly similar to either synonym alone!
# This is the fundamental problem: no semantic relationship between words.

# BoW cannot distinguish different senses of the same word
sentences = [
    "I deposited money at the bank",      # financial bank
    "We sat by the river bank",           # river bank
    "He banked the aircraft steeply",     # aviation bank (verb)
]

# In BoW, "bank" contributes the exact same feature value
# regardless of which sense is meant.
# Word embeddings (Lesson 50) also have this problem — they're static.
# Contextual embeddings (BERT) finally solve it.

print("Fundamental BoW / static embedding limitations:")
print("  'dog bites man' ≡ 'man bites dog'      (word order)")
print("  'car' ≈ 0 similarity to 'automobile'   (synonyms)")
print("  'bank' (finance) ≡ 'bank' (river)       (polysemy)")
print("  'not good' can look like 'good'         (negation)")
print("\nMotivation for next steps:")
print("  Word Embeddings (Lesson 50)  → fix: semantic similarity")
print("  Seq2Seq + Attention (L51-52) → fix: context")
print("  Transformers (Lesson 53)     → fix: all of the above")