import numpy as np

# Vocabulary of 5 words (in reality: 50,000+)
vocab = {'cat': 0, 'dog': 1, 'automobile': 2, 'car': 3, 'banana': 4}
vocab_size = len(vocab)

# One-hot encoding
def one_hot(word, vocab):
    vec = np.zeros(vocab_size)
    vec[vocab[word]] = 1.0
    return vec

cat      = one_hot('cat', vocab)
dog      = one_hot('dog', vocab)
car      = one_hot('car', vocab)
auto     = one_hot('automobile', vocab)
banana   = one_hot('banana', vocab)

# Problem 1: Enormous dimensionality
# With 50,000 words: each word is a 50,000-dimensional vector
# A batch of 32 sentences each with 50 words = 32 × 50 × 50,000 = 80 MILLION values
print("Problem 1 — Dimensionality:")
print(f"  One-hot with vocab_size=50,000: {50000}-dim vector per word")
print(f"  Dense embedding with dim=300:   300-dim vector per word")
print(f"  Memory ratio: {50000/300:.0f}× smaller with embeddings\n")

# Problem 2: All words are equidistant — cosine similarity = 0 for ALL pairs
from numpy.linalg import norm

def cosine_sim(a, b):
    return np.dot(a, b) / (norm(a) * norm(b) + 1e-10)

print("Problem 2 — Semantic similarity (should be high for related words):")
print(f"  cosine(cat, dog):       {cosine_sim(cat, dog):.3f}   ← should be HIGH")
print(f"  cosine(car, automobile):{cosine_sim(car, auto):.3f}   ← should be HIGH")
print(f"  cosine(cat, banana):    {cosine_sim(cat, banana):.3f}  ← should be low")
print(f"  All are 0.0 — one-hot encodes NO semantic information!\n")

# Problem 3: No relationship between words
# In one-hot space: cat, dog, car, banana — all equally "different"
# There's no axis that captures "is this an animal?" or "is this a vehicle?"

import torch
import torch.nn as nn

# nn.Embedding: a trainable lookup table
# vocab_size=10000 words, each embedded into 300 dimensions
embedding = nn.Embedding(num_embeddings=10000, embedding_dim=300)

# Convert token ID to dense vector
token_id = torch.tensor([42])  # word at index 42 in vocabulary
dense_vec = embedding(token_id)
print(f"Token ID: {token_id.item()}")
print(f"Dense vector shape: {dense_vec.shape}")  # (1, 300)
print(f"Dense vector (first 5 dims): {dense_vec[0, :5].detach().numpy()}")

# Process a whole sentence (sequence of token IDs)
sentence = torch.tensor([42, 17, 891, 45, 230])  # 5-word sentence
embedded_sentence = embedding(sentence)
print(f"\nSentence token IDs: {sentence.numpy()}")
print(f"Embedded sentence shape: {embedded_sentence.shape}")  # (5, 300)
# 5 words × 300 dimensions

# During training, the embedding weights are learned alongside model weights.
# Alternatively, load pre-trained Word2Vec or GloVe weights (next sections).
total_params = embedding.weight.numel()
print(f"\nEmbedding table parameters: {total_params:,}")  # 10000 × 300 = 3,000,000

# The distributional hypothesis in action:
# If we collected all context words for "coffee" and "tea" from Wikipedia,
# they would look very similar:

context_coffee = ['hot', 'drink', 'cup', 'morning', 'bitter', 'brew',
                  'espresso', 'latte', 'caffeine', 'aroma', 'roasted']
context_tea    = ['hot', 'drink', 'cup', 'morning', 'green', 'herbal',
                  'steep', 'kettle', 'chamomile', 'caffeine', 'brew']
context_banana = ['yellow', 'peel', 'fruit', 'ripe', 'potassium',
                  'monkey', 'bunch', 'plantain', 'tropical', 'sweet']

# Measure context overlap (proxy for semantic similarity)
overlap_coffee_tea    = len(set(context_coffee) & set(context_tea))
overlap_coffee_banana = len(set(context_coffee) & set(context_banana))

print(f"Coffee/Tea context overlap:    {overlap_coffee_tea} words")    # high
print(f"Coffee/Banana context overlap: {overlap_coffee_banana} words")  # lower

# Word2Vec learns embeddings by training a model to predict these contexts.
# After training, coffee and tea have similar embedding vectors because
# they appear in almost identical contexts throughout the corpus.

# Context window illustration
sentence = ["The", "cat", "sat", "on", "the", "mat"]
target_position = 2  # "sat" is the target
window_size = 2

target_word   = sentence[target_position]
context_words = (sentence[max(0, target_position - window_size) : target_position] +
                 sentence[target_position + 1 : target_position + window_size + 1])

print(f"Sentence: {sentence}")
print(f"Target word: '{target_word}' (position {target_position})")
print(f"Context words (window={window_size}): {context_words}")
# Context words (window=2): ['The', 'cat', 'on', 'the']

# Skip-gram training pairs from this window:
print("\nSkip-gram training pairs (target → context):")
for ctx in context_words:
    print(f"  '{target_word}' → '{ctx}'")

import torch
import torch.nn as nn

class Word2VecSkipGram(nn.Module):
    """
    Simplified Skip-gram model.
    The embedding layer (W_in) weights ARE the word embeddings.
    W_out are only used during training and discarded afterwards.
    """
    def __init__(self, vocab_size, embed_dim):
        super().__init__()
        # Input embedding matrix: vocab_size × embed_dim
        # This is the matrix we actually want to learn
        self.embeddings = nn.Embedding(vocab_size, embed_dim)

        # Output layer: projects embedding back to vocabulary distribution
        self.output = nn.Linear(embed_dim, vocab_size)

    def forward(self, target_ids):
        """
        target_ids: (batch,) — integer token IDs of center words
        Returns: (batch, vocab_size) — log probability of each context word
        """
        # Look up embeddings for target words
        embeds = self.embeddings(target_ids)    # (batch, embed_dim)

        # Project to vocabulary size and apply log softmax
        scores = self.output(embeds)             # (batch, vocab_size)
        return torch.log_softmax(scores, dim=1)  # (batch, vocab_size)


# Model instantiation
VOCAB_SIZE = 10000
EMBED_DIM  = 300

model = Word2VecSkipGram(VOCAB_SIZE, EMBED_DIM)
print(f"Input embedding matrix:  {model.embeddings.weight.shape}")   # (10000, 300)
print(f"Output matrix:           {model.output.weight.shape}")        # (10000, 300)
print(f"Total parameters:        {sum(p.numel() for p in model.parameters()):,}")
# ≈ 6M params — but the embedding matrix is what we actually use after training

# After training, extract the learned embeddings:
word_embeddings = model.embeddings.weight.data  # (vocab_size, embed_dim)
print(f"\nLearned embeddings shape: {word_embeddings.shape}")

import time
import torch
import torch.nn as nn

vocab_size = 50000
embed_dim  = 300
batch_size = 256

# Simulate full softmax output layer computation
W_out = torch.randn(vocab_size, embed_dim)
embeds = torch.randn(batch_size, embed_dim)

start = time.time()
for _ in range(1000):  # 1000 batches
    scores = embeds @ W_out.T               # (256, 50000)
    probs  = torch.softmax(scores, dim=-1)  # expensive!
print(f"Full softmax (1000 batches): {time.time()-start:.2f}s")
# Slow! And this is without the backward pass.

import torch
import torch.nn as nn
import torch.optim as optim

class Word2VecNegSampling(nn.Module):
    """
    Skip-gram with Negative Sampling — the actual Word2Vec training objective.
    """
    def __init__(self, vocab_size, embed_dim):
        super().__init__()
        self.target_embeddings  = nn.Embedding(vocab_size, embed_dim)
        self.context_embeddings = nn.Embedding(vocab_size, embed_dim)

    def forward(self, target, context, negatives):
        """
        target:    (batch,)                  — center word IDs
        context:   (batch,)                  — true context word IDs
        negatives: (batch, num_neg_samples)  — fake context word IDs

        Returns: loss (scalar)
        """
        # Positive pairs: target word should be close to true context
        t_embed = self.target_embeddings(target)          # (B, D)
        c_embed = self.context_embeddings(context)        # (B, D)
        pos_score = torch.sum(t_embed * c_embed, dim=1)   # (B,)
        pos_loss  = -torch.log(torch.sigmoid(pos_score) + 1e-10)

        # Negative pairs: target word should be FAR from negative samples
        n_embed = self.context_embeddings(negatives)      # (B, K, D)
        # Expand target to match: (B, 1, D) broadcast with (B, K, D)
        neg_score = torch.bmm(n_embed, t_embed.unsqueeze(2)).squeeze(2)  # (B, K)
        neg_loss  = -torch.log(torch.sigmoid(-neg_score) + 1e-10).sum(1) # (B,)

        return (pos_loss + neg_loss).mean()


# Hyperparameters
VOCAB_SIZE      = 10000
EMBED_DIM       = 300
NUM_NEG_SAMPLES = 5       # typically 5–20

model     = Word2VecNegSampling(VOCAB_SIZE, EMBED_DIM)
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Simulated training batch
batch_size = 128
target    = torch.randint(0, VOCAB_SIZE, (batch_size,))
context   = torch.randint(0, VOCAB_SIZE, (batch_size,))
negatives = torch.randint(0, VOCAB_SIZE, (batch_size, NUM_NEG_SAMPLES))

optimizer.zero_grad()
loss = model(target, context, negatives)
loss.backward()
optimizer.step()
print(f"Training loss (random init): {loss.item():.4f}")

from gensim.models import Word2Vec

# Sentences: each sentence is a list of word strings
# In practice, this is your preprocessed corpus
sentences = [
    ["the", "cat", "sat", "on", "the", "mat"],
    ["the", "dog", "ran", "across", "the", "field"],
    ["cats", "and", "dogs", "are", "common", "pets"],
    ["machine", "learning", "and", "deep", "learning", "are", "related"],
    ["neural", "networks", "learn", "from", "data"],
]

# Train Word2Vec Skip-gram model
model = Word2Vec(
    sentences=sentences,
    vector_size=100,    # embedding dimension
    window=5,           # context window size
    min_count=1,        # minimum word frequency (ignore rarer words)
    workers=4,          # parallel workers
    sg=1,               # 1 = Skip-gram, 0 = CBOW
    negative=5,         # number of negative samples
    epochs=100          # training epochs
)

# Access embeddings
print("Vocabulary size:", len(model.wv))
print("Embedding for 'cat':", model.wv['cat'][:5], "...")
print("Embedding shape:", model.wv['cat'].shape)  # (100,)

# Most similar words
print("\nWords most similar to 'cat':")
similar = model.wv.most_similar('cat', topn=3)
for word, score in similar:
    print(f"  {word}: {score:.4f}")

import gensim.downloader as api
import numpy as np
from numpy.linalg import norm

# Load pre-trained Word2Vec (requires internet download, ~1.7GB)
# model = api.load('word2vec-google-news-300')

# For illustration, we'll show the code that would run:
print("Famous Word2Vec vector arithmetic:")
print()

# king - man + woman ≈ queen
# result = model.most_similar(positive=['king', 'woman'], negative=['man'], topn=3)
# print("king - man + woman:")
# for word, score in result:
#     print(f"  {word}: {score:.4f}")
# → queen: 0.7118, princess: 0.6417, monarch: 0.6291

print("king - man + woman ≈ queen")
print("  Expected: [('queen', 0.71), ('princess', 0.64), ('monarch', 0.63)]")
print()

# Paris - France + Italy ≈ Rome
# result = model.most_similar(positive=['Paris', 'Italy'], negative=['France'])
print("Paris - France + Italy ≈ Rome")
print("  Expected: [('Rome', 0.73), ('Milan', 0.67), ('Naples', 0.64)]")
print()

# walked - walk + swim ≈ swam  (verb tense direction!)
print("walked - walk + swim ≈ swam")
print("  Expected: [('swam', 0.71), ('swum', 0.65), ('swimming', 0.62)]")
print()

# largest - large + small ≈ smallest  (adjective superlative direction!)
print("largest - large + small ≈ smallest")
print("  Expected: [('smallest', 0.82), ('tiniest', 0.76), ('littlest', 0.71)]")

import numpy as np
from numpy.linalg import norm

def analogy(word_a, word_b, word_c, model, topn=5):
    """
    Computes analogy: word_a is to word_b as word_c is to ???
    Formula: result ≈ word_c - word_a + word_b
    """
    vec = model.wv[word_b] - model.wv[word_a] + model.wv[word_c]

    # Normalize
    vec = vec / (norm(vec) + 1e-10)

    # Find nearest neighbors
    results = []
    for word, embedding in model.wv.key_to_index.items():
        if word in [word_a, word_b, word_c]:
            continue
        word_vec = model.wv[word] / (norm(model.wv[word]) + 1e-10)
        sim = np.dot(vec, word_vec)
        results.append((word, sim))

    return sorted(results, key=lambda x: x[1], reverse=True)[:topn]


# With a trained model:
# print(analogy('man', 'woman', 'king', model))
# → [('queen', 0.71), ...]

# Geometric interpretation:
print("Geometric interpretation of king - man + woman:")
print()
print("  Embedding space has directions:")
print("    'gender axis':  female_vec = any_female - any_male")
print("    'royalty axis': royal_vec = king or queen - commoner")
print()
print("  king_vec = royal_component + male_component + (king-specific)")
print("  − man_vec = − male_component − (man-specific)")
print("  + woman_vec = + female_component + (woman-specific)")
print("  ≈ royal_component + female_component ≈ queen_vec")

# SEMANTIC analogies (meaning relationships):
# France:Paris :: Germany:Berlin         (country → capital)
# dog:puppy :: cat:kitten               (animal → young form)
# happy:unhappy :: possible:impossible  (word → antonym via prefix)

# SYNTACTIC analogies (grammatical relationships):
# run:running :: swim:swimming           (infinitive → gerund)
# walk:walked :: go:went                 (present → past tense)
# good:better :: bad:worse               (adjective → comparative)
# mouse:mice :: goose:geese              (singular → irregular plural)

# Accuracy on Google's analogy benchmark (14,337 analogies):
print("Word2Vec accuracy on analogy tasks (Google News 300d):")
print("  Semantic analogies: ~75% accuracy")
print("  Syntactic analogies: ~68% accuracy")
print("  Overall: ~71% accuracy")
print()
print("This was considered extraordinary in 2013 — previous methods")
print("achieved 10-30% on the same benchmark.")

import numpy as np

# GloVe objective (conceptual):
# For each word pair (i, j) with co-occurrence count X_{ij}:
# Minimize: (w_i · w_j + b_i + b_j - log(X_{ij}))^2 × f(X_{ij})
# Where f(X) is a weighting function that downweights very common pairs

# f(X) = (X / x_max)^alpha if X < x_max, else 1
# This prevents very frequent pairs from dominating the loss

def glove_weight(x, x_max=100, alpha=0.75):
    """GloVe co-occurrence weighting function."""
    if x < x_max:
        return (x / x_max) ** alpha
    return 1.0

counts = [1, 5, 10, 50, 100, 500, 1000]
print("GloVe weighting function f(X):")
for c in counts:
    print(f"  f({c:5d}) = {glove_weight(c):.4f}")

import gensim.downloader as api

# Load pre-trained GloVe vectors
# Available sizes: 50, 100, 200, 300 dimensions
# Trained on: Wikipedia 2014 + Gigaword 5 (6B tokens)

# glove_model = api.load('glove-wiki-gigaword-300')  # ~500MB download

# With loaded model:
# Word similarity
# sim = glove_model.similarity('coffee', 'tea')    # → ~0.81
# sim = glove_model.similarity('coffee', 'banana') # → ~0.35

# Analogy
# result = glove_model.most_similar(
#     positive=['Paris', 'Germany'], negative=['France']
# )
# → [('Berlin', 0.78), ('Munich', 0.68), ...]

print("GloVe vs Word2Vec comparison:")
print()
print("  Word2Vec (Skip-gram):")
print("    - Learns from local context windows")
print("    - Better at capturing syntactic analogies")
print("    - Faster to train on the same corpus")
print()
print("  GloVe:")
print("    - Uses global co-occurrence statistics")
print("    - Often better at word similarity tasks")
print("    - Factorises a matrix → more mathematically principled")
print("    - Works well with smaller corpora")
print()
print("  In practice: performance difference is usually <3% — try both!")
print("  For most tasks, pre-trained GloVe 300d or Word2Vec 300d are equivalent.")

# FastText: represent each word as sum of character n-gram vectors

def get_ngrams(word, min_n=3, max_n=6):
    """Generate character n-grams for a word."""
    # Add boundary markers
    word_boundary = f"<{word}>"
    ngrams = []
    for n in range(min_n, max_n + 1):
        for i in range(len(word_boundary) - n + 1):
            ngrams.append(word_boundary[i:i+n])
    return ngrams

# The word "running" decomposes into subword n-grams:
print("Character n-grams for 'running' (n=3..6):")
ngrams = get_ngrams("running")
print(f"  {ngrams}")

# Key insight: 'running', 'runner', 'runs' share many n-grams:
# 'running':  ['<ru', 'run', 'unn', 'nni', 'nin', 'ing', 'ng>', ...]
# 'runner':   ['<ru', 'run', 'unn', 'nne', 'ner', 'er>', ...]
# 'runs':     ['<ru', 'run', 'uns', 'ns>', ...]
# They share 'run', '<ru', 'unn' — their similarity is FORCED by the architecture!

print("\nShared n-grams between 'running' and 'runner':")
run_ngrams    = set(get_ngrams("running"))
runner_ngrams = set(get_ngrams("runner"))
shared = run_ngrams & runner_ngrams
print(f"  {sorted(shared)}")
print(f"  Overlap: {len(shared)}/{len(run_ngrams)} n-grams")

import fasttext
# pip install fasttext

# Train FastText model
# model = fasttext.train_unsupervised(
#     'corpus.txt',
#     model='skipgram',   # or 'cbow'
#     dim=300,
#     ws=5,               # window size
#     minCount=5,
#     minn=3,             # min n-gram size
#     maxn=6,             # max n-gram size
#     neg=5,              # negative samples
#     epoch=5
# )

# The killer feature: vectors for OOV words!
# The word 'deeplearning' might not be in vocabulary
# But FastText can generate its vector from n-grams:
# 'deep' + 'eep' + 'epl' + 'plee' + 'lear' + 'earn' + 'arni' + ...

# model.get_word_vector('deeplearning')  # Works even for OOV!
# model.get_nearest_neighbors('deeplearning')
# → [('deep_learning', 0.91), ('machine_learning', 0.87), ...]

print("FastText OOV handling:")
print("  Training vocabulary contains: 'deep', 'learning', 'machine'")
print("  Test word: 'deeplearning' (not in vocabulary!)")
print("  FastText: generates vector from character n-grams")
print("  → Similar to 'deep' + 'learning' vectors!")
print()
print("FastText shines for:")
print("  - Morphologically rich languages (German, Finnish, Arabic, Turkish)")
print("  - Handling misspellings ('teh' → similar to 'the')")
print("  - Rare or technical terms")
print("  - Social media text with creative spelling ('gr8', 'luv')")

# Download pre-trained FastText vectors (157 languages!):
# import fasttext.util
# fasttext.util.download_model('en', if_exists='ignore')
# ft = fasttext.load_model('cc.en.300.bin')

import numpy as np

# With Word2Vec / GloVe:
# "bank" has ONE vector — somewhere between its financial and geographical meanings
# This vector is a blend of ALL contexts in which "bank" appeared

# Example sentences:
financial_bank = "I deposited the cheque at the bank yesterday"
river_bank     = "We sat by the river bank watching the ducks"
aviation_bank  = "The pilot banked the aircraft steeply to avoid the storm"

# In Word2Vec, ALL THREE uses of "bank" get the EXACT SAME embedding!
# The embedding is trained to be the average context word for all uses.
# For very common polysemous words (bank, lead, bear, lie, right, match)
# the single vector is a compromise that doesn't serve any single meaning well.

print("Static embedding problem (polysemy):")
print()
print("  Sentence 1: 'I deposited the cheque at the BANK yesterday'")
print("  Sentence 2: 'We sat by the river BANK watching the ducks'")
print()
print("  Word2Vec gives both 'bank' uses the SAME vector:")
print("  bank_vector = average of financial + geographical contexts")
print()
print("  This means:")
print("  - 'bank' in sentence 1 is NOT more similar to 'deposit' than in sentence 2")
print("  - 'bank' in sentence 2 is NOT more similar to 'river' than in sentence 1")
print("  - Information lost!")

# Words most severely affected by static embedding limitation:
print("\nMost polysemous common words (each has 10+ distinct senses):")
print("  bank, lead, light, fair, bar, play, spring, right, can, left")

# Timeline of contextual embeddings

print("Evolution from static to contextual embeddings:")
print()
print("2013: Word2Vec (Google)")
print("  - Static embeddings, trained on context prediction")
print("  - 'bank' → one fixed 300-dim vector forever")
print()
print("2014: GloVe (Stanford)")
print("  - Static embeddings, trained on global co-occurrence")
print("  - Same polysemy limitation")
print()
print("2017: FastText (Facebook)")
print("  - Static embeddings, subword characters")
print("  - Solves OOV, still static per word")
print()
print("2018: ELMo (Allen NLP)")
print("  - FIRST contextual embeddings!")
print("  - Bidirectional LSTM, 'bank' gets different vector in each sentence")
print("  - SOTA on 6 NLP benchmarks when released")
print()
print("2018: BERT (Google)")
print("  - Transformer-based contextual embeddings (Lessons 53+)")
print("  - 'bank' in 'river bank' → completely different vector than")
print("    'bank' in 'bank account'")
print("  - Revolutionized ALL of NLP")
print()
print("2020+: GPT-3, GPT-4, T5, LLaMA, Gemini...")
print("  - Large-scale contextual models trained on internet-scale text")