Embeddings Deep Dive

import numpy as np

def cosine_similarity(a: list[float], b: list[float]) -> float:
    """Calculate cosine similarity between two vectors."""
    a, b = np.array(a), np.array(b)
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

def euclidean_distance(a: list[float], b: list[float]) -> float:
    """Calculate Euclidean distance between two vectors."""
    a, b = np.array(a), np.array(b)
    return np.linalg.norm(a - b)

# Example
vec_king = [0.2, 0.8, 0.5, 0.1]
vec_queen = [0.21, 0.79, 0.48, 0.12]
vec_car = [0.9, 0.1, 0.3, 0.7]

print(f"king ↔ queen: {cosine_similarity(vec_king, vec_queen):.4f}")  # ~0.999
print(f"king ↔ car:   {cosine_similarity(vec_king, vec_car):.4f}")    # ~0.56

from openai import OpenAI
import numpy as np
from typing import List

client = OpenAI()

def get_embeddings(texts: List[str], model: str = "text-embedding-3-small") -> List[List[float]]:
    """Batch embed texts using OpenAI API.
    
    The API supports up to 2048 texts per request.
    For larger batches, chunk into groups.
    """
    # Remove newlines (they degrade quality)
    texts = [t.replace("\n", " ").strip() for t in texts]
    
    response = client.embeddings.create(
        input=texts,
        model=model
    )
    
    # Return embeddings in input order
    return [item.embedding for item in response.data]


def batch_embed_large(texts: List[str], batch_size: int = 1000) -> List[List[float]]:
    """Handle large document collections with batching."""
    all_embeddings = []
    
    for i in range(0, len(texts), batch_size):
        batch = texts[i:i + batch_size]
        embeddings = get_embeddings(batch)
        all_embeddings.extend(embeddings)
        print(f"Embedded {min(i + batch_size, len(texts))}/{len(texts)}")
    
    return all_embeddings


# Usage: Semantic similarity search
documents = [
    "Python is a high-level programming language",
    "Machine learning models require training data",
    "The cat sat on the mat",
    "Deep learning uses neural networks with many layers",
    "JavaScript runs in web browsers",
    "Natural language processing analyzes human text",
]

# Embed all documents
doc_embeddings = get_embeddings(documents)

# Embed a query
query = "How do neural networks learn?"
query_embedding = get_embeddings([query])[0]

# Calculate similarities
similarities = [
    cosine_similarity(query_embedding, doc_emb) 
    for doc_emb in doc_embeddings
]

# Rank by similarity
ranked = sorted(
    zip(documents, similarities), 
    key=lambda x: x[1], 
    reverse=True
)

print("\nQuery:", query)
print("\nResults (ranked by similarity):")
for doc, score in ranked[:3]:
    print(f"  [{score:.4f}] {doc}")

# Output:
#   [0.8234] Deep learning uses neural networks with many layers
#   [0.7891] Machine learning models require training data
#   [0.7102] Natural language processing analyzes human text

from sentence_transformers import SentenceTransformer
import numpy as np

# Load model (downloads ~90MB on first use)
model = SentenceTransformer('all-MiniLM-L6-v2')

# Embed texts (runs on CPU or GPU)
sentences = [
    "Embeddings capture semantic meaning",
    "Vector representations encode text as numbers",
    "The weather is sunny today",
]

# Returns numpy array of shape (3, 384)
embeddings = model.encode(sentences, show_progress_bar=True)

print(f"Shape: {embeddings.shape}")  # (3, 384)
print(f"Type: {type(embeddings)}")   # numpy.ndarray

# Similarity between first two (semantically related)
sim_12 = np.dot(embeddings[0], embeddings[1]) / (
    np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1])
)
print(f"Semantic vs Vector: {sim_12:.4f}")  # ~0.72

# Similarity between first and third (unrelated)
sim_13 = np.dot(embeddings[0], embeddings[2]) / (
    np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[2])
)
print(f"Semantic vs Weather: {sim_13:.4f}")  # ~0.15

from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt

# Assume `embeddings` is shape (N, 1536) from OpenAI
# and `labels` is a list of category labels

# PCA: reduce to 2D
pca = PCA(n_components=2)
coords_pca = pca.fit_transform(embeddings)
print(f"Variance explained: {pca.explained_variance_ratio_.sum():.2%}")

# t-SNE: reduce to 2D (better for visualization)
tsne = TSNE(n_components=2, perplexity=30, random_state=42)
coords_tsne = tsne.fit_transform(embeddings)

# Plot t-SNE results
plt.figure(figsize=(10, 8))
categories = list(set(labels))
colors = plt.cm.tab10(range(len(categories)))

for cat, color in zip(categories, colors):
    mask = [l == cat for l in labels]
    plt.scatter(
        coords_tsne[mask, 0], 
        coords_tsne[mask, 1], 
        c=[color], label=cat, alpha=0.7
    )

plt.legend()
plt.title("Document Embeddings (t-SNE)")
plt.savefig("embedding_clusters.png", dpi=150)

# OpenAI's native dimensionality reduction (v3 models only)
response = client.embeddings.create(
    input="Hello world",
    model="text-embedding-3-large",
    dimensions=256  # Reduce from 3072 → 256 (saves storage + search speed)
)

# The returned vector is already normalized at the lower dimension
embedding = response.data[0].embedding
print(len(embedding))  # 256