import numpy as np

# One neuron with 3 inputs
# -----------------------
# Inputs: a 3-dimensional feature vector
x = np.array([0.5, -1.2, 0.8])   # e.g., [height, weight, age] (normalized)

# Weights: one per input — these are LEARNED during training
w = np.array([0.4, -0.3, 0.7])   # current weights

# Bias: a single learned scalar
b = 0.1

# Step 1: Weighted sum (the "summation" in the cell body)
z = np.dot(w, x) + b
# Equivalently: z = w[0]*x[0] + w[1]*x[1] + w[2]*x[2] + b
print(f"Weighted sum z = w·x + b = {z:.4f}")
# z = 0.4*0.5 + (-0.3)*(-1.2) + 0.7*0.8 + 0.1
# z = 0.2 + 0.36 + 0.56 + 0.1 = 1.22

# Step 2: Activation function (the "threshold" decision)
def sigmoid(z):
    return 1 / (1 + np.exp(-z))

output = sigmoid(z)
print(f"Neuron output = sigmoid({z:.4f}) = {output:.4f}")
# A single number between 0 and 1 — the neuron's "firing strength"

import numpy as np

# Implement a single neuron as a function
def neuron(x, w, b, activation='sigmoid'):
    """
    x: input vector, shape (n,)
    w: weight vector, shape (n,)
    b: bias scalar
    """
    z = np.dot(w, x) + b   # weighted sum

    if activation == 'sigmoid':
        return 1 / (1 + np.exp(-z))
    elif activation == 'relu':
        return np.maximum(0, z)
    elif activation == 'tanh':
        return np.tanh(z)
    else:
        return z  # linear (no activation)

# Example 1: Detecting "is income high and debt is low?"
# Input: [income_normalised, debt_normalised]
# High income = positive feature; high debt = negative feature
w = np.array([2.0, -2.0])   # income matters positively, debt negatively
b = -0.5                     # threshold: need reasonable net signal to fire
x_rich_low_debt  = np.array([1.0, 0.2])   # high income, low debt
x_poor_high_debt = np.array([0.3, 0.9])   # low income, high debt
x_medium         = np.array([0.6, 0.5])   # medium both

print("Is this person a good credit risk?")
print(f"  Rich, low debt:   {neuron(x_rich_low_debt, w, b):.4f}  (close to 1 = good)")
print(f"  Poor, high debt:  {neuron(x_poor_high_debt, w, b):.4f}  (close to 0 = bad)")
print(f"  Medium:           {neuron(x_medium, w, b):.4f}  (uncertain)")

# Example 2: Role of bias
print("\nEffect of bias on activation threshold:")
w_simple = np.array([1.0])
x_half   = np.array([0.0])   # zero input

for b_val in [-3, 0, 3]:
    out = neuron(x_half, w_simple, b_val)
    print(f"  bias={b_val:+.0f}, input=0: output = {out:.4f}")

import numpy as np

def step(z):
    """Step activation: 1 if z > 0, else 0"""
    return 1 if z > 0 else 0

def perceptron_train(X, y, lr=0.1, epochs=100):
    """Manually train a perceptron using the perceptron learning rule."""
    n_features = X.shape[1]
    w = np.zeros(n_features)
    b = 0.0

    for epoch in range(epochs):
        errors = 0
        for xi, yi in zip(X, y):
            z = np.dot(w, xi) + b
            y_pred = step(z)
            error = yi - y_pred       # 0, +1, or -1
            w += lr * error * xi     # adjust weights
            b += lr * error          # adjust bias
            errors += abs(error)
        if errors == 0:
            print(f"Converged at epoch {epoch+1}")
            break
    return w, b

# ── AND gate ──
# Output is 1 only when BOTH inputs are 1
X_and = np.array([[0,0], [0,1], [1,0], [1,1]])
y_and = np.array([0, 0, 0, 1])

w, b = perceptron_train(X_and, y_and)
print(f"\nAND gate learned weights: w={w}, b={b:.2f}")
print("Predictions:")
for xi, yi in zip(X_and, y_and):
    pred = step(np.dot(w, xi) + b)
    print(f"  {xi} → {pred}  (correct: {yi})")

# ── OR gate ──
# Output is 1 when AT LEAST ONE input is 1
X_or = np.array([[0,0], [0,1], [1,0], [1,1]])
y_or = np.array([0, 1, 1, 1])

w_or, b_or = perceptron_train(X_or, y_or)
print(f"\nOR gate learned weights: w={w_or}, b={b_or:.2f}")
for xi, yi in zip(X_or, y_or):
    pred = step(np.dot(w_or, xi) + b_or)
    print(f"  {xi} → {pred}  (correct: {yi})")

# ── XOR gate — the Perceptron FAILS ──
X_xor = np.array([[0,0], [0,1], [1,0], [1,1]])
y_xor = np.array([0, 1, 1, 0])   # XOR: 1 only when inputs differ

# The perceptron will never converge on XOR
# Let's show this with a limited epoch count
w_xor, b_xor = perceptron_train(X_xor, y_xor, lr=0.1, epochs=1000)

print("\nXOR — predictions after 1000 epochs:")
errors = 0
for xi, yi in zip(X_xor, y_xor):
    pred = step(np.dot(w_xor, xi) + b_xor)
    correct = "✓" if pred == yi else "✗"
    print(f"  {xi} → {pred}  (correct: {yi}) {correct}")
    if pred != yi: errors += 1
print(f"Total errors: {errors}/4 — the perceptron cannot solve XOR!")

import numpy as np

# Manual 2-layer network solving XOR
# Architecture: 2 inputs → 2 hidden neurons (sigmoid) → 1 output (sigmoid)
# We'll set the weights manually to show how it works

# These weights were chosen to implement XOR:
# Hidden neuron 1: fires when (x1=1 OR x2=1) but NOT (x1=1 AND x2=1)
# Hidden neuron 2: fires when (x1=1 AND x2=1)
# Output: fires when h1 fires AND h2 doesn't fire

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

# Weights from input layer to hidden layer (2x2 matrix)
# W1[i,j] = weight from input j to hidden neuron i
W1 = np.array([[20, 20],    # Hidden neuron 1: large positive weights → OR-like
               [20, 20]])   # Hidden neuron 2: large positive weights → AND-like
b1 = np.array([-10, -30])   # Bias 1: fires with >= 1 input; Bias 2: fires only with both

# Weights from hidden layer to output (1x2)
W2 = np.array([[20, -20]])  # Output: fires if h1 fires, inhibited if h2 fires
b2 = np.array([-10])        # Threshold for output

def forward_xor(x):
    # Hidden layer
    z1 = W1 @ x + b1           # shape (2,)
    h1 = sigmoid(z1)            # hidden activations
    # Output layer
    z2 = W2 @ h1 + b2           # shape (1,)
    output = sigmoid(z2)        # final output
    return h1, output[0]

X_xor = np.array([[0,0], [0,1], [1,0], [1,1]])
y_xor = np.array([0, 1, 1, 0])

print("XOR solved with a 2-layer network:")
print(f"{'Input':>10} | {'Hidden h1, h2':>20} | {'Output':>8} | {'Target':>6}")
print("-" * 60)
for x, y in zip(X_xor, y_xor):
    h, out = forward_xor(x)
    print(f"  {x} | {h[0]:.3f}, {h[1]:.3f}    | {out:.4f}   | {y}")

import numpy as np

# ── Build a complete 2-layer MLP from scratch in NumPy ──
# Architecture: 2 inputs → 4 hidden neurons (ReLU) → 1 output (sigmoid)
# Task: binary classification

class MLP:
    def __init__(self, input_dim, hidden_dim, output_dim, seed=42):
        np.random.seed(seed)
        # Layer 1 weights: shape (hidden_dim, input_dim)
        # Xavier initialization (more on this in Section 8)
        scale1 = np.sqrt(2.0 / input_dim)
        self.W1 = np.random.randn(hidden_dim, input_dim) * scale1
        self.b1 = np.zeros(hidden_dim)

        # Layer 2 weights: shape (output_dim, hidden_dim)
        scale2 = np.sqrt(2.0 / hidden_dim)
        self.W2 = np.random.randn(output_dim, hidden_dim) * scale2
        self.b2 = np.zeros(output_dim)

    def relu(self, z):
        return np.maximum(0, z)

    def sigmoid(self, z):
        return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

    def forward(self, X):
        """
        X: shape (batch_size, input_dim)
        Returns: output probabilities, shape (batch_size, output_dim)
        """
        # Layer 1: linear + ReLU
        # X.shape: (batch, input_dim)
        # W1.T shape: (input_dim, hidden_dim)
        self.a1 = X @ self.W1.T + self.b1      # (batch, hidden_dim)
        self.h1 = self.relu(self.a1)            # (batch, hidden_dim)

        # Layer 2: linear + sigmoid
        self.a2 = self.h1 @ self.W2.T + self.b2  # (batch, output_dim)
        self.output = self.sigmoid(self.a2)       # (batch, output_dim)

        return self.output

    def predict(self, X, threshold=0.5):
        probs = self.forward(X)
        return (probs >= threshold).astype(int)

# Test the MLP
np.random.seed(0)
model = MLP(input_dim=2, hidden_dim=4, output_dim=1)

# Simple circular dataset: class 1 = inside circle, class 0 = outside
n = 200
angles = np.random.uniform(0, 2*np.pi, n)
radii  = np.random.uniform(0, 2, n)
X = np.column_stack([radii * np.cos(angles), radii * np.sin(angles)])
y = (radii < 1.2).astype(int).reshape(-1, 1)  # inside radius 1.2 = class 1

# Forward pass
probs = model.forward(X)
predictions = model.predict(X)

# Accuracy before training (random weights)
accuracy = (predictions == y).mean()
print(f"MLP architecture: 2 → 4 (ReLU) → 1 (Sigmoid)")
print(f"Parameters: W1={model.W1.shape}, b1={model.b1.shape}, W2={model.W2.shape}, b2={model.b2.shape}")
print(f"Total parameters: {model.W1.size + model.b1.size + model.W2.size + model.b2.size}")
print(f"Accuracy with random weights (before training): {accuracy:.4f}")
print(f"Output range: [{probs.min():.4f}, {probs.max():.4f}]")

import numpy as np

# ── The four core activation functions ──

def sigmoid(x):
    """Squashes input to (0, 1). Good for output layer in binary classification."""
    return 1 / (1 + np.exp(-np.clip(x, -500, 500)))

def sigmoid_derivative(x):
    s = sigmoid(x)
    return s * (1 - s)   # Maximum value: 0.25 at x=0

def tanh(x):
    """Squashes input to (-1, 1). Zero-centered — better than sigmoid for hidden layers."""
    return np.tanh(x)

def tanh_derivative(x):
    return 1 - np.tanh(x)**2  # Maximum value: 1.0 at x=0

def relu(x):
    """Most popular hidden activation. Fast, solves vanishing gradient for + inputs."""
    return np.maximum(0, x)

def relu_derivative(x):
    return (x > 0).astype(float)   # 1 if x > 0, 0 if x <= 0

def softmax(x):
    """Converts a vector of scores to probabilities. Used in multi-class output."""
    e_x = np.exp(x - np.max(x))  # subtract max for numerical stability
    return e_x / e_x.sum()

# ── Compare on a range of inputs ──
x_vals = np.array([-3, -1, 0, 1, 3])

print(f"{'x':>6} | {'sigmoid':>8} | {'tanh':>8} | {'ReLU':>8}")
print("-" * 42)
for x in x_vals:
    print(f"{x:>6.1f} | {sigmoid(x):>8.4f} | {tanh(x):>8.4f} | {relu(x):>8.4f}")

# ── Softmax example for multi-class output ──
print("\nSoftmax example (3-class classification):")
scores = np.array([2.5, 1.0, -0.5])     # Raw output scores (logits)
probs  = softmax(scores)
print(f"Logits: {scores}")
print(f"Probabilities: {probs.round(4)}")
print(f"Sum of probabilities: {probs.sum():.4f}")   # Always 1.0
print(f"Predicted class: {np.argmax(probs)}")

import numpy as np

# ── Leaky ReLU: a fix for dying neurons ──
def leaky_relu(x, alpha=0.01):
    """
    For negative inputs: output is alpha * x (not zero).
    This ensures a small gradient always flows, preventing dead neurons.
    """
    return np.where(x > 0, x, alpha * x)

def leaky_relu_derivative(x, alpha=0.01):
    return np.where(x > 0, 1.0, alpha)

# ── GELU: used in BERT, GPT, modern Transformers ──
def gelu(x):
    """
    Gaussian Error Linear Unit.
    Smooth approximation of ReLU. 
    Outperforms ReLU in large language models.
    """
    return 0.5 * x * (1 + np.tanh(np.sqrt(2/np.pi) * (x + 0.044715 * x**3)))

# ── ELU: Exponential Linear Unit ──
def elu(x, alpha=1.0):
    """For negative inputs: alpha*(exp(x)-1) — smooth, negative mean → addresses bias shift."""
    return np.where(x > 0, x, alpha * (np.exp(x) - 1))

# Compare at key values
x_vals = np.array([-2, -1, -0.5, 0, 0.5, 1, 2])
print(f"{'x':>6} | {'ReLU':>6} | {'LeakyReLU':>10} | {'ELU':>7} | {'GELU':>7}")
print("-" * 50)
for x in x_vals:
    print(f"{x:>6.1f} | {max(0,x):>6.3f} | {leaky_relu(x):>10.4f} | {elu(x):>7.4f} | {gelu(x):>7.4f}")

# Key property: for x > 0, all are identical or near-identical to ReLU
# For x < 0, they differ — Leaky/ELU/GELU allow gradient to flow

import numpy as np

def show_activation_statistics(W, name):
    """
    Forward pass through a deep network with given init.
    Track activation mean and std at each layer.
    """
    x = np.random.randn(1000, 100)   # 1000 samples, 100 features
    print(f"\n{'='*50}")
    print(f"Init: {name}")
    print(f"{'Layer':>6} | {'Mean':>8} | {'Std':>8} | {'Dead %':>8}")
    print("-" * 40)
    for layer in range(10):
        z = x @ W[layer].T
        x = np.maximum(0, z)  # ReLU
        dead_frac = (x == 0).mean() * 100
        print(f"{layer+1:>6} | {x.mean():>8.4f} | {x.std():>8.4f} | {dead_frac:>7.1f}%")

np.random.seed(42)
n_layers, fan_in, fan_out = 10, 100, 100

# Bad init 1: Too small (std = 0.01)
W_small = [np.random.randn(fan_out, fan_in) * 0.01 for _ in range(n_layers)]
show_activation_statistics(W_small, "Too small (std=0.01)")

# Bad init 2: Too large (std = 1.0)
W_large = [np.random.randn(fan_out, fan_in) * 1.0  for _ in range(n_layers)]
show_activation_statistics(W_large, "Too large (std=1.0)")

# Good init: Kaiming/He (std = sqrt(2/fan_in))
kaiming_std = np.sqrt(2.0 / fan_in)
W_kaiming = [np.random.randn(fan_out, fan_in) * kaiming_std for _ in range(n_layers)]
show_activation_statistics(W_kaiming, "Kaiming/He (std=sqrt(2/fan_in))")