Backpropagation
& Training

# Consider: L = (wx + b)²  — a tiny "network" with one weight
# Forward: store intermediate values

w, x, b = 2.0, 3.0, 1.0

z = w * x            # 6.0
a = z + b            # 7.0
L = a ** 2           # 49.0  ← the "loss"

# Backward: chain rule from L back to w
dL_da = 2 * a        # dL/da = 2a = 14.0  (power rule)
da_dz = 1.0          # da/dz = 1  (add gate passes gradient unchanged)
dz_dw = x            # dz/dw = x = 3.0  (multiply gate)

dL_dw = dL_da * da_dz * dz_dw   # 14 × 1 × 3 = 42.0

# Verify with finite difference (numerical gradient check)
eps = 1e-5
L_plus  = ((w + eps) * x + b) ** 2
L_minus = ((w - eps) * x + b) ** 2
numerical_grad = (L_plus - L_minus) / (2 * eps)

print(f"Analytical: {dL_dw:.4f}")        # 42.0000
print(f"Numerical:  {numerical_grad:.4f}")  # 42.0000  ✓
# They match — the chain rule derivation is correct

import numpy as np

class LinearReLU:
    def __init__(self, in_dim, out_dim):
        self.W = np.random.randn(in_dim, out_dim) * 0.01
        self.b = np.zeros(out_dim)

    def forward(self, x):
        self.x = x                    # cache for backward
        self.z = x @ self.W + self.b  # linear: (batch, out_dim)
        self.a = np.maximum(0, self.z)  # ReLU
        return self.a

    def backward(self, dL_da, lr=0.01):
        # Chain rule: dL/dz = dL/da * ReLU'(z)
        dL_dz = dL_da * (self.z > 0)     # gate the gradient

        # Gradients w.r.t. parameters
        dL_dW = self.x.T @ dL_dz / len(self.x)  # avg over batch
        dL_db = dL_dz.mean(axis=0)

        # Gradient to pass to the previous layer
        dL_dx = dL_dz @ self.W.T

        # Update parameters
        self.W -= lr * dL_dW
        self.b -= lr * dL_db

        return dL_dx   # pass backward

# Tiny forward + backward test
layer = LinearReLU(3, 4)
x = np.random.randn(8, 3)  # batch=8, 3 features

output = layer.forward(x)
print("Output shape:", output.shape)  # (8, 4)

# Fake upstream gradient (same shape as output)
fake_upstream = np.random.randn(*output.shape)
grad_x = layer.backward(fake_upstream)
print("Grad input shape:", grad_x.shape)  # (8, 3) — matches input ✓

import numpy as np

class Adam:
    def __init__(self, lr=1e-3, beta1=0.9, beta2=0.999, eps=1e-8):
        self.lr = lr
        self.b1, self.b2, self.eps = beta1, beta2, eps
        self.m = {}   # first moments (per parameter)
        self.v = {}   # second moments
        self.t = 0    # time step

    def update(self, params, grads):
        self.t += 1
        updated = {}

        for key in params:
            if key not in self.m:
                self.m[key] = np.zeros_like(params[key])
                self.v[key] = np.zeros_like(params[key])

            g = grads[key]

            # Update biased first and second moment estimates
            self.m[key] = self.b1 * self.m[key] + (1 - self.b1) * g
            self.v[key] = self.b2 * self.v[key] + (1 - self.b2) * g**2

            # Bias-corrected estimates
            m_hat = self.m[key] / (1 - self.b1**self.t)
            v_hat = self.v[key] / (1 - self.b2**self.t)

            # Parameter update
            updated[key] = params[key] - self.lr * m_hat / (np.sqrt(v_hat) + self.eps)

        return updated

# Usage
optimizer = Adam(lr=0.001)
params  = {'W1': np.random.randn(4, 2), 'b1': np.zeros(4)}
grads   = {'W1': np.random.randn(4, 2), 'b1': np.random.randn(4)}
params  = optimizer.update(params, grads)

import numpy as np

def cosine_schedule(epoch, total_epochs, lr_min=1e-6, lr_max=1e-3):
    """Smoothly decrease lr from lr_max to lr_min using cosine curve."""
    return lr_min + 0.5 * (lr_max - lr_min) * (1 + np.cos(np.pi * epoch / total_epochs))

def linear_warmup_cosine(step, warmup_steps, total_steps, lr_max=1e-3):
    """Warmup then cosine decay — standard transformer schedule."""
    if step < warmup_steps:
        return lr_max * step / warmup_steps   # linear warmup
    progress = (step - warmup_steps) / (total_steps - warmup_steps)
    return lr_max * 0.5 * (1 + np.cos(np.pi * progress))  # cosine decay

# Visualize the schedule
total = 100
lrs_cosine = [cosine_schedule(e, total) for e in range(total)]
lrs_warmup = [linear_warmup_cosine(s, warmup_steps=10, total_steps=total) for s in range(total)]

print(f"Epoch 0:  lr = {lrs_warmup[0]:.6f}")   # starts near 0
print(f"Epoch 10: lr = {lrs_warmup[10]:.6f}")  # peak after warmup
print(f"Epoch 50: lr = {lrs_warmup[50]:.6f}")  # halfway down cosine
print(f"Epoch 99: lr = {lrs_warmup[99]:.6f}")  # near 0 at end

import numpy as np

def sigmoid(x): return 1 / (1 + np.exp(-x))
def sigmoid_grad(x): s = sigmoid(x); return s * (1 - s)

# Simulate gradient flow through 20 sigmoid layers
n_layers = 20
x = np.random.randn(100)  # initial activations

grad = np.ones(100)  # gradient starts as 1 at the output
grad_norms_sigmoid = []

for _ in range(n_layers):
    grad = grad * sigmoid_grad(x)  # chain rule through sigmoid
    grad_norms_sigmoid.append(np.linalg.norm(grad))
    x = sigmoid(x)

# Same thing with ReLU
x = np.random.randn(100)
grad = np.ones(100)
grad_norms_relu = []

for _ in range(n_layers):
    grad = grad * (x > 0)   # ReLU gradient: 1 or 0
    grad_norms_relu.append(np.linalg.norm(grad))
    x = np.maximum(0, x)

print("Sigmoid gradient norms through 20 layers:")
print(f"  Layer  1: {grad_norms_sigmoid[0]:.6f}")
print(f"  Layer 10: {grad_norms_sigmoid[9]:.6f}")
print(f"  Layer 20: {grad_norms_sigmoid[19]:.8f}")  # ≈ 0.000000xx

print("\nReLU gradient norms through 20 layers:")
print(f"  Layer  1: {grad_norms_relu[0]:.4f}")
print(f"  Layer 20: {grad_norms_relu[19]:.4f}")     # stays more stable