INPUT (784 for 28x28 image)
  ↓
DENSE (128 units, ReLU)     # first hidden layer
  ↓
DROPOUT (0.3)               # regularization
  ↓
DENSE (64 units, ReLU)      # second hidden layer
  ↓
BATCH NORM                  # normalize activations
  ↓
DENSE (10 units, Softmax)   # output: 10 classes

Params: 784×128 + 128×64 + 64×10 = 109,xxx weights

# Convolutional layers learn spatial features
# Conv → Pool → Conv → Pool → Flatten → Dense

import torch.nn as nn

class CNN(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),  # 3ch→32ch
            nn.ReLU(),
            nn.MaxPool2d(2, 2),                           # /2 spatially
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(64 * 8 * 8, 256),
            nn.ReLU(),
            nn.Dropout(0.5),
            nn.Linear(256, num_classes)
        )
    def forward(self, x):
        x = self.features(x)      # conv layers
        x = x.flatten(start_dim=1) # flatten
        return self.classifier(x)  # dense layers

# Recurrent networks: hidden state carries memory across timesteps
import torch.nn as nn

# Simple RNN (vanishing gradient problem)
rnn = nn.RNN(input_size=50, hidden_size=128, batch_first=True)

# LSTM — solves vanishing gradient with gates
lstm = nn.LSTM(
    input_size=100,    # embedding dimension
    hidden_size=256,   # hidden state size
    num_layers=2,      # stacked LSTMs
    dropout=0.3,       # between layers
    bidirectional=True, # process forward AND backward
    batch_first=True
)

# GRU — simpler than LSTM, similar performance
gru = nn.GRU(input_size=100, hidden_size=256, batch_first=True)

# LSTM gates:
# Forget gate:  what to remove from cell state
# Input gate:   what new info to store
# Output gate:  what to output from cell state

# Self-attention: each token attends to all other tokens
# Q = Query, K = Key, V = Value

# Attention(Q,K,V) = softmax(QK^T / sqrt(dk)) × V

# Multi-head attention: multiple parallel attention heads
# Each head learns different relationship patterns

import torch.nn as nn

mha = nn.MultiheadAttention(embed_dim=512, num_heads=8, batch_first=True)

# Transformer encoder block:
# Input → MultiHeadAttention → Add&Norm → FFN → Add&Norm

# BERT: bidirectional encoder (good for classification, NER)
# GPT:  autoregressive decoder (good for text generation)
# T5:   encoder-decoder (good for seq2seq tasks)

# Positional encoding: adds position info since attention has no order
# Absolute (BERT) or Relative (RoPE in LLaMA) or Learned

# Key hyperparameters:
# num_heads: 8, 12, 16  |  d_model: 512, 768, 1024
# num_layers: 6, 12, 24 |  d_ff (FFN): 4×d_model

import torch
import torch.nn as nn
from torch.utils.data import DataLoader

device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = MyModel().to(device)
optimiser = torch.optim.AdamW(model.parameters(), lr=3e-4, weight_decay=0.01)
criterion = nn.CrossEntropyLoss()
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimiser, T_max=10)

for epoch in range(num_epochs):
    # ── Training ──
    model.train()
    for X, y in train_loader:
        X, y = X.to(device), y.to(device)
        optimiser.zero_grad()       # 1. clear gradients
        pred = model(X)             # 2. forward pass
        loss = criterion(pred, y)   # 3. compute loss
        loss.backward()             # 4. backprop
        nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # 5. clip
        optimiser.step()            # 6. update weights
    scheduler.step()

    # ── Validation ──
    model.eval()
    with torch.no_grad():           # disable gradient tracking
        for X, y in val_loader:
            X, y = X.to(device), y.to(device)
            val_pred = model(X)

import tensorflow as tf
from tensorflow import keras

# Sequential API
model = keras.Sequential([
    keras.layers.Dense(256, activation='relu', input_shape=(784,)),
    keras.layers.BatchNormalization(),
    keras.layers.Dropout(0.3),
    keras.layers.Dense(128, activation='relu'),
    keras.layers.Dense(10, activation='softmax')
])

model.compile(
    optimizer=keras.optimizers.Adam(learning_rate=1e-3),
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

callbacks = [
    keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True),
    keras.callbacks.ReduceLROnPlateau(factor=0.5, patience=3),
    keras.callbacks.ModelCheckpoint('best.keras', save_best_only=True)
]

history = model.fit(
    X_train, y_train,
    epochs=50, batch_size=64,
    validation_split=0.2,
    callbacks=callbacks
)

model.evaluate(X_test, y_test)
predictions = model.predict(X_new)