Convolutional
Neural Networks

import torch
import torch.nn as nn

# Input: batch=8, channels=3, height=32, width=32 (like CIFAR-10)
x = torch.randn(8, 3, 32, 32)

# Conv layer: 3 input channels → 32 filters, 3×3 kernel, padding=1
conv = nn.Conv2d(in_channels=3, out_channels=32,
                  kernel_size=3, stride=1, padding=1)
out = conv(x)
print("After conv:", out.shape)   # (8, 32, 32, 32) — same spatial, 32 channels

# Max pooling 2×2 — halves spatial dimensions
pool = nn.MaxPool2d(kernel_size=2, stride=2)
out = pool(out)
print("After pool:", out.shape)    # (8, 32, 16, 16)

# Second conv block
conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
out = pool(conv2(out))
print("After 2nd block:", out.shape)  # (8, 64, 8, 8)

# Global average pooling — collapses spatial dims
gap = nn.AdaptiveAvgPool2d(1)
out = gap(out).squeeze(-1).squeeze(-1)
print("After GAP:", out.shape)       # (8, 64) — ready for linear classifier

# How many parameters does each conv layer have?
# Conv2d(3, 32, 3×3): 3×32×3×3 + 32 bias = 896 params
# vs FC layer connecting 32×32×3 to 32: 98,304 params — 110× more
n_params_conv = sum(p.numel() for p in conv.parameters())
print(f"Conv params: {n_params_conv}")  # 896

import torch
import torch.nn as nn
import torch.nn.functional as F

class ResidualBlock(nn.Module):
    """Basic ResNet block: two 3×3 convs with a skip connection."""
    def __init__(self, channels, stride=1):
        super().__init__()
        self.conv1 = nn.Conv2d(channels, channels, 3, stride=stride, padding=1, bias=False)
        self.bn1   = nn.BatchNorm2d(channels)
        self.conv2 = nn.Conv2d(channels, channels, 3, stride=1,      padding=1, bias=False)
        self.bn2   = nn.BatchNorm2d(channels)

        # If stride > 1, input spatial dims change — need a projection shortcut
        self.shortcut = nn.Identity()
        if stride != 1:
            self.shortcut = nn.Sequential(
                nn.Conv2d(channels, channels, 1, stride=stride, bias=False),
                nn.BatchNorm2d(channels)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))  # first conv block
        out = self.bn2(self.conv2(out))           # second conv (no activation yet)
        out = out + self.shortcut(x)           # ADD the skip connection ← the key
        out = F.relu(out)                       # activation after addition
        return out

# Test it
block = ResidualBlock(channels=64)
x = torch.randn(4, 64, 16, 16)
out = block(x)
print(out.shape)   # (4, 64, 16, 16) — same shape as input

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.utils.data import DataLoader

# ── Architecture ────────────────────────────────────────────────────────
class SmallResNet(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        # Stem: initial feature extraction
        self.stem = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1, bias=False),
            nn.BatchNorm2d(32),
            nn.ReLU()
        )
        # Residual blocks — gradually increase channels, reduce spatial
        self.layer1 = self._make_layer(32, n_blocks=2)
        self.layer2 = self._make_layer(64, n_blocks=2, downsample=True)
        self.layer3 = self._make_layer(128, n_blocks=2, downsample=True)
        # Classifier head
        self.head = nn.Sequential(
            nn.AdaptiveAvgPool2d(1),   # global average pool → (B, 128, 1, 1)
            nn.Flatten(),              # → (B, 128)
            nn.Linear(128, num_classes)
        )

    def _make_layer(self, channels, n_blocks, downsample=False):
        layers = []
        if downsample:
            layers.append(nn.Conv2d(channels//2, channels, 1, stride=2, bias=False))
            layers.append(nn.BatchNorm2d(channels))
            layers.append(nn.ReLU())
        for _ in range(n_blocks):
            layers.append(ResidualBlock(channels))
        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.stem(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        return self.head(x)

# ── Data ────────────────────────────────────────────────────────────────
train_tf = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize([0.4914,0.4822,0.4465],[0.247,0.243,0.261])
])
val_tf = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.4914,0.4822,0.4465],[0.247,0.243,0.261])
])
train_loader = DataLoader(datasets.CIFAR10('./data',train=True, transform=train_tf,download=True),batch_size=128,shuffle=True,num_workers=2)
val_loader   = DataLoader(datasets.CIFAR10('./data',train=False,transform=val_tf), batch_size=256,num_workers=2)

# ── Training ─────────────────────────────────────────────────────────────
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = SmallResNet().to(device)
optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=0.01)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=100)
criterion = nn.CrossEntropyLoss()

print(f"Model params: {sum(p.numel() for p in model.parameters()):,}")
# ~350K params — achieves ~88% accuracy on CIFAR-10 after 100 epochs

import torch
import torch.nn as nn
import torchvision.models as models

# ── Load pretrained ResNet-50 ─────────────────────────────────────────
model = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)

# ── Strategy 1: Feature Extraction — freeze everything ───────────────
for param in model.parameters():
    param.requires_grad = False   # freeze all layers

# Replace the final classification head (1000 classes → your N classes)
n_classes = 5   # e.g. 5-class flower dataset
model.fc = nn.Sequential(
    nn.Dropout(0.3),
    nn.Linear(model.fc.in_features, n_classes)
)
# Only model.fc parameters have requires_grad=True
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f"Trainable params: {trainable:,}")   # ~10K vs 25M total

# ── Strategy 2: Fine-tuning — unfreeze for second stage ──────────────
# After training the head for ~5 epochs, unfreeze everything
for param in model.parameters():
    param.requires_grad = True

# Use differential learning rates
import torch.optim as optim
optimizer = optim.AdamW([
    {'params': model.layer1.parameters(), 'lr': 1e-5},  # frozen-ish
    {'params': model.layer2.parameters(), 'lr': 1e-5},
    {'params': model.layer3.parameters(), 'lr': 5e-5},
    {'params': model.layer4.parameters(), 'lr': 1e-4},
    {'params': model.fc.parameters(),     'lr': 1e-3},  # head trains fastest
], weight_decay=0.01)