AI / Machine Learning
/ Deep Learning —
PhD-Level Foundations

import numpy as np

# SVD decomposition
A = np.array([[3, 1], [2, 4], [0, 5]])
U, S, Vt = np.linalg.svd(A, full_matrices=False)
print(f"Singular values: {S}")

# Reconstruct A from SVD
A_approx = U @ np.diag(S) @ Vt

# PCA via SVD (zero-center first)
X = np.random.randn(100, 5)
X_centered = X - X.mean(axis=0)
U, S, Vt = np.linalg.svd(X_centered, full_matrices=False)
# First 2 principal components
X_pca = X_centered @ Vt[:2].T

from scipy import stats
import numpy as np

# MLE: fit a normal distribution to data
data = np.random.normal(loc=5.0, scale=2.0, size=1000)
mu, sigma = stats.norm.fit(data)
print(f"MLE estimate: mu={mu:.3f}, sigma={sigma:.3f}")

# Log-likelihood (what MLE maximizes)
log_lik = np.sum(stats.norm.logpdf(data, mu, sigma))

# KL divergence between two Gaussians
p = stats.norm(0, 1)
q = stats.norm(0.5, 1.2)
x = np.linspace(-5, 5, 1000)
kl = np.trapz(p.pdf(x) * (p.logpdf(x) - q.logpdf(x)), x)
print(f"KL divergence: {kl:.4f}")

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('clf', RandomForestClassifier(
        n_estimators=100,
        max_depth=5,
        random_state=42
    ))
])

# 5-fold cross-validation
scores = cross_val_score(pipeline, X, y, cv=5, scoring='roc_auc')
print(f"AUC: {scores.mean():.3f} +/- {scores.std():.3f}")

# Feature importances
pipeline.fit(X_train, y_train)
importances = pipeline['clf'].feature_importances_

import torch
import torch.nn as nn

class MLP(nn.Module):
    def __init__(self, in_dim, hidden, out_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_dim, hidden),
            nn.BatchNorm1d(hidden),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(hidden, out_dim)
        )
    def forward(self, x):
        return self.net(x)

model = MLP(128, 256, 10)
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

# Training step
optimizer.zero_grad()
logits = model(x_batch)
loss = criterion(logits, y_batch)
loss.backward()
optimizer.step()

import torch.nn as nn
import torchvision.models as models

# Build a simple CNN
class SimpleCNN(nn.Module):
    def __init__(self, n_classes):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.AdaptiveAvgPool2d((4, 4))
        )
        self.classifier = nn.Linear(64 * 16, n_classes)

# Transfer learning: freeze backbone, train head
backbone = models.resnet18(pretrained=True)
for param in backbone.parameters():
    param.requires_grad = False
backbone.fc = nn.Linear(512, n_classes)  # new head

from transformers import AutoTokenizer, AutoModel
import torch

# Load a pretrained encoder
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModel.from_pretrained("bert-base-uncased")

text = "Transformers changed NLP forever."
inputs = tokenizer(text, return_tensors="pt")

with torch.no_grad():
    outputs = model(**inputs)

# [CLS] token embedding as sentence representation
cls_embedding = outputs.last_hidden_state[:, 0, :]
print(f"Embedding shape: {cls_embedding.shape}")

# Attention weights (requires output_attentions=True)
outputs_attn = model(**inputs, output_attentions=True)
attn = outputs_attn.attentions  # tuple of (batch, heads, seq, seq)