Q-Learning Algorithm

This algorithm uses a value-based approach for discrete state-action spaces. The update rule follows Bellman equation:

$$ Q(s,a) \leftarrow Q(s,a) + \alpha [r + \gamma \max_{a'} Q(s',a') - Q(s,a)] $$ Implementation example using OpenAI Gym:

import numpy as np
import gym

env = gym.make('CartPole-v1')
state_space = env.observation_space.shape[0]
action_space = env.action_space.n

q_table = np.zeros((state_space, action_space))
learning_rate = 0.1
discount_factor = 0.99
epsilon = 0.1
episodes = 1000

for _ in range(episodes):
    state = env.reset()
    done = False
    while not done:
        if np.random.rand() < epsilon:
            action = env.action_space.sample()
        else:
            action = np.argmax(q_table[state])
        
        next_state, reward, done, _ = env.step(action)
        q_table[state, action] += learning_rate * (reward + discount_factor * np.max(q_table[next_state]) - q_table[state, action])
        state = next_state

print("Training completed")

Deep Q-Networks (DQN)

Combines Q-learning with neural networks. Key components include experience replay and target networks.

import torch
import torch.nn as nn
import gym
from collections import deque
import random

class DQN(nn.Module):
    def __init__(self, input_dim, output_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(input_dim, 128),
            nn.ReLU(),
            nn.Linear(128, 64),
            nn.ReLU(),
            nn.Linear(64, output_dim)
        )
    
    def forward(self, x):
        return self.net(x)

env = gym.make('MountainCar-v0')
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n

model = DQN(state_dim, action_dim)
optimizer = torch.optim.Adam(model.parameters())
replay_buffer = deque(maxlen=10000)
batch_size = 32
gamma = 0.99
epsilon = 1.0

for episode in range(200):
    state = env.reset()
    done = False
    while not done:
        if random.random() < epsilon:
            action = env.action_space.sample()
        else:
            with torch.no_grad():
                action = model(torch.tensor(state)).argmax().item()
        
        next_state, reward, done, _ = env.step(action)
        replay_buffer.append((state, action, reward, next_state, done))
        state = next_state
    
    if len(replay_buffer) > batch_size:
        batch = random.sample(replay_buffer, batch_size)
        states, actions, rewards, next_states, dones = zip(*batch)
        
        states = torch.tensor(np.array(states))
        next_states = torch.tensor(np.array(next_states))
        actions = torch.tensor(actions)
        rewards = torch.tensor(rewards)
        dones = torch.tensor(dones)
        
        current_q = model(states).gather(1, actions.unsqueeze(1)).squeeze()
        next_q = model(next_states).max(1)[0]
        target_q = rewards + (1 - dones) * gamma * next_q
        
        loss = nn.MSELoss()(current_q, target_q)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    epsilon *= 0.995

Generative Models

Generative Adversarial Networks (GAN)

Composed of generator and discriminator networks competing through minimax game.

import torch
import torch.nn as nn
import torch.optim as optim

class Generator(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(100, 256),
            nn.ReLU(),
            nn.Linear(256, 128),
            nn.ReLU(),
            nn.Linear(128, 1)
        )
    
    def forward(self, x):
        return self.net(x)

class Discriminator(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(1, 128),
            nn.ReLU(),
            nn.Linear(128, 1),
            nn.Sigmoid()
        )
    
    def forward(self, x):
        return self.net(x)

generator = Generator()
discriminator = Discriminator()
g_optim = optim.Adam(generator.parameters())
d_optim = optim.Adam(discriminator.parameters())
criterion = nn.BCELoss()

for epoch in range(200):
    # Train discriminator
    real_data = torch.randn(64, 1) * 5
    fake_data = generator(torch.randn(64, 100))
    
    d_real = discriminator(real_data)
    d_fake = discriminator(fake_data.detach())
    
    d_loss = criterion(d_real, torch.ones_like(d_real)) + criterion(d_fake, torch.zeros_like(d_fake))
    d_optim.zero_grad()
    d_loss.backward()
    d_optim.step()
    
    # Train generator
    fake_data = generator(torch.randn(64, 100))
    g_output = discriminator(fake_data)
    g_loss = criterion(g_output, torch.ones_like(g_output))
    g_optim.zero_grad()
    g_loss.backward()
    g_optim.step()

Diffusion Models

Progressively add/remove noise through Markov chains. Key components include U-Net architecture and denoising process.

Natural Language Processing

Transformer Architecture

Utilizes self-attention mechanism for parallel sequence processing. Key components: multi-head attention, positional encoding, and feed-forward networks.

BERT Fine-tuning

Pre-trained transformer model with masked language modeling and next sentence prediction tasks.

from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments
from datasets import load_dataset

dataset = load_dataset("sst2")
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)

def tokenize_text(examples):
    return tokenizer(examples['sentence'], padding=True, truncation=True)

tokenized = dataset.map(tokenize_text, batched=True)

training_args = TrainingArguments(
    output_dir="./bert_finetuned",
    num_train_epochs=3,
    per_device_train_batch_size=8
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized["train"],
    eval_dataset=tokenized["validation"]
)

trainer.train()

Computer Vision

YOLO Object Detection

Real-time detection system that divides images into grids and predicts bounding boxes and class probabilities.

Mask R-CNN

Extends Faster R-CNN with instance segmentation capabilities. Includes ROI Align and mask prediction branches.