Deep Learning

import torch
import torch.nn as nn
import torch.nn.functional as F

# Basit bir neural network
class SimpleNet(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(SimpleNet, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, hidden_size)
        self.fc3 = nn.Linear(hidden_size, output_size)
        self.dropout = nn.Dropout(0.2)
        
    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = self.fc3(x)
        return x

# Model oluştur
model = SimpleNet(784, 256, 10)
print(model)

import torch
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np

# Aktivasyon fonksiyonları
x = torch.linspace(-5, 5, 100)

# ReLU (Rectified Linear Unit)
relu_output = F.relu(x)

# Sigmoid
sigmoid_output = torch.sigmoid(x)

# Tanh
tanh_output = torch.tanh(x)

# Leaky ReLU
leaky_relu_output = F.leaky_relu(x, negative_slope=0.01)

# GELU (Gaussian Error Linear Unit)
gelu_output = F.gelu(x)

# Swish/SiLU
swish_output = F.silu(x)

import torch
import torch.nn as nn

# Classification loss functions
criterion_ce = nn.CrossEntropyLoss()
criterion_bce = nn.BCELoss()
criterion_bce_logits = nn.BCEWithLogitsLoss()

# Regression loss functions
criterion_mse = nn.MSELoss()
criterion_mae = nn.L1Loss()
criterion_smooth_l1 = nn.SmoothL1Loss()
criterion_huber = nn.HuberLoss()

# Örnek kullanım
predictions = torch.randn(32, 10)  # batch_size=32, classes=10
targets = torch.randint(0, 10, (32,))     # true labels

loss = criterion_ce(predictions, targets)
print(f"CrossEntropy Loss: {loss.item():.4f}")

# Regression örneği
pred_continuous = torch.randn(32, 1)
target_continuous = torch.randn(32, 1)

mse_loss = criterion_mse(pred_continuous, target_continuous)
print(f"MSE Loss: {mse_loss.item():.4f}")

import torch
import torch.nn as nn
import torch.optim as optim

# Model, loss ve optimizer tanımla
model = nn.Sequential(
    nn.Linear(784, 256),
    nn.ReLU(),
    nn.Linear(256, 128),
    nn.ReLU(),
    nn.Linear(128, 10)
)

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# Training loop - Backpropagation adımları
def train_step(data, targets):
    # 1. Forward pass
    outputs = model(data)
    loss = criterion(outputs, targets)
    
    # 2. Gradyanları sıfırla
    optimizer.zero_grad()
    
    # 3. Backward pass (backpropagation)
    loss.backward()
    
    # 4. Ağırlıkları güncelle
    optimizer.step()
    
    return loss.item()

# Manual gradient computation example
x = torch.randn(32, 784, requires_grad=True)
y = torch.randint(0, 10, (32,))

loss_value = train_step(x, y)
print(f"Training loss: {loss_value:.4f}")

import torch
import torch.nn as nn
import torch.nn.functional as F

class CNN(nn.Module):
    def __init__(self, num_classes=10):
        super(CNN, self).__init__()
        
        # Convolutional layers
        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, padding=1)
        
        # Pooling layers
        self.pool = nn.MaxPool2d(2, 2)
        
        # Fully connected layers
        self.fc1 = nn.Linear(128 * 4 * 4, 512)
        self.fc2 = nn.Linear(512, num_classes)
        
        # Dropout for regularization
        self.dropout = nn.Dropout(0.5)
        
    def forward(self, x):
        # Conv block 1: 32x32 -> 16x16
        x = self.pool(F.relu(self.conv1(x)))
        
        # Conv block 2: 16x16 -> 8x8
        x = self.pool(F.relu(self.conv2(x)))
        
        # Conv block 3: 8x8 -> 4x4
        x = self.pool(F.relu(self.conv3(x)))
        
        # Flatten for fully connected layers
        x = x.view(-1, 128 * 4 * 4)
        
        # Fully connected layers
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        
        return x

# Model oluştur
model = CNN(num_classes=10)
print(f"Toplam parametre sayısı: {sum(p.numel() for p in model.parameters()):,}")

# ResNet Block Implementation
class ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels, stride=1):
        super(ResidualBlock, self).__init__()
        
        self.conv1 = nn.Conv2d(in_channels, out_channels, 
                              kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channels)
        
        self.conv2 = nn.Conv2d(out_channels, out_channels, 
                              kernel_size=3, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channels)
        
        # Skip connection
        self.shortcut = nn.Sequential()
        if stride != 1 or in_channels != out_channels:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, 
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(out_channels)
            )
    
    def forward(self, x):
        residual = x
        
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        
        out += self.shortcut(residual)  # Skip connection
        out = F.relu(out)
        
        return out

# Attention Mechanism for Vision
class SelfAttention(nn.Module):
    def __init__(self, in_channels):
        super(SelfAttention, self).__init__()
        self.in_channels = in_channels
        
        self.query = nn.Conv2d(in_channels, in_channels // 8, 1)
        self.key = nn.Conv2d(in_channels, in_channels // 8, 1)
        self.value = nn.Conv2d(in_channels, in_channels, 1)
        
        self.gamma = nn.Parameter(torch.zeros(1))
        
    def forward(self, x):
        batch_size, C, H, W = x.size()
        
        q = self.query(x).view(batch_size, -1, H * W).permute(0, 2, 1)
        k = self.key(x).view(batch_size, -1, H * W)
        v = self.value(x).view(batch_size, -1, H * W)
        
        attention = torch.bmm(q, k)
        attention = F.softmax(attention, dim=-1)
        
        out = torch.bmm(v, attention.permute(0, 2, 1))
        out = out.view(batch_size, C, H, W)
        
        return self.gamma * out + x

import torch
import torch.nn as nn

class SimpleRNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, num_layers=1):
        super(SimpleRNN, self).__init__()
        
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        
        # RNN layer
        self.rnn = nn.RNN(input_size, hidden_size, num_layers, batch_first=True)
        
        # Output layer
        self.fc = nn.Linear(hidden_size, output_size)
        
    def forward(self, x):
        # Initialize hidden state
        batch_size = x.size(0)
        h0 = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        
        # Forward pass through RNN
        out, hidden = self.rnn(x, h0)
        
        # Use last output for prediction
        out = self.fc(out[:, -1, :])
        
        return out

# LSTM Implementation
class LSTMModel(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, num_layers=2):
        super(LSTMModel, self).__init__()
        
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        
        # LSTM layers
        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, 
                             batch_first=True, dropout=0.2)
        
        # Output layers
        self.fc1 = nn.Linear(hidden_size, hidden_size // 2)
        self.fc2 = nn.Linear(hidden_size // 2, output_size)
        self.dropout = nn.Dropout(0.3)
        
    def forward(self, x):
        batch_size = x.size(0)
        
        # Initialize hidden and cell states
        h0 = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        c0 = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        
        # LSTM forward pass
        lstm_out, (hidden, cell) = self.lstm(x, (h0, c0))
        
        # Use last time step output
        out = lstm_out[:, -1, :]
        
        # Fully connected layers
        out = torch.relu(self.fc1(out))
        out = self.dropout(out)
        out = self.fc2(out)
        
        return out

# Model örnekleri
rnn_model = SimpleRNN(input_size=10, hidden_size=64, output_size=1)
lstm_model = LSTMModel(input_size=10, hidden_size=128, output_size=1)

# LSTM Cell Implementation
class LSTMCell(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(LSTMCell, self).__init__()
        
        self.input_size = input_size
        self.hidden_size = hidden_size
        
        # Combined linear transformation for all gates
        self.i2h = nn.Linear(input_size, 4 * hidden_size)
        self.h2h = nn.Linear(hidden_size, 4 * hidden_size)
        
    def forward(self, x, hidden):
        hx, cx = hidden
        
        # Linear transformations
        preact = self.i2h(x) + self.h2h(hx)
        
        # Split into 4 gates
        gates = preact.chunk(4, 1)
        
        # Gate computations
        forget_gate = torch.sigmoid(gates[0])  # What to forget
        input_gate = torch.sigmoid(gates[1])   # What to store
        cell_gate = torch.tanh(gates[2])        # New information
        output_gate = torch.sigmoid(gates[3])  # What to output
        
        # Cell state update
        new_c = forget_gate * cx + input_gate * cell_gate
        
        # Hidden state update
        new_h = output_gate * torch.tanh(new_c)
        
        return new_h, new_c

# GRU Cell Implementation
class GRUCell(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(GRUCell, self).__init__()
        
        self.input_size = input_size
        self.hidden_size = hidden_size
        
        # Gate transformations
        self.i2h = nn.Linear(input_size, 3 * hidden_size)
        self.h2h = nn.Linear(hidden_size, 3 * hidden_size)
        
    def forward(self, x, hidden):
        gi = self.i2h(x)
        gh = self.h2h(hidden)
        i_reset, i_update, i_new = gi.chunk(3, 1)
        h_reset, h_update, h_new = gh.chunk(3, 1)
        
        # Reset and update gates
        reset_gate = torch.sigmoid(i_reset + h_reset)
        update_gate = torch.sigmoid(i_update + h_update)
        
        # New gate
        new_gate = torch.tanh(i_new + reset_gate * h_new)
        
        # Output
        new_h = (1 - update_gate) * new_gate + update_gate * hidden
        
        return new_h

# Bidirectional LSTM example
class BiLSTM(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(BiLSTM, self).__init__()
        
        self.lstm = nn.LSTM(input_size, hidden_size, 
                             batch_first=True, bidirectional=True)
        self.fc = nn.Linear(hidden_size * 2, output_size)  # *2 for bidirectional
        
    def forward(self, x):
        lstm_out, _ = self.lstm(x)
        output = self.fc(lstm_out[:, -1, :])  # Last time step
        return output

import torch
import torch.nn as nn
import torch.nn.functional as F
import math

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, num_heads):
        super(MultiHeadAttention, self).__init__()
        
        assert d_model % num_heads == 0
        
        self.d_model = d_model
        self.num_heads = num_heads
        self.d_k = d_model // num_heads
        
        # Linear projections for Q, K, V
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)
        
    def scaled_dot_product_attention(self, Q, K, V, mask=None):
        # Attention scores
        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)
        
        if mask is not None:
            scores = scores.masked_fill(mask == 0, -1e9)
        
        # Apply softmax
        attention_weights = F.softmax(scores, dim=-1)
        
        # Apply attention to values
        output = torch.matmul(attention_weights, V)
        
        return output, attention_weights
    
    def forward(self, query, key, value, mask=None):
        batch_size = query.size(0)
        
        # Linear projections
        Q = self.W_q(query)
        K = self.W_k(key)
        V = self.W_v(value)
        
        # Reshape for multi-head attention
        Q = Q.view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
        K = K.view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
        V = V.view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
        
        # Apply attention
        attn_output, attn_weights = self.scaled_dot_product_attention(Q, K, V, mask)
        
        # Concatenate heads
        attn_output = attn_output.transpose(1, 2).contiguous().view(
            batch_size, -1, self.d_model
        )
        
        # Final linear projection
        output = self.W_o(attn_output)
        
        return output

class PositionalEncoding(nn.Module):
    def __init__(self, d_model, max_len=5000):
        super(PositionalEncoding, self).__init__()
        
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len).unsqueeze(1).float()
        
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * 
                          -(math.log(10000.0) / d_model))
        
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        
        self.register_buffer('pe', pe.unsqueeze(0))
        
    def forward(self, x):
        return x + self.pe[:, :x.size(1)]

class TransformerBlock(nn.Module):
    def __init__(self, d_model, num_heads, d_ff, dropout=0.1):
        super(TransformerBlock, self).__init__()
        
        self.attention = MultiHeadAttention(d_model, num_heads)
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        
        # Feed-forward network
        self.ffn = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.ReLU(),
            nn.Linear(d_ff, d_model)
        )
        
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, x, mask=None):
        # Self-attention with residual connection
        attn_output = self.attention(x, x, x, mask)
        x = self.norm1(x + self.dropout(attn_output))
        
        # Feed-forward with residual connection
        ffn_output = self.ffn(x)
        x = self.norm2(x + self.dropout(ffn_output))
        
        return x

# Simple Attention Implementation
class SimpleAttention(nn.Module):
    def __init__(self, hidden_size):
        super(SimpleAttention, self).__init__()
        self.hidden_size = hidden_size
        self.attention = nn.Linear(hidden_size, 1)
        
    def forward(self, encoder_outputs):
        # encoder_outputs: (batch_size, seq_len, hidden_size)
        
        # Calculate attention scores
        scores = self.attention(encoder_outputs)  # (batch_size, seq_len, 1)
        scores = scores.squeeze(2)           # (batch_size, seq_len)
        
        # Apply softmax to get attention weights
        attention_weights = F.softmax(scores, dim=1)  # (batch_size, seq_len)
        
        # Apply attention weights to encoder outputs
        attention_weights = attention_weights.unsqueeze(2)  # (batch_size, seq_len, 1)
        context = torch.sum(encoder_outputs * attention_weights, dim=1)  # (batch_size, hidden_size)
        
        return context, attention_weights.squeeze(2)

# Cross Attention (for Transformer decoder)
class CrossAttention(nn.Module):
    def __init__(self, d_model, num_heads):
        super(CrossAttention, self).__init__()
        self.num_heads = num_heads
        self.d_k = d_model // num_heads
        
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)
        
    def forward(self, query, key, value):
        # Query from decoder, Key and Value from encoder
        batch_size = query.size(0)
        
        Q = self.W_q(query).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
        K = self.W_k(key).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
        V = self.W_v(value).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
        
        # Attention computation
        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)
        attention = F.softmax(scores, dim=-1)
        context = torch.matmul(attention, V)
        
        # Concatenate and project
        context = context.transpose(1, 2).contiguous().view(batch_size, -1, -1)
        output = self.W_o(context)
        
        return output

# BERT-style Masked Language Model
class BERTModel(nn.Module):
    def __init__(self, vocab_size, d_model=768, num_heads=12, num_layers=12):
        super(BERTModel, self).__init__()
        
        self.d_model = d_model
        self.embedding = nn.Embedding(vocab_size, d_model)
        self.pos_encoding = PositionalEncoding(d_model)
        
        # Transformer encoder layers
        self.encoder_layers = nn.ModuleList([
            TransformerBlock(d_model, num_heads, d_model * 4)
            for _ in range(num_layers)
        ])
        
        # MLM head
        self.mlm_head = nn.Linear(d_model, vocab_size)
        
        # Classification head
        self.cls_head = nn.Linear(d_model, 2)  # for binary classification
        
    def forward(self, input_ids, attention_mask=None):
        # Token embeddings + positional encoding
        x = self.embedding(input_ids) * math.sqrt(self.d_model)
        x = self.pos_encoding(x)
        
        # Pass through encoder layers
        for layer in self.encoder_layers:
            x = layer(x, attention_mask)
        
        # MLM predictions
        mlm_logits = self.mlm_head(x)
        
        # Classification prediction (using [CLS] token)
        cls_logits = self.cls_head(x[:, 0, :])  # First token is [CLS]
        
        return mlm_logits, cls_logits

# GPT-style Autoregressive Language Model
class GPTModel(nn.Module):
    def __init__(self, vocab_size, d_model=768, num_heads=12, num_layers=12, max_len=1024):
        super(GPTModel, self).__init__()
        
        self.d_model = d_model
        self.max_len = max_len
        
        self.token_embedding = nn.Embedding(vocab_size, d_model)
        self.pos_embedding = nn.Embedding(max_len, d_model)
        
        # Transformer decoder layers (with causal masking)
        self.decoder_layers = nn.ModuleList([
            TransformerBlock(d_model, num_heads, d_model * 4)
            for _ in range(num_layers)
        ])
        
        self.layer_norm = nn.LayerNorm(d_model)
        self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
        
    def create_causal_mask(self, seq_len):
        # Lower triangular mask for causal attention
        mask = torch.tril(torch.ones(seq_len, seq_len))
        return mask.unsqueeze(0).unsqueeze(0)  # Add batch and head dims
        
    def forward(self, input_ids):
        batch_size, seq_len = input_ids.shape
        
        # Create position indices
        pos_ids = torch.arange(0, seq_len, dtype=torch.long).unsqueeze(0)
        
        # Token + position embeddings
        token_emb = self.token_embedding(input_ids)
        pos_emb = self.pos_embedding(pos_ids)
        x = token_emb + pos_emb
        
        # Causal mask
        causal_mask = self.create_causal_mask(seq_len)
        
        # Pass through decoder layers
        for layer in self.decoder_layers:
            x = layer(x, causal_mask)
        
        # Layer norm and language modeling head
        x = self.layer_norm(x)
        logits = self.lm_head(x)
        
        return logits

# Model examples
bert_model = BERTModel(vocab_size=30000)
gpt_model = GPTModel(vocab_size=30000)

import torch.optim as optim
from torch.optim.lr_scheduler import StepLR, CosineAnnealingLR, ReduceLROnPlateau

# Different optimizers
def get_optimizer(model, optimizer_name, lr=0.001):
    if optimizer_name == 'sgd':
        return optim.SGD(model.parameters(), lr=lr, momentum=0.9, weight_decay=1e-4)
    
    elif optimizer_name == 'adam':
        return optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.999), weight_decay=1e-4)
    
    elif optimizer_name == 'adamw':
        return optim.AdamW(model.parameters(), lr=lr, betas=(0.9, 0.999), weight_decay=0.01)
    
    elif optimizer_name == 'rmsprop':
        return optim.RMSprop(model.parameters(), lr=lr, alpha=0.99, momentum=0.9)
    
    elif optimizer_name == 'adagrad':
        return optim.Adagrad(model.parameters(), lr=lr, weight_decay=1e-4)

# Learning Rate Schedulers
def get_scheduler(optimizer, scheduler_name):
    if scheduler_name == 'step':
        return StepLR(optimizer, step_size=30, gamma=0.1)
    
    elif scheduler_name == 'cosine':
        return CosineAnnealingLR(optimizer, T_max=50, eta_min=1e-6)
    
    elif scheduler_name == 'plateau':
        return ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10)

# Advanced training loop with mixed precision
from torch.cuda.amp import GradScaler, autocast

def train_with_mixed_precision(model, train_loader, optimizer, criterion, device):
    model.train()
    scaler = GradScaler()
    
    total_loss = 0
    for batch_idx, (data, targets) in enumerate(train_loader):
        data, targets = data.to(device), targets.to(device)
        
        optimizer.zero_grad()
        
        # Forward pass with autocast
        with autocast():
            outputs = model(data)
            loss = criterion(outputs, targets)
        
        # Backward pass with gradient scaling
        scaler.scale(loss).backward()
        scaler.step(optimizer)
        scaler.update()
        
        total_loss += loss.item()
    
    return total_loss / len(train_loader)

# Gradient clipping for stability
def train_with_grad_clip(model, train_loader, optimizer, criterion, max_grad_norm=1.0):
    model.train()
    
    for data, targets in train_loader:
        optimizer.zero_grad()
        
        outputs = model(data)
        loss = criterion(outputs, targets)
        loss.backward()
        
        # Clip gradients
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
        
        optimizer.step()

# Example usage
model = SimpleNet(784, 256, 10)
optimizer = get_optimizer(model, 'adamw', lr=0.001)
scheduler = get_scheduler(optimizer, 'cosine')

# Dropout Implementation
class DropoutNet(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, dropout_rate=0.5):
        super(DropoutNet, self).__init__()
        
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, hidden_size)
        self.fc3 = nn.Linear(hidden_size, output_size)
        
        self.dropout = nn.Dropout(dropout_rate)
        
    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.dropout(x)  # Apply dropout after activation
        
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        
        x = self.fc3(x)  # No dropout on output layer
        return x

# L1 and L2 Regularization
def l1_regularization(model, lambda_l1):
    l1_penalty = 0
    for param in model.parameters():
        l1_penalty += torch.sum(torch.abs(param))
    return lambda_l1 * l1_penalty

def l2_regularization(model, lambda_l2):
    l2_penalty = 0
    for param in model.parameters():
        l2_penalty += torch.sum(param ** 2)
    return lambda_l2 * l2_penalty

# Training with regularization
def train_with_regularization(model, train_loader, optimizer, criterion, 
                               lambda_l1=0.01, lambda_l2=0.01):
    model.train()
    total_loss = 0
    
    for data, targets in train_loader:
        optimizer.zero_grad()
        
        # Forward pass
        outputs = model(data)
        base_loss = criterion(outputs, targets)
        
        # Add regularization
        l1_loss = l1_regularization(model, lambda_l1)
        l2_loss = l2_regularization(model, lambda_l2)
        
        total_loss_batch = base_loss + l1_loss + l2_loss
        
        # Backward pass
        total_loss_batch.backward()
        optimizer.step()
        
        total_loss += total_loss_batch.item()
    
    return total_loss / len(train_loader)

# Early Stopping Implementation
class EarlyStopping:
    def __init__(self, patience=7, min_delta=0, restore_best_weights=True):
        self.patience = patience
        self.min_delta = min_delta
        self.restore_best_weights = restore_best_weights
        self.best_loss = None
        self.counter = 0
        self.best_weights = None
        
    def __call__(self, val_loss, model):
        if self.best_loss is None:
            self.best_loss = val_loss
            self.save_checkpoint(model)
        elif val_loss < self.best_loss - self.min_delta:
            self.best_loss = val_loss
            self.counter = 0
            self.save_checkpoint(model)
        else:
            self.counter += 1
            
        if self.counter >= self.patience:
            if self.restore_best_weights:
                self.restore_checkpoint(model)
            return True
        return False
    
    def save_checkpoint(self, model):
        self.best_weights = model.state_dict().copy()
    
    def restore_checkpoint(self, model):
        model.load_state_dict(self.best_weights)

# Data Augmentation for regularization
from torchvision import transforms

transform_train = transforms.Compose([
    transforms.RandomHorizontalFlip(p=0.5),
    transforms.RandomRotation(degrees=10),
    transforms.RandomCrop(32, padding=4),
    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

# Label Smoothing
class LabelSmoothingLoss(nn.Module):
    def __init__(self, num_classes, smoothing=0.1):
        super(LabelSmoothingLoss, self).__init__()
        self.num_classes = num_classes
        self.smoothing = smoothing
        
    def forward(self, pred, target):
        confidence = 1.0 - self.smoothing
        smooth_target = torch.full((pred.size(-1),), self.smoothing / (self.num_classes - 1))
        smooth_target.scatter_(0, target, confidence)
        
        return F.kl_div(F.log_softmax(pred, dim=-1), smooth_target, reduction='batchmean')

# Batch Normalization Implementation
class BatchNormNet(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(BatchNormNet, self).__init__()
        
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.bn1 = nn.BatchNorm1d(hidden_size)
        
        self.fc2 = nn.Linear(hidden_size, hidden_size)
        self.bn2 = nn.BatchNorm1d(hidden_size)
        
        self.fc3 = nn.Linear(hidden_size, output_size)
        
    def forward(self, x):
        # Layer → BatchNorm → Activation pattern
        x = self.fc1(x)
        x = self.bn1(x)
        x = F.relu(x)
        
        x = self.fc2(x)
        x = self.bn2(x)
        x = F.relu(x)
        
        x = self.fc3(x)
        return x

# CNN with Batch Normalization
class CNNWithBN(nn.Module):
    def __init__(self, num_classes=10):
        super(CNNWithBN, self).__init__()
        
        # Conv layers with BatchNorm
        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1)
        self.bn1 = nn.BatchNorm2d(64)
        
        self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)
        self.bn2 = nn.BatchNorm2d(128)
        
        self.conv3 = nn.Conv2d(128, 256, kernel_size=3, padding=1)
        self.bn3 = nn.BatchNorm2d(256)
        
        self.pool = nn.MaxPool2d(2, 2)
        self.global_pool = nn.AdaptiveAvgPool2d(1)
        
        self.fc = nn.Linear(256, num_classes)
        
    def forward(self, x):
        # Conv → BN → ReLU → Pool pattern
        x = self.pool(F.relu(self.bn1(self.conv1(x))))
        x = self.pool(F.relu(self.bn2(self.conv2(x))))
        x = self.pool(F.relu(self.bn3(self.conv3(x))))
        
        # Global average pooling
        x = self.global_pool(x)
        x = x.view(x.size(0), -1)
        
        x = self.fc(x)
        return x

# Layer Normalization (for sequences)
class LayerNormNet(nn.Module):
    def __init__(self, d_model):
        super(LayerNormNet, self).__init__()
        
        self.layer_norm = nn.LayerNorm(d_model)
        self.linear1 = nn.Linear(d_model, d_model * 4)
        self.linear2 = nn.Linear(d_model * 4, d_model)
        self.dropout = nn.Dropout(0.1)
        
    def forward(self, x):
        # Pre-norm architecture (common in Transformers)
        residual = x
        x = self.layer_norm(x)
        x = self.linear1(x)
        x = F.gelu(x)
        x = self.dropout(x)
        x = self.linear2(x)
        x = self.dropout(x)
        return residual + x

# Group Normalization
class GroupNormNet(nn.Module):
    def __init__(self, in_channels, out_channels, num_groups=8):
        super(GroupNormNet, self).__init__()
        
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1)
        self.group_norm = nn.GroupNorm(num_groups, out_channels)
        
    def forward(self, x):
        x = self.conv(x)
        x = self.group_norm(x)
        x = F.relu(x)
        return x

# Simple GAN Implementation
class Generator(nn.Module):
    def __init__(self, noise_dim=100, img_dim=784):
        super(Generator, self).__init__()
        
        self.model = nn.Sequential(
            nn.Linear(noise_dim, 256),
            nn.LeakyReLU(0.2),
            nn.Linear(256, 512),
            nn.LeakyReLU(0.2),
            nn.Linear(512, 1024),
            nn.LeakyReLU(0.2),
            nn.Linear(1024, img_dim),
            nn.Tanh()  # Output in [-1, 1]
        )
        
    def forward(self, noise):
        return self.model(noise)

class Discriminator(nn.Module):
    def __init__(self, img_dim=784):
        super(Discriminator, self).__init__()
        
        self.model = nn.Sequential(
            nn.Linear(img_dim, 512),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.3),
            nn.Linear(512, 256),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.3),
            nn.Linear(256, 1),
            nn.Sigmoid()  # Probability output
        )
        
    def forward(self, img):
        return self.model(img)

# GAN Training Loop
def train_gan(generator, discriminator, dataloader, num_epochs=100):
    criterion = nn.BCELoss()
    optimizer_g = optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999))
    optimizer_d = optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))
    
    for epoch in range(num_epochs):
        for batch_idx, (real_imgs, _) in enumerate(dataloader):
            batch_size = real_imgs.size(0)
            real_imgs = real_imgs.view(batch_size, -1)
            
            # Labels
            real_labels = torch.ones(batch_size, 1)
            fake_labels = torch.zeros(batch_size, 1)
            
            # Train Discriminator
            optimizer_d.zero_grad()
            
            # Real images
            real_outputs = discriminator(real_imgs)
            real_loss = criterion(real_outputs, real_labels)
            
            # Fake images
            noise = torch.randn(batch_size, 100)
            fake_imgs = generator(noise)
            fake_outputs = discriminator(fake_imgs.detach())
            fake_loss = criterion(fake_outputs, fake_labels)
            
            d_loss = real_loss + fake_loss
            d_loss.backward()
            optimizer_d.step()
            
            # Train Generator
            optimizer_g.zero_grad()
            
            fake_outputs = discriminator(fake_imgs)
            g_loss = criterion(fake_outputs, real_labels)  # Fool discriminator
            g_loss.backward()
            optimizer_g.step()
            
        print(f"Epoch [{epoch}/{num_epochs}], D_loss: {d_loss.item():.4f}, G_loss: {g_loss.item():.4f}")

# DCGAN for better image generation
class DCGANGenerator(nn.Module):
    def __init__(self, noise_dim=100, num_channels=3):
        super(DCGANGenerator, self).__init__()
        
        self.main = nn.Sequential(
            # Input: noise_dim
            nn.ConvTranspose2d(noise_dim, 512, 4, 1, 0, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(True),
            
            # 4x4 -> 8x8
            nn.ConvTranspose2d(512, 256, 4, 2, 1, bias=False),
            nn.BatchNorm2d(256),
            nn.ReLU(True),
            
            # 8x8 -> 16x16
            nn.ConvTranspose2d(256, 128, 4, 2, 1, bias=False),
            nn.BatchNorm2d(128),
            nn.ReLU(True),
            
            # 16x16 -> 32x32
            nn.ConvTranspose2d(128, 64, 4, 2, 1, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            
            # 32x32 -> 64x64
            nn.ConvTranspose2d(64, num_channels, 4, 2, 1, bias=False),
            nn.Tanh()
        )
        
    def forward(self, noise):
        return self.main(noise)