neural-network-design

name	Neural Network Design
description	Design and architect neural networks with various architectures including CNNs, RNNs, Transformers, and attention mechanisms using PyTorch and TensorFlow

Neural Network Design

Overview

This skill covers designing and implementing neural network architectures including CNNs, RNNs, Transformers, and ResNets using PyTorch and TensorFlow, with focus on architecture selection, layer composition, and optimization techniques.

When to Use

• Designing custom neural network architectures for computer vision tasks like image classification or object detection
• Building sequence models for time series forecasting, natural language processing, or video analysis
• Implementing transformer-based models for language understanding or generation tasks
• Creating hybrid architectures that combine CNNs, RNNs, and attention mechanisms
• Optimizing network depth, width, and skip connections for better training and performance
• Selecting appropriate activation functions, normalization layers, and regularization techniques

Core Architecture Types

• Feedforward Networks (MLPs): Fully connected layers
• Convolutional Networks (CNNs): Image processing
• Recurrent Networks (RNNs, LSTMs, GRUs): Sequence processing
• Transformers: Self-attention based architecture
• Hybrid Models: Combining multiple architecture types

Network Design Principles

• Depth vs Width: Trade-offs between layers and units
• Skip Connections: Residual networks for deeper training
• Normalization: Batch norm, layer norm for stability
• Regularization: Dropout, L1/L2 preventing overfitting
• Activation Functions: ReLU, GELU, Swish for non-linearity

PyTorch and TensorFlow Implementation

import torch
import torch.nn as nn
import tensorflow as tf
from tensorflow import keras
import numpy as np
import matplotlib.pyplot as plt

# 1. Feedforward Neural Network (MLP)
print("=== 1. Feedforward Neural Network ===")

class MLPPyTorch(nn.Module):
    def __init__(self, input_size, hidden_sizes, output_size):
        super().__init__()
        layers = []
        prev_size = input_size

        for hidden_size in hidden_sizes:
            layers.append(nn.Linear(prev_size, hidden_size))
            layers.append(nn.BatchNorm1d(hidden_size))
            layers.append(nn.ReLU())
            layers.append(nn.Dropout(0.3))
            prev_size = hidden_size

        layers.append(nn.Linear(prev_size, output_size))
        self.model = nn.Sequential(*layers)

    def forward(self, x):
        return self.model(x)

mlp = MLPPyTorch(input_size=784, hidden_sizes=[512, 256, 128], output_size=10)
print(f"MLP Parameters: {sum(p.numel() for p in mlp.parameters()):,}")

# 2. Convolutional Neural Network (CNN)
print("\n=== 2. Convolutional Neural Network ===")

 (nn.Module):
     ():
        ().__init__()
        
        .conv1 = nn.Conv2d(, , kernel_size=, padding=)
        .bn1 = nn.BatchNorm2d()
        .pool1 = nn.MaxPool2d(, )

        .conv2 = nn.Conv2d(, , kernel_size=, padding=)
        .bn2 = nn.BatchNorm2d()
        .pool2 = nn.MaxPool2d(, )

        .conv3 = nn.Conv2d(, , kernel_size=, padding=)
        .bn3 = nn.BatchNorm2d()
        .pool3 = nn.MaxPool2d(, )

        
        .fc1 = nn.Linear( *  * , )
        .dropout = nn.Dropout()
        .fc2 = nn.Linear(, )
        .relu = nn.ReLU()

     ():
        x = .relu(.bn1(.conv1(x)))
        x = .pool1(x)
        x = .relu(.bn2(.conv2(x)))
        x = .pool2(x)
        x = .relu(.bn3(.conv3(x)))
        x = .pool3(x)
        x = x.view(x.size(), -)
        x = .relu(.fc1(x))
        x = .dropout(x)
        x = .fc2(x)
         x

cnn = CNNPyTorch()
()


()

 (nn.Module):
     ():
        ().__init__()
        .lstm = nn.LSTM(input_size, hidden_size, num_layers,
                           batch_first=, dropout=)
        .fc = nn.Linear(hidden_size, output_size)

     ():
        lstm_out, (h_n, c_n) = .lstm(x)
        last_hidden = h_n[-]
        output = .fc(last_hidden)
         output

lstm = LSTMPyTorch(input_size=, hidden_size=, num_layers=, output_size=)
()


()

 (nn.Module):
     ():
        ().__init__()
        .attention = nn.MultiheadAttention(d_model, num_heads, dropout=dropout)
        .norm1 = nn.LayerNorm(d_model)
        .norm2 = nn.LayerNorm(d_model)

        .feedforward = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(d_ff, d_model),
            nn.Dropout(dropout)
        )

     ():
        
        attn_out, _ = .attention(x, x, x)
        x = .norm1(x + attn_out)

        
        ff_out = .feedforward(x)
        x = .norm2(x + ff_out)
         x

 (nn.Module):
     ():
        ().__init__()
        .embedding = nn.Embedding(vocab_size, d_model)
        .transformer_blocks = nn.ModuleList([
            TransformerBlock(d_model, num_heads, d_ff)
             _  (num_layers)
        ])
        .fc = nn.Linear(d_model, )

     ():
        x = .embedding(x)
         block  .transformer_blocks:
            x = block(x)
        x = x.mean(dim=)  
        x = .fc(x)
         x

transformer = TransformerPyTorch(vocab_size=, d_model=, num_heads=,
                                 num_layers=, d_ff=)
()


()

 (nn.Module):
     ():
        ().__init__()
        .conv1 = nn.Conv2d(in_channels, out_channels, , stride=stride, padding=)
        .bn1 = nn.BatchNorm2d(out_channels)
        .conv2 = nn.Conv2d(out_channels, out_channels, , padding=)
        .bn2 = nn.BatchNorm2d(out_channels)
        .relu = nn.ReLU()

        .shortcut = nn.Sequential()
         stride !=   in_channels != out_channels:
            .shortcut = nn.Sequential(
                nn.Conv2d(in_channels, out_channels, , stride=stride),
                nn.BatchNorm2d(out_channels)
            )

     ():
        residual = .shortcut(x)
        out = .relu(.bn1(.conv1(x)))
        out = .bn2(.conv2(out))
        out += residual
        out = .relu(out)
         out

 (nn.Module):
     ():
        ().__init__()
        .conv1 = nn.Conv2d(, , , stride=, padding=)
        .bn1 = nn.BatchNorm2d()
        .maxpool = nn.MaxPool2d(, stride=, padding=)

        .layer1 = ._make_layer(, , , stride=)
        .layer2 = ._make_layer(, , , stride=)
        .layer3 = ._make_layer(, , , stride=)
        .layer4 = ._make_layer(, , , stride=)

        .avgpool = nn.AdaptiveAvgPool2d((, ))
        .fc = nn.Linear(, )

     ():
        layers = [ResidualBlock(in_channels, out_channels, stride)]
         _  (, blocks):
            layers.append(ResidualBlock(out_channels, out_channels))
         nn.Sequential(*layers)

     ():
        x = .maxpool(.bn1(.conv1(x)))
        x = .layer1(x)
        x = .layer2(x)
        x = .layer3(x)
        x = .layer4(x)
        x = .avgpool(x)
        x = x.view(x.size(), -)
        x = .fc(x)
         x

resnet = ResNetPyTorch()
()


()

tf_model = keras.Sequential([
    keras.layers.Conv2D(, (, ), activation=, input_shape=(, , )),
    keras.layers.BatchNormalization(),
    keras.layers.MaxPooling2D((, )),

    keras.layers.Conv2D(, (, ), activation=),
    keras.layers.BatchNormalization(),
    keras.layers.MaxPooling2D((, )),

    keras.layers.Conv2D(, (, ), activation=),
    keras.layers.BatchNormalization(),
    keras.layers.GlobalAveragePooling2D(),

    keras.layers.Dense(, activation=),
    keras.layers.Dropout(),
    keras.layers.Dense(, activation=)
])

()
tf_model.summary()


models_info = {
    : mlp,
    : cnn,
    : lstm,
    : transformer,
    : resnet,
}

param_counts = {name: (p.numel()  p  model.parameters())
                 name, model  models_info.items()}

fig, axes = plt.subplots(, , figsize=(, ))


axes[].barh((param_counts.keys()), (param_counts.values()), color=)
axes[].set_xlabel()
axes[].set_title()
axes[].set_xscale()


architectures = {
    : ,
    : ,
    : ,
    : ,
    : 
}

y_pos = np.arange((architectures))
axes[].axis()
table_data = [[name, architectures[name]]  name  architectures.keys()]
table = axes[].table(cellText=table_data, colLabels=[, ],
                      cellLoc=, loc=, bbox=[, , , ])
table.auto_set_font_size()
table.set_fontsize()
table.scale(, )

plt.tight_layout()
plt.savefig(, dpi=, bbox_inches=)
()

()

Architecture Selection Guide

• MLP: Tabular data, simple classification
• CNN: Image classification, object detection
• LSTM/GRU: Time series, sequential data
• Transformer: NLP, long-range dependencies
• ResNet: Very deep networks, image tasks

Key Design Considerations

• Input/output shape compatibility
• Receptive field size for CNNs
• Sequence length for RNNs
• Attention head count for Transformers
• Skip connection placement for ResNets

Deliverables

• Network architecture definition
• Parameter count analysis
• Layer-by-layer description
• Data flow diagrams
• Performance benchmarks
• Deployment requirements