PyTorch Foundation

From

• Pytorch Basic Tutorial

Pytorch Basic

Dataset and Dataloader

torch.utils.data.Datasetstores the sample and labels. torch.utils.data.Datasetwraps an iterable around the dataset.

Example datasets include Image Datasets, Text Datasets, and Audio Datasets

Custom Dataset class must have:

• __init__: run once when instantiating the object. The example below specifies the label, directory, and transformation of feature data and label data.

• __len__: returns the number of samples in the dataset

• __getitem__: loads and returns a sample from the dataset at the given index idx with necessary transformation

# Source: PyTorch Official Tutorial 
# https://pytorch.org/tutorials/beginner/basics/data_tutorial.html#iterating-and-visualizing-the-dataset
import os
import pandas as pd
from torchvision.io import read_image

# Custom Dataset Construction
class CustomImageDataset(Dataset):
    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):
        self.img_labels = pd.read_csv(annotations_file)
        self.img_dir = img_dir
        self.transform = transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.img_labels)

    def __getitem__(self, idx):
        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])
        image = read_image(img_path)i
        label = self.img_labels.iloc[idx, 1]
        if self.transform:
            image = self.transform(image)
        if self.target_transform:
            label = self.target_transform(label)
        return image, label
        
   
# Iterate Dataset through dataloader      
from torch.utils.data import DataLoader
train_dataloader = DataLoader(training_data, batch_size=64, shuffle=True)
test_dataloader = DataLoader(test_data, batch_size=64, shuffle=True)

train_features, train_labels = next(iter(train_dataloader))
# train_features first dimension is batch size 
img = train_features[0].squeeze()
label = train_labels[0]
plt.imshow(img, cmap="gray")
plt.show()


# Lambda function example of create one-hot encoding label 
ds = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor(),
    target_transform=Lambda(lambda y: torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))
)

Building Neural Network

class NeuralNetwork(nn.Module):
    def __init__(self):
        super().__init__()
        self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(28*28, 512),
            nn.ReLU(),
            nn.Linear(512, 512),
            nn.ReLU(),
            nn.Linear(512, 10),
        )

    def forward(self, x):
        x = self.flatten(x)
        logits = self.linear_relu_stack(x)
        return logits
        
print(f"Model structure: {model}\n\n")

model = NeuralNetwork().to(device)
print(model)

for name, param in model.named_parameters():
    print(f"Layer: {name} | Size: {param.size()} | Values : {param[:2]} \n")

device = "cuda" if torch.cuda.is_available() else "cpu"

X = torch.rand(1, 28, 28, device=device)
logits = model(X)
pred_probab = nn.Softmax(dim=1)(logits)
y_pred = pred_probab.argmax(1)
print(f"Predicted class: {y_pred}")

# see parameters
print(f"Model structure: {model}\n\n")

for name, param in model.named_parameters():
    print(f"Layer: {name} | Size: {param.size()} | Values : {param[:2]} \n")

• inherit parent class

• forward specify how data is passed through all the layers.

Differentiation through autograd

import torch

x = torch.ones(5)  # input tensor
y = torch.zeros(3)  # expected output
w = torch.randn(5, 3, requires_grad=True)
b = torch.randn(3, requires_grad=True)

# forward computation 
z = torch.matmul(x, w)+b
loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)

# backward computation 
loss.backward()
print(w.grad)
print(b.grad)


# Disable gradient tracking 
with torch.no_grad():
    z = torch.matmul(x, w)+b
print(z.requires_grad)

# Alternative method for gradient tracking 
z = torch.matmul(x, w)+b
z_det = z.detach()

• Gradient is only available if requires_grad property is set to true.

• Gradient calculation is only done once using backward

• If several backward call is needed, we need to pass retain_graph = True to the backward call

• torch.no_grad() disable gradient tracking or detach, we can use them to mark parameters in neural network as frozen parameters

Training Loop and Saving Model

# Initialize the loss function and optimizer
loss_fn = nn.CrossEntropyLoss()
# optimizer need to specify what are the parameters need to be optimized 
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

def train_loop(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    for batch, (X, y) in enumerate(dataloader):
        # Compute prediction and loss
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), (batch + 1) * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

• Optimizer input includes the parameters needing optimized

• At each bach, predict, calculate loss, clean optimizer, calculate gradient, and then step.

def test_loop(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    test_loss, correct = 0, 0

    with torch.no_grad():
        for X, y in dataloader:
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()

    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

• Iterate through each set of data to calculate loss without tracking gradient.

epochs = 10
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train_loop(train_dataloader, model, loss_fn, optimizer)
    test_loop(test_dataloader, model, loss_fn)
print("Done!")


# Saving entire model 
torch.save(model, 'model.pth')
model = torch.load('model.pth')

# Saving only parameters
model = models.vgg16(pretrained=True)
torch.save(model.state_dict(), 'model_weights.pth')
model = models.vgg16() # we do not specify pretrained=True, i.e. do not load default weights
model.load_state_dict(torch.load('model_weights.pth'))
model.eval() # call model.eval to set dropout and batch normalization layer