GANs

Implement sample_noise

def sample_noise(batch_size, dim, seed=None):
    """
    Generate a PyTorch Tensor of uniform random noise.
 
    Input:
    - batch_size: Integer giving the batch size of noise to generate.
    - dim: Integer giving the dimension of noise to generate.
 
    Output:
    - A PyTorch Tensor of shape (batch_size, dim) containing uniform
      random noise in the range (-1, 1).
    """
    if seed is not None:
        torch.manual_seed(seed)
 
	# CODE GOES HERE

---

Answer

def sample_noise(batch_size, dim, seed=None):
    if seed is not None:
        torch.manual_seed(seed)
    return -2 * torch.rand(batch_size, dim) + 1
 

Look at Unflatten

class Unflatten(nn.Module):
    """
    An Unflatten module receives an input of shape (N, C*H*W) and reshapes it
    to produce an output of shape (N, C, H, W).
    """
    def __init__(self, N=-1, C=128, H=7, W=7):
        super(Unflatten, self).__init__()
        self.N = N
        self.C = C
        self.H = H
        self.W = W
    def forward(self, x):
        return x.view(self.N, self.C, self.H, self.W)

Implement Discriminator

The Architecture

• Fully connected layer with input size 784 and output size 256
• LeakyReLU with alpha 0.01
• Fully connected layer with input_size 256 and output size 256
• LeakyReLU with alpha 0.01
• Fully connected layer with input size 256 and output size 1

def discriminator(seed=None):
    """
    Build and return a PyTorch model implementing the architecture above.
    """
 
    if seed is not None:
        torch.manual_seed(seed)
 
    model = None
 
	# CODE GOES HERE

---

Answer

def discriminator(seed=None):
    """
    Build and return a PyTorch model implementing the architecture above.
    """
 
    if seed is not None:
        torch.manual_seed(seed)
 
    model = nn.Sequential(
        nn.Flatten(),
        nn.Linear(784, 256),
        nn.LeakyReLU(0.01),
        nn.Linear(256, 256),
        nn.LeakyReLU(0.01),
        nn.Linear(256, 1)
    )
 
    return model

Implement Generator

The Architecture

• Fully connected layer from noise_dim to 1024
• ReLU
• Fully connected layer with size 1024
• ReLU
• Fully connected layer with size 784
• TanH (to clip the image to be in the range of [-1,1])

def generator(noise_dim=NOISE_DIM, seed=None):
    """
    Build and return a PyTorch model implementing the architecture above.
    """
 
    if seed is not None:
        torch.manual_seed(seed)
 
    model = None
 
	# CODE GOES HERE

---

Answer

def generator(noise_dim=NOISE_DIM, seed=None):
    """
    Build and return a PyTorch model implementing the architecture above.
    """
 
    if seed is not None:
        torch.manual_seed(seed)
 
 
 
    model = nn.Sequential(
        nn.Linear(noise_dim, 1024),
        nn.ReLU(),
        nn.Linear(1024, 1024),
        nn.ReLU(),
        nn.Linear(1024, 784),
        nn.Tanh()
    )
 
 
    return model

Understand BCE Loss

def bce_loss(input, target):
    """
    Numerically stable version of the binary cross-entropy loss function.
 
    As per https://github.com/pytorch/pytorch/issues/751
    See the TensorFlow docs for a derivation of this formula:
    https://www.tensorflow.org/api_docs/python/tf/nn/sigmoid_cross_entropy_with_logits
 
    Inputs:
    - input: PyTorch Tensor of shape (N, ) giving scores.
    - target: PyTorch Tensor of shape (N,) containing 0 and 1 giving targets.
 
    Returns:
    - A PyTorch Tensor containing the mean BCE loss over the minibatch of input data.
    """
    neg_abs = - input.abs()
    loss = input.clamp(min=0) - input * target + (1 + neg_abs.exp()).log()
    return loss.mean()

Implement Generator Loss

You can use BCE Loss

def discriminator_loss(logits_real, logits_fake):
    """
    Computes the discriminator loss described above.
 
    Inputs:
    - logits_real: PyTorch Tensor of shape (N,) giving scores for the real data.
    - logits_fake: PyTorch Tensor of shape (N,) giving scores for the fake data.
 
    Returns:
    - loss: PyTorch Tensor containing (scalar) the loss for the discriminator.
    """
    loss = None
 
	# CODE GOES HERE

---

Answer

def discriminator_loss(logits_real, logits_fake):
 
    loss_fake = bce_loss(logits_fake, torch.zeros_like(logits_fake))
    loss_real = bce_loss(logits_real, torch.ones_like(logits_real))
    loss = loss_fake + loss_real
    return loss

Implement Discriminator Loss

You can use BCE Loss

def generator_loss(logits_fake):
    """
    Computes the generator loss described above.
 
    Inputs:
    - logits_fake: PyTorch Tensor of shape (N,) giving scores for the fake data.
 
    Returns:
    - loss: PyTorch Tensor containing the (scalar) loss for the generator.
    """
    loss = None
 
	# CODE GOES HERE

---

Answer

def generator_loss(logits_fake):
 
    loss = bce_loss(logits_fake, torch.ones_like(logits_fake))
 
    # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
    return loss

Implement get_optimizer

def get_optimizer(model):
    """
    Construct and return an Adam optimizer for the model with learning rate 1e-3,
    beta1=0.5, and beta2=0.999.
 
    Input:
    - model: A PyTorch model that we want to optimize.
 
    Returns:
    - An Adam optimizer for the model with the desired hyperparameters.
    """
    optimizer = None
 
	# CODE GOES HERE

---

Answer

def get_optimizer(model):
 
    optimizer = optim.Adam(model.parameters(), lr=1e-3, betas=(0.5, 0.999))
 
    return optimizer

Understand Training Loop

def run_a_gan(D, G, D_solver, G_solver, discriminator_loss, generator_loss, loader_train, show_every=250,
              batch_size=128, noise_size=96, num_epochs=10):
    """
    Train a GAN!
 
    Inputs:
    - D, G: PyTorch models for the discriminator and generator
    - D_solver, G_solver: torch.optim Optimizers to use for training the
      discriminator and generator.
    - discriminator_loss, generator_loss: Functions to use for computing the generator and
      discriminator loss, respectively.
    - show_every: Show samples after every show_every iterations.
    - batch_size: Batch size to use for training.
    - noise_size: Dimension of the noise to use as input to the generator.
    - num_epochs: Number of epochs over the training dataset to use for training.
    """
    images = []
    iter_count = 0
    for epoch in range(num_epochs):
        for x, _ in loader_train:
            if len(x) != batch_size:
                continue
            D_solver.zero_grad()
            real_data = x.type(dtype)
            logits_real = D(2* (real_data - 0.5)).type(dtype)
 
            g_fake_seed = sample_noise(batch_size, noise_size).type(dtype)
            fake_images = G(g_fake_seed).detach()
            logits_fake = D(fake_images.view(batch_size, 1, 28, 28))
 
            d_total_error = discriminator_loss(logits_real, logits_fake)
            d_total_error.backward()
            D_solver.step()
 
            G_solver.zero_grad()
            g_fake_seed = sample_noise(batch_size, noise_size).type(dtype)
            fake_images = G(g_fake_seed)
 
            gen_logits_fake = D(fake_images.view(batch_size, 1, 28, 28))
            g_error = generator_loss(gen_logits_fake)
            g_error.backward()
            G_solver.step()
 
            if (iter_count % show_every == 0):
                print('Iter: {}, D: {:.4}, G:{:.4}'.format(iter_count,d_total_error.item(),g_error.item()))
                imgs_numpy = fake_images.data.cpu().numpy()
                images.append(imgs_numpy[0:16])
 
            iter_count += 1
 
    return images

Least Squares GAN

Least Squared GAN is a newer, more stable alternative to the original GAN loss function.

We’ll be implementing Equation 9 from the paper

Note: whe plugging in for $D(x)$ and $D(G(z))$ use the output from the discriminator (scores_real and scores_fake)

Implement ls_discriminator_loss

def ls_discriminator_loss(scores_real, scores_fake):
    """
    Compute the Least-Squares GAN loss for the discriminator.
 
    Inputs:
    - scores_real: PyTorch Tensor of shape (N,) giving scores for the real data.
    - scores_fake: PyTorch Tensor of shape (N,) giving scores for the fake data.
 
    Outputs:
    - loss: A PyTorch Tensor containing the loss.
    """
    loss = None
 
	# CODE GOES HERE

---

Answer

def ls_discriminator_loss(scores_real, scores_fake):
 
    loss = 0.5 * ((scores_real - 1).square() + scores_fake.square()).mean()
 
    return loss

Implement ls_generator_loss

def ls_generator_loss(scores_fake):
    """
    Computes the Least-Squares GAN loss for the generator.
 
    Inputs:
    - scores_fake: PyTorch Tensor of shape (N,) giving scores for the fake data.
 
    Outputs:
    - loss: A PyTorch Tensor containing the loss.
    """
    loss = None
 
	CODE GOES HERE

---

Answer

def ls_generator_loss(scores_fake):
 
    loss = 0.5 * (scores_fake - 1).square().mean()
 
    # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****
    return loss

Implement build_dc_classifier

Use a discrminator inspire by the TensorFlow MNIST tutorial, which is pretty dang efficient

• Reshape into image tensor (Use Unflatten!)
• Conv2D: 32 Filters, 5x5, Stride 1
• Leaky ReLU(alpha=0.01)
• Max Pool 2x2, Stride 2
• Conv2D: 64 Filters, 5x5, Stride 1
• Leaky ReLU(alpha=0.01)
• Max Pool 2x2, Stride 2
• Flatten
• Fully Connected with output size 4 x 4 x 64
• Leaky ReLU(alpha=0.01)
• Fully Connected with output size 1

def build_dc_classifier(batch_size):
    """
    Build and return a PyTorch model for the DCGAN discriminator implementing
    the architecture above.
    """
 

---

Answer

def build_dc_classifier(batch_size):
    """
    Build and return a PyTorch model for the DCGAN discriminator implementing
    the architecture above.
    """
 
    return nn.Sequential(
        Unflatten(batch_size, 1, 28, 28),
        nn.Conv2d(1, 32, 5, 1),
        nn.LeakyReLU(0.01),
        nn.MaxPool2d(2, 2),
        nn.Conv2d(32, 64, 5, 1),
        nn.LeakyReLU(0.01),
        nn.MaxPool2d(2, 2),
        Flatten(),
        nn.Linear(4*4*64, 4*4*64),
        nn.LeakyReLU(0.01),
        nn.Linear(4*4*64, 1)
    )

Similarly, implement Generator

Architecture

• Fully connected with output size 1024
• ReLU
• BatchNorm
• Fully connected with output size 7 x 7 x 128
• ReLU
• BatchNorm
• Reshape into Image Tensor of shape 7, 7, 128
• Conv2D^T (Transpose): 64 filters of 4x4, stride 2, ‘same’ padding (use padding=1)
• ReLU
• BatchNorm
• Conv2D^T (Transpose): 1 filter of 4x4, stride 2, ‘same’ padding (use padding=1)
• TanH
• Should have a 28x28x1 image, reshape back into 784 vector

def build_dc_generator(noise_dim=NOISE_DIM):
    """
    Build and return a PyTorch model implementing the DCGAN generator using
    the architecture described above.
    """
	# CODE GOES HERE

---

Answer

def build_dc_generator(noise_dim=NOISE_DIM):
    """
    Build and return a PyTorch model implementing the DCGAN generator using
    the architecture described above.
    """
 
    return nn.Sequential(
        nn.Linear(noise_dim, 1024),
        nn.ReLU(),
        nn.BatchNorm1d(1024),
        nn.Linear(1024, 7*7*128),
        nn.ReLU(),
        nn.BatchNorm1d(7*7*128),
        Unflatten(-1, 128, 7, 7),
        nn.ConvTranspose2d(128, 64, 4, 2, 1),
        nn.ReLU(),
        nn.BatchNorm2d(64),
        nn.ConvTranspose2d(64, 1, 4, 2, 1),
        nn.Tanh(),
        Flatten()
    )