ai-art

  1import torch
  2import torch.nn as nn
  3
  4import numpy as np
  5
  6from einops import rearrange
  7
  8
  9class VectorQuantizer(nn.Module):
 10    """
 11    Improved version over VectorQuantizer, can be used as a drop-in replacement. Mostly
 12    avoids costly matrix multiplications and allows for post-hoc remapping of indices.
 13    """
 14    # NOTE: due to a bug the beta term was applied to the wrong term. for
 15    # backwards compatibility we use the buggy version by default, but you can
 16    # specify legacy=False to fix it.
 17    def __init__(self, n_e, e_dim, beta, remap=None, unknown_index="random",
 18                 sane_index_shape=False, legacy=True):
 19        super().__init__()
 20        self.n_e = n_e
 21        self.e_dim = e_dim
 22        self.beta = beta
 23        self.legacy = legacy
 24
 25        self.embedding = nn.Embedding(self.n_e, self.e_dim)
 26        self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e)
 27
 28        self.remap = remap
 29        if self.remap is not None:
 30            self.register_buffer("used", torch.tensor(np.load(self.remap)))
 31            self.re_embed = self.used.shape[0]
 32            self.unknown_index = unknown_index  # "random" or "extra" or integer
 33            if self.unknown_index == "extra":
 34                self.unknown_index = self.re_embed
 35                self.re_embed = self.re_embed + 1
 36            print(f"Remapping {self.n_e} indices to {self.re_embed} indices. "
 37                  f"Using {self.unknown_index} for unknown indices.")
 38        else:
 39            self.re_embed = n_e
 40
 41        self.sane_index_shape = sane_index_shape
 42
 43    def remap_to_used(self, inds):
 44        ishape = inds.shape
 45        assert len(ishape) > 1
 46        inds = inds.reshape(ishape[0], -1)
 47        used = self.used.to(inds)
 48        match = (inds[:, :, None] == used[None, None, ...]).long()
 49        new = match.argmax(-1)
 50        unknown = match.sum(2) < 1
 51        if self.unknown_index == "random":
 52            new[unknown] = \
 53                torch.randint(0, self.re_embed, size=new[unknown].shape).to(device=new.device)
 54        else:
 55            new[unknown] = self.unknown_index
 56        return new.reshape(ishape)
 57
 58    def unmap_to_all(self, inds):
 59        ishape = inds.shape
 60        assert len(ishape) > 1
 61        inds = inds.reshape(ishape[0], -1)
 62        used = self.used.to(inds)
 63        if self.re_embed > self.used.shape[0]:      # extra token
 64            inds[inds >= self.used.shape[0]] = 0    # simply set to zero
 65        back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds)
 66        return back.reshape(ishape)
 67
 68    def forward(self, z, temp=None, rescale_logits=False, return_logits=False):
 69        assert temp is None or temp == 1.0, "Only for interface compatible with Gumbel"
 70        assert rescale_logits is False, "Only for interface compatible with Gumbel"
 71        assert return_logits is False, "Only for interface compatible with Gumbel"
 72
 73        # reshape z -> (batch, height, width, channel) and flatten
 74        z = rearrange(z, 'b c h w -> b h w c').contiguous()
 75        z_flattened = z.view(-1, self.e_dim)
 76        # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
 77
 78        d = torch.sum(z_flattened ** 2, dim=1, keepdim=True) + \
 79            torch.sum(self.embedding.weight**2, dim=1) - 2 * \
 80            torch.einsum('bd,dn->bn', z_flattened, rearrange(self.embedding.weight, 'n d -> d n'))
 81
 82        min_encoding_indices = torch.argmin(d, dim=1)
 83        z_q = self.embedding(min_encoding_indices).view(z.shape)
 84        perplexity = None
 85        min_encodings = None
 86
 87        # compute loss for embedding
 88        if not self.legacy:
 89            loss = self.beta * torch.mean((z_q.detach() - z)**2) + \
 90                torch.mean((z_q - z.detach()) ** 2)
 91        else:
 92            loss = torch.mean((z_q.detach() - z)**2) + self.beta * \
 93                torch.mean((z_q - z.detach()) ** 2)
 94
 95        # preserve gradients
 96        z_q = z + (z_q - z).detach()
 97
 98        # reshape back to match original input shape
 99        z_q = rearrange(z_q, 'b h w c -> b c h w').contiguous()
100
101        if self.remap is not None:
102            min_encoding_indices = min_encoding_indices.reshape(z.shape[0], -1)  # add batch axis
103            min_encoding_indices = self.remap_to_used(min_encoding_indices)
104            min_encoding_indices = min_encoding_indices.reshape(-1, 1)  # flatten
105
106        if self.sane_index_shape:
107            min_encoding_indices = min_encoding_indices.reshape(
108                z_q.shape[0], z_q.shape[2], z_q.shape[3])
109
110        return z_q, loss, (perplexity, min_encodings, min_encoding_indices)
111
112    def get_codebook_entry(self, indices, shape):
113        # shape specifying (batch, height, width, channel)
114        if self.remap is not None:
115            indices = indices.reshape(shape[0], -1)  # add batch axis
116            indices = self.unmap_to_all(indices)
117            indices = indices.reshape(-1)  # flatten again
118
119        # get quantized latent vectors
120        z_q = self.embedding(indices)
121
122        if shape is not None:
123            z_q = z_q.view(shape)
124            # reshape back to match original input shape
125            z_q = z_q.permute(0, 3, 1, 2).contiguous()
126
127        return z_q