push files

app.py

@@ -0,0 +1,255 @@
+import streamlit as st
+import numpy as np
+import random
+
+from gist1.vqvae_gpt import VQVAETransformer
+from utils.misc import  load_params
+from utils.isoutil import plot_isovist_sequence_grid
+
+
+import torch
+
+
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+else:
+    device = torch.device("cpu")
+
+
+model_paths = ["./models/vqvaegpt_1.pth",
+               "./models/vqvaegpt_2.pth",
+               "./models/vqvaegpt_3.pth"]
+cfg_path = "./models/param.json"
+cfg = load_params(cfg_path)
+
+
+
+@st.cache_resource
+def get_model(index):
+    TransformerPath = model_paths[index]
+    transformer = VQVAETransformer(cfg)
+    transformer.load_state_dict(torch.load(TransformerPath))
+    transformer = transformer.to(device)
+    transformer.eval()
+    return transformer
+
+
+def split_indices(indices, loc_len=1, isovist_len=16):
+    seg_length = loc_len + isovist_len
+    batch_size = indices.shape[0]
+    splits = indices.reshape(batch_size, -1, seg_length) # BS(L+I)
+    ilocs, iisovists = torch.split(splits, [loc_len, isovist_len], dim=2) # BSL , BSI
+    return ilocs, iisovists
+
+@st.cache_data
+def indices_to_loc(_model, indices):
+    indices = torch.tensor(indices).long().view(1,-1).to(device)
+    return _model.indices_to_loc(indices).detach().cpu().numpy()
+
+@st.cache_data
+def indices_to_isovist(_model, indices):
+    indices = torch.tensor(indices).long().view(1,-1).to(device)
+    return _model.z_to_isovist(indices).detach().cpu().numpy()
+
+def indices_to_loc_isovist(model, indices):
+    ilocs, iisovists = split_indices(indices, loc_len=1, isovist_len=16)
+    locs = []
+    sampled_isovists = []
+    for i in range(iisovists.shape[1]): 
+        # iloc = ilocs[:, i, :]
+        # locs.append(model.indices_to_loc(iloc).detach().cpu().numpy()) # S X BL
+        # iisovist = iisovists[:, i, :] # BI
+        # sampled_isovists.append(model.z_to_isovist(iisovist).detach().cpu().numpy()) # S X BCW
+
+        iloc = ilocs[:, i, :].squeeze().tolist()
+        iisovist = iisovists[:, i, :].squeeze().tolist()
+        iisovist = tuple(iisovist)
+        locs.append(indices_to_loc(model, iloc))
+        sampled_isovists.append(indices_to_isovist(model, iisovist))
+        # sampled_isovists.append(code_to_isovist(model, iisovist))
+
+    locs = np.stack(locs, axis=1)
+    sampled_isovists = np.stack(sampled_isovists, axis=1) #BSCW
+    return locs, sampled_isovists
+
+def plot_isovist(locs, sampled_isovists, lim, alpha, calculate_lim):
+    loc = locs[0]
+    sampled_isovist = sampled_isovists[0]
+    sampled_isovist = np.squeeze(sampled_isovist, axis=1)
+    fig = plot_isovist_sequence_grid(loc, sampled_isovist, figsize=(8, 6), center=True, lim=lim, alpha=alpha, calculate_lim=calculate_lim).transpose((1, 2, 0))
+    return fig
+
+def sample(model, start_indices, top_k=100, seed=0, seq_length=None, zeroing=False, lim=1.5, alpha=0.02, loc_init=False, calculate_lim=False):
+    start_indices = start_indices.long().to(device)
+    steps = seq_length * (1 + 16) # loc dim + latent 
+    if loc_init:
+        steps -= 1
+    sample_indices = model.sample_memorized(start_indices, steps=steps, top_k=top_k, seed=seed, zeroing=zeroing)
+    locs, sampled_isovists = indices_to_loc_isovist(model, sample_indices)
+    im = plot_isovist(locs, sampled_isovists, lim, alpha, calculate_lim)
+    return im, sample_indices
+
+
+def plot_indices(model, indices, lim=1.5, alpha=0.02, calculate_lim=False):
+    locs, sampled_isovists = indices_to_loc_isovist(model, indices)
+    im = plot_isovist(locs, sampled_isovists, lim, alpha, calculate_lim)
+    return im
+
+
+
+st.subheader("GIsT: Generative Isovist Transformers")
+st.text("Pres [init] to initiate or start over")
+
+options =["Base model", "Palladio", "Mies"]
+
+if 'model' not in st.session_state:
+    st.session_state.model = None
+
+if st.session_state.model is not None:
+    index = options.index(st.session_state.model)
+else:
+    index = 0
+
+option = st.selectbox("Select model",(options), index=index)
+st.session_state.model = option
+
+
+if 'tokens'  not in st.session_state:
+    st.session_state.tokens = None
+
+if 'image' not in st.session_state:
+    st.session_state.image = np.ones((600,800,3),dtype=np.uint8) * 240
+
+if 'seed' not in st.session_state:
+    st.session_state.seed = random.randint(0, 10000000)
+
+
+
+index = options.index(st.session_state.model)
+transformer = get_model(index)
+
+
+e = 1025
+ne = 1026
+n = 1027
+nw = 1028
+w = 1029
+sw = 1030
+s = 1031
+se = 1032
+
+alpha = 0.015
+lim = 2.0
+
+init = st.button('init')
+
+cont = st.container()
+
+
+
+
+rows = []
+for i in range(3):
+    rows.append(st.columns(3, gap='small'))
+
+
+
+
+upleft = rows[0][0].button('upleft', use_container_width=True)
+up = rows[0][1].button('up', use_container_width=True)
+upright = rows[0][2].button('upright', use_container_width=True)
+left = rows[1][0].button('left', use_container_width=True)
+undo = rows[1][1].button('undo', use_container_width=True)
+right = rows[1][2].button('right', use_container_width=True)
+downleft = rows[2][0].button('downleft', use_container_width=True)
+down = rows[2][1].button('down', use_container_width=True)
+downright = rows[2][2].button('downright', use_container_width=True)
+
+st.text("use desktop mode for best experiece in mobile device")
+
+seed = st.number_input('seed', 0, 10000000, st.session_state.seed,1)
+
+
+def gen_next(sample_indices, dir):
+    # seed = st.session_state.seed
+    sample_indices = torch.concat([sample_indices, torch.tensor([[dir]]).to(device)],dim=1)
+    im, sample_indices = sample(transformer, sample_indices, top_k=50, seq_length=1, seed=seed,  lim=lim, alpha=alpha, loc_init=True, calculate_lim=True)
+    return im, sample_indices
+
+def undo_gen(sample_indices):
+    sample_indices = sample_indices[:, :-17]
+    im = plot_indices(transformer, sample_indices, lim=lim,alpha=alpha, calculate_lim=True)
+    return im, sample_indices
+
+if init:
+    st.session_state.tokens = torch.ones((1, 1)).long().to(device) * 1024
+    tokens = st.session_state.tokens
+    # seed = st.session_state.seed
+    im, sample_indices = sample(transformer, tokens, top_k=50, seq_length=1, seed=seed,  lim=lim, alpha=alpha, loc_init=True)
+    st.session_state.image = im
+    st.session_state.tokens = sample_indices
+    st.session_state.lim = 2.0
+
+if upleft:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, nw)
+    else:
+        st.warning('Please init the generation')
+
+if up:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, n)
+    else:
+        st.warning('Please init the generation')
+
+if upright:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, ne)
+    else:
+        st.warning('Please init the generation')
+
+if left:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, w)
+    else:
+        st.warning('Please init the generation')
+
+if right:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, e)
+    else:
+        st.warning('Please init the generation')
+
+if downleft:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, sw)
+    else:
+        st.warning('Please init the generation')
+
+if down:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, s)
+    else:
+        st.warning('Please init the generation')
+
+if downright:
+    if st.session_state.tokens is not None:
+        st.session_state.image, st.session_state.tokens = gen_next(st.session_state.tokens, se)
+    else:
+        st.warning('Please init the generation')
+
+
+if undo:
+    if st.session_state.tokens is not None:
+        if st.session_state.tokens.shape[1] >= 34:
+            st.session_state.image, st.session_state.tokens = undo_gen(st.session_state.tokens)
+        else:
+            st.warning('no more step to undo')
+    else:
+        st.warning('Please init the generation')
+
+
+
+cont.image(st.session_state.image)
+
+

gist1/gpt.py

@@ -0,0 +1,192 @@
+# reference
+# https://blog.floydhub.com/the-transformer-in-pytorch/
+# https://github.com/hyunwoongko/transformer for the transformer architecture
+# https://github.com/Whiax/BERT-Transformer-Pytorch/blob/main/train.py (norm layer first)
+# https://github.com/karpathy/nanoGPT
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import _LRScheduler
+import numpy as np
+
+import time
+
+import copy
+
+def new_gelu(x):
+    """
+    Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT).
+    Reference: Gaussian Error Linear Units (GELU) paper: https://arxiv.org/abs/1606.08415
+    """
+    return 0.5 * x * (1.0 + torch.tanh(np.sqrt(2.0 / np.pi) * (x + 0.044715 * torch.pow(x, 3.0))))
+
+
+# https://github.com/jadore801120/attention-is-all-you-need-pytorch/blob/fec78a687210851f055f792d45300d27cc60ae41/transformer/Modules.py
+class ScaledDotProductAttention(nn.Module): 
+    def __init__(self, temperature, dropout=0.1):
+        super().__init__()
+        self.temperature = temperature
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, q, k, v, mask=None):
+        
+        attn = torch.matmul(q / self.temperature, k.transpose(-2, -1))
+
+        if mask is not None:
+            attn = attn.masked_fill(mask == 0, -1e9)
+
+        attn = F.softmax(attn, dim=-1)
+        attn = self.dropout(attn)
+        output = torch.matmul(attn, v)
+
+        return output
+
+class CausalMultiHeadAttention(nn.Module):
+    def __init__(self, heads, d_model, block_size, dropout=0.1):
+        super().__init__()
+
+        self.d_model = d_model
+        self.d_k = d_model // heads
+        self.h = heads
+
+        self.q_linear = nn.Linear(d_model, d_model, bias=False)
+        self.v_linear = nn.Linear(d_model, d_model, bias=False)
+        self.k_linear = nn.Linear(d_model, d_model, bias=False)
+        
+
+        self.attention = ScaledDotProductAttention(temperature=self.d_k**0.5)
+
+        # self.dropout = nn.Dropout(dropout)
+        self.out = nn.Linear(d_model, d_model, bias=False)
+
+        # causal mask
+        self.register_buffer("causal_mask", torch.tril(torch.ones(block_size, block_size))
+                                .view(1, 1, block_size, block_size))
+    
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, q, k, v):
+        bs, T, C = q.size()
+
+        # perform linear operation and  split into h heads
+        k = self.k_linear(k).view(bs, -1, self.h, self.d_k)
+        q = self.q_linear(q).view(bs, -1, self.h, self.d_k)
+        v = self.v_linear(v).view(bs, -1, self.h, self.d_k)
+
+        # transpose to get dimension of  bs * h * sl * d_model
+
+        k = k.transpose(1,2)
+        q = q.transpose(1,2)
+        v = v.transpose(1,2)
+
+        # causal_mask
+        mask = self.causal_mask[:,:,:T,:T]
+
+        # calculate attention
+        attn = self.attention(q, k, v, mask)
+
+        # concatenate heads and  put trough final linear layer
+        concat = attn.transpose(1,2).contiguous().view(bs, -1, self.d_model)
+
+        output = self.dropout(self.out(concat))
+
+        return output
+
+
+class FeedForward(nn.Module):
+    def __init__(self, d_model, dropout=0.1):
+        super().__init__()
+        # we set d_ff as a default to 2048
+        self.linear_1 = nn.Linear(d_model, 4 * d_model)
+        self.dropout = nn.Dropout(dropout)
+        self.linear_2 = nn.Linear(4 * d_model, d_model)
+    
+    def forward(self, x):
+        x = self.linear_1(x)
+        x = new_gelu(x)
+        x = self.linear_2(x)
+        x = self.dropout(x)
+        return x
+
+# the implementation reference https://www.arxiv-vanity.com/papers/1911.03179/
+class Block(nn.Module):
+    def __init__(self, d_model, heads, block_size, dropout=0.1):
+        super().__init__()
+        self.norm_1 = nn.LayerNorm(d_model, eps=1e-6)
+        self.norm_2 = nn.LayerNorm(d_model, eps=1e-6)
+        self.attn = CausalMultiHeadAttention(heads, d_model, block_size)
+        self.ff = FeedForward(d_model)
+        # self.dropout_1 = nn.Dropout(dropout)
+        # self.dropout_2 = nn.Dropout(dropout)
+
+    def forward(self, x):
+        # normalize
+        x2 = self.norm_1(x)
+        # compute self attention
+        x2 = self.attn(x2, x2, x2)
+        # x2 = self.dropout_1(x2)
+        # residual
+        x = x + x2
+        # normalize
+        x2= self.norm_2(x)
+        # positionwise feed forward network
+        x2 = self.ff(x2)
+        # x2 = self.dropout_2(x2)
+        # residual
+        x = x + x2
+        return x
+
+# layer multiplier
+def get_clones(module, N):
+    return nn.ModuleList([copy.deepcopy(module)for i in range(N)])
+
+class GPT(nn.Module):
+    def __init__(self, vocab_size, d_model, N, heads, block_size=80, dropout=0.1):
+        super().__init__()
+        self.N = N
+        self.embed = nn.Embedding(vocab_size, d_model)
+        # self.pe = nn.Embedding(block_size, d_model)
+        self.pe = nn.Parameter(torch.zeros(1, block_size, d_model)) 
+        self.dropout = nn.Dropout(dropout)
+        self.layers = get_clones(Block(d_model, heads, block_size), N)
+        self.norm = nn.LayerNorm(d_model, eps=1e-6)
+        self.out = nn.Linear(d_model, vocab_size, bias=False)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, (nn.Linear, nn.Embedding)):
+            module.weight.data.normal_(mean=0.0, std=0.02)
+            if isinstance(module, nn.Linear) and module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+    
+    def forward(self, src):
+        b, t = src.size()
+        # pos = torch.arange(0, t, dtype=torch.long, device=device).unsqueeze(0) # shape (1, t)
+        tok_emb = self.embed(src)
+        #pos_emb = self.pe(pos)
+        position_embeddings = self.pe[:, :t, :] 
+        x = tok_emb + position_embeddings
+        x = self.dropout(x)
+        x = self.norm(x)
+        for i in range(self.N):
+            x = self.layers[i](x)
+        x = self.norm(x)
+        x = self.out(x)
+        return x
+
+
+class Scheduler(_LRScheduler):
+    def __init__(self, optimizer, dim_embed, warmpup_steps, last_epoch=-1, verbose=False):       
+        self.dim_embed = dim_embed
+        self.warmup_steps = warmpup_steps
+        self.num_param_groups = len(optimizer.param_groups)
+        super().__init__(optimizer, last_epoch, verbose)
+
+    def get_lr(self):
+        lr = self.dim_embed**(-0.5) * min(self._step_count**(-0.5),self._step_count * self.warmup_steps**(-1.5))
+        return [lr] * self.num_param_groups

gist1/vqvae.py

@@ -0,0 +1,290 @@
+# reference https://github.com/zalandoresearch/pytorch-vq-vae
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class VectorQuantizer(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost):
+        super().__init__()
+
+        self.embedding_dim = embedding_dim
+        self.num_embeddings = num_embeddings
+
+        self.embedding = nn.Embedding(self.num_embeddings, self.embedding_dim)
+        self.embedding.weight.data.uniform_(-1/self.num_embeddings, 1/self.num_embeddings)
+        self.commitment_cost = commitment_cost
+
+    def forward(self, inputs):
+        # convert input from BCW -> BWC
+        inputs = inputs.permut(0, 2, 1).contiguous()
+        input_shape = inputs.shape
+
+        # flatten input
+        flat_input = inputs.view(-1, self.embedding_dim)
+
+        # calculate distances
+        distances = (torch.sum(flat_input**2, dim=1, keepdim=True)
+                    + torch.sum(self.embedding.weight**2, dim=1)
+                    - 2 * torch.matmul(flat_input, self.embedding.weight.t()))
+
+        # encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        encodings = torch.zeros(encoding_indices.shape[0], self.num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # quantize and unflatten
+        quantized = torch.matmul(encodings, self.embedding.weight).view(input_shape)
+        
+        # loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        q_latent_loss = F.mse_loss(quantized, input.detach())
+        loss = q_latent_loss + self.commitment_cost * e_latent_loss
+
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+        # convert quantized from BWC -> BCW
+        return loss, quantized.permute(0, 2, 1).contiguous(), perplexity, encodings
+
+
+class VectorQuantizerEMA(nn.Module):
+    def __init__(self, num_embeddings, embedding_dim, commitment_cost, decay, epsilon=1e-5):
+        super().__init__()
+
+        self.embedding_dim = embedding_dim
+        self.num_embeddings = num_embeddings
+
+        self.embedding = nn.Embedding(self.num_embeddings, self.embedding_dim)
+        self.embedding.weight.data.normal_()
+        self.commitment_cost = commitment_cost
+
+        self.register_buffer('ema_cluster_size', torch.zeros(num_embeddings))
+        self.ema_w = nn.Parameter(torch.Tensor(num_embeddings, self.embedding_dim))
+        self.ema_w.data.normal_()
+
+        self.decay = decay
+        self.epsilon = epsilon
+
+    def forward(self, inputs):
+        #convert inputs from BCW -> BWC
+        inputs = inputs.permute(0, 2, 1).contiguous()
+        input_shape = inputs.shape
+
+        # flatten input
+        flat_input = inputs.view(-1, self.embedding_dim)
+
+        # calculate distances
+        distances = (torch.sum(flat_input**2, dim=1, keepdim=True)
+                    + torch.sum(self.embedding.weight**2, dim=1)
+                    - 2 * torch.matmul(flat_input, self.embedding.weight.t()))
+
+        # encoding
+        encoding_indices = torch.argmin(distances, dim=1).unsqueeze(1)
+        encodings = torch.zeros(encoding_indices.shape[0], self.num_embeddings, device=inputs.device)
+        encodings.scatter_(1, encoding_indices, 1)
+
+        # quantize and unflatten
+        quantized = torch.matmul(encodings, self.embedding.weight).view(input_shape)
+
+        # use EMA to update the embedding vectors
+        if self.training:
+            self.ema_cluster_size = self.ema_cluster_size * self.decay + (1 - self.decay) * torch.sum(encodings, 0)
+
+            # laplace smoothing of the cluster size
+            n = torch.sum(self.ema_cluster_size)
+            self.ema_cluster_size = self.ema_cluster_size + self.epsilon / (n + self.num_embeddings * self.epsilon * n)
+            dw = torch.matmul(encodings.t(), flat_input)
+            self.ema_w = nn.Parameter(self.ema_w * self.decay + (1 - self.decay) * dw)
+
+            self.embedding.weight = nn.Parameter(self.ema_w / self.ema_cluster_size.unsqueeze(1))
+        
+        # loss
+        e_latent_loss = F.mse_loss(quantized.detach(), inputs)
+        loss = self.commitment_cost * e_latent_loss
+
+        # straight trough estimator
+        quantized = inputs + (quantized - inputs).detach()
+        avg_probs = torch.mean(encodings, dim=0)
+        perplexity = torch.exp(-torch.sum(avg_probs * torch.log(avg_probs + 1e-10)))
+
+        # convert quantized from BWC -> BCW
+        return loss, quantized.permute(0, 2, 1).contiguous(), perplexity, encoding_indices
+
+
+class Residual(nn.Module):
+    def __init__(self, in_channels, num_hiddnes, num_residual_hiddens):
+        super().__init__()
+        self.block = nn.Sequential( nn.ReLU(inplace=True),
+                                    nn.Conv1d(  in_channels=in_channels,
+                                                out_channels=num_residual_hiddens,
+                                                kernel_size=3, stride=1, padding=1, bias=False, padding_mode='circular'),
+                                    nn.ReLU(inplace=True),
+                                    nn.Conv1d(in_channels=num_residual_hiddens,
+                                    out_channels=num_hiddnes,
+                                    kernel_size=1, stride=1, bias=False)
+                                    )
+    def forward(self, x):
+        return x + self.block(x)
+
+
+class ResidualStack(nn.Module):
+    def __init__(self, in_channels, num_hiddens, num_residual_layers, num_residual_hiddens):
+        super().__init__()
+        self.num_residual_layers = num_residual_layers
+        self.layers = nn.ModuleList([Residual(in_channels, num_hiddens, num_residual_hiddens)
+                                    for _ in range(self.num_residual_layers)])
+    
+    def forward(self, x):
+        for i in range(self.num_residual_layers):
+            x = self.layers[i](x)
+        return F.relu(x)
+
+
+class Encoder(nn.Module):
+    def __init__(self, in_channels, num_hiddens, num_residual_layers, num_residual_hiddens):
+        super().__init__()
+        # 256 -> 128
+        self.conv_1 = nn.Conv1d(in_channels=in_channels,
+                                out_channels=num_hiddens//2,
+                                kernel_size=4,
+                                stride=2, padding=1, padding_mode='circular')
+        # 128 -> 64
+        self.conv_2 = nn.Conv1d(in_channels=num_hiddens//2,
+                                out_channels=num_hiddens,
+                                kernel_size=4,
+                                stride=2, padding=1, padding_mode='circular')
+        # 64 -> 32
+        self.conv_3 = nn.Conv1d(in_channels=num_hiddens,
+                                out_channels=num_hiddens,
+                                kernel_size=4,
+                                stride=2, padding=1, padding_mode='circular')
+        # 32 -> 16
+        self.conv_4 = nn.Conv1d(in_channels=num_hiddens,
+                                out_channels=num_hiddens,
+                                kernel_size=4,
+                                stride=2, padding=1, padding_mode='circular')
+        self.conv_final = nn.Conv1d(in_channels=num_hiddens,
+                                    out_channels=num_hiddens,
+                                    kernel_size=3,
+                                    stride=1, padding=1, padding_mode='circular')
+        self.residual_stack = ResidualStack(in_channels=num_hiddens,
+                                            num_hiddens=num_hiddens,
+                                            num_residual_hiddens=num_residual_hiddens,
+                                            num_residual_layers=num_residual_layers)
+
+    def forward(self, inputs):
+        x = self.conv_1(inputs)
+        x = F.relu(x)
+
+        x = self.conv_2(x)
+        x = F.relu(x)
+
+        x = self.conv_3(x)
+        x = F.relu(x)
+
+        x = self.conv_4(x)
+        x = F.relu(x)
+
+        x = self.conv_final(x)
+        x = self.residual_stack(x)
+        
+        return x
+
+    
+class Decoder(nn.Module):
+    def __init__(self, in_channels, num_hiddens, num_residual_layers, num_residual_hiddens):
+        super().__init__()
+        self.conv_init = nn.Conv1d( in_channels=in_channels,
+                                    out_channels=num_hiddens,
+                                    kernel_size=3,
+                                    stride=1, padding=1)
+        self.residual_stack = ResidualStack(in_channels=num_hiddens,
+                                            num_hiddens=num_hiddens,
+                                            num_residual_layers=num_residual_layers,
+                                            num_residual_hiddens=num_residual_hiddens)
+        
+        # 16 -> 32
+        self.conv_trans_0 = nn.ConvTranspose1d( in_channels=num_hiddens,
+                                                out_channels=num_hiddens,
+                                                kernel_size=4,
+                                                stride=2, padding=1)
+
+        # 32 -> 64
+        self.conv_trans_1 = nn.ConvTranspose1d( in_channels=num_hiddens,
+                                                out_channels=num_hiddens,
+                                                kernel_size=4,
+                                                stride=2, padding=1)
+        # 64 -> 128
+        self.conv_trans_2 = nn.ConvTranspose1d( in_channels=num_hiddens,
+                                                out_channels=num_hiddens//2,
+                                                kernel_size=4,
+                                                stride=2, padding=1)
+        # 128 -> 256
+        self.conv_trans_3 = nn.ConvTranspose1d( in_channels=num_hiddens//2,
+                                                out_channels=1,
+                                                kernel_size=4,
+                                                stride=2, padding=1)
+    
+    def forward(self, inputs):
+        x = self.conv_init(inputs)
+
+        x = self.residual_stack(x)
+
+        x = self.conv_trans_0(x)
+        x = F.relu(x)
+
+        x = self.conv_trans_1(x)
+        x = F.relu(x)
+
+        x = self.conv_trans_2(x)
+        x = F.relu(x)
+
+        return self.conv_trans_3(x)
+
+
+class VQVAE(nn.Module):
+    def __init__(self, num_hiddens, num_residual_layers, num_residual_hiddens, num_embeddings,
+                        embedding_dim, commitment_cost, decay=0):
+        super().__init__()
+        self.encoder = Encoder( 1, num_hiddens,
+                                num_residual_layers,
+                                num_residual_hiddens)
+        self.pre_vq_conv = nn.Conv1d(   in_channels=num_hiddens,
+                                        out_channels=embedding_dim,
+                                        kernel_size=1,
+                                        stride=1)
+        
+        if decay > 0.0:
+            self.vq = VectorQuantizerEMA(num_embeddings, embedding_dim, commitment_cost, decay)
+        else:
+            self.vq = VectorQuantizer(num_embeddings, embedding_dim, commitment_cost)
+        
+        self.decoder = Decoder( embedding_dim,
+                                num_hiddens,
+                                num_residual_layers,
+                                num_residual_hiddens)
+        
+    def encode(self, x):
+        z = self.encoder(x)
+        z = self.pre_vq_conv(z)
+        _, quantized, _, encoding_indices = self.vq(z)
+
+        return quantized, encoding_indices
+
+    def decode(self, x):
+        return self.decoder(x)
+    
+    def forward(self, x):
+        z = self.encoder(x)
+        z = self.pre_vq_conv(z)
+
+        loss, quantized, perplexity, _ = self.vq(z)
+        x_recon = self.decoder(quantized)
+        
+        return loss, x_recon, perplexity
+
+
+
+

gist1/vqvae_gpt.py

@@ -0,0 +1,288 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from gist1.gpt import GPT
+from gist1.vqvae import VQVAE
+
+from utils.misc import save_params, load_params
+import os
+import time
+
+
+class VQVAETransformer(nn.Module):
+    def __init__(self, args):
+        super().__init__()
+        self.vqvae = self.load_vqvae(args)
+        self.transformer = self.load_transformer(args)
+        # self.sos_token = self.get_sos_token(args)
+        self.pkeep = args['pkeep']
+        self.vqvae_vocab_size = args['vocab_size']
+        self.loc_vocab_size = args['loc_vocab_size']
+        self.block_size = args['block_size']
+
+    def load_vqvae(self, args):
+        # VQVAE_path = args['vqvae_checkpoint']
+        # VQVAE_cfg = args['vqvae_cfg']
+        # cfg = load_params(VQVAE_cfg)
+        # seed= cfg['seed']
+        # torch.manual_seed(seed)
+        num_hiddens = args['vqvae_num_hiddens']
+        num_residual_layers = args['vqvae_num_residual_layers']
+        num_residual_hiddens = args['vqvae_num_residual_hiddens']
+        num_embeddings = args['latent_dim']
+        latent_dim = args['vqvae_latent_dim']
+        commitment_cost = args['vqvae_commitment_cost']
+        decay = args['vqvae_decay']
+        model = VQVAE(num_hiddens, num_residual_layers, num_residual_hiddens,
+                num_embeddings, latent_dim, commitment_cost,
+                decay)
+        # model.load_state_dict(torch.load(VQVAE_path))
+        # model = model.eval()
+        
+        # update args from vqvae cfg
+        args['vocab_size'] = num_embeddings
+
+        return model
+
+    def load_vqvae_weight(self, args):
+        VQVAE_path = args['vqvae_checkpoint']
+        self.vqvae.load_state_dict(torch.load(VQVAE_path))
+        self.vqvae.eval()
+
+    def load_transformer(self, args):
+        # seed= args['seed']
+        # torch.manual_seed(seed)
+        latent_dim = args['latent_dim']
+        heads = args['heads']
+        N = args['N']
+        block_size = args['block_size']
+        vocab_size = args['vocab_size'] + args['loc_vocab_size']
+        model = GPT(vocab_size, latent_dim, N, heads, block_size)
+        return model
+    
+    @torch.no_grad()
+    def encode_to_z(self, x):
+        quantized, indices = self.vqvae.encode(x)
+        indices = indices.view(quantized.shape[0], -1)
+        return quantized, indices
+
+    @ torch.no_grad()
+    def z_to_isovist(self, indices):
+        indices[indices > self.vqvae_vocab_size-1] = self.vqvae_vocab_size-1
+        embedding_dim = self.vqvae.vq.embedding_dim
+        ix_to_vectors = self.vqvae.vq.embedding(indices).reshape(indices.shape[0], -1, embedding_dim)
+        ix_to_vectors = ix_to_vectors.permute(0, 2, 1)
+        isovist = self.vqvae.decode(ix_to_vectors)
+        return isovist
+    
+    def loc_to_indices(self, x):
+        starting_index = self.vqvae_vocab_size
+        indices = x.long() + starting_index
+        return indices
+    
+    def indices_to_loc(self, indices):
+        starting_index = self.vqvae_vocab_size
+        locs = indices - starting_index
+        locs[locs < 0] = 0
+        locs[locs > (self.loc_vocab_size-1)] = self.loc_vocab_size-1
+        return locs
+    
+    def seq_encode(self, locs, isovists):
+        # BSW
+        indices_seq = []
+        # indices_loc = []
+        for i in range(isovists.shape[1]): # iterate trought the sequence
+            loc = locs[:, i].unsqueeze(1) # BL
+            indices_seq.append(self.loc_to_indices(loc))
+            isovist = isovists[:, i, :].unsqueeze(1) # BCW
+            _, indices = self.encode_to_z(isovist)
+            indices_seq.append(indices)
+        indices = torch.cat(indices_seq, dim=1)
+        return indices
+
+
+    def forward(self, indices):
+        device = indices.device
+        # indices = self.seq_encode(locs, isovists)
+
+
+        if self.training and self.pkeep < 1.0:
+            mask = torch.bernoulli(self.pkeep*torch.ones(indices.shape, device=device))
+            mask = mask.round().to(dtype=torch.int64)
+            random_indices = torch.randint_like(indices,  self.vqvae_vocab_size) # doesn't include sos token
+            new_indices = mask*indices + (1-mask)*random_indices
+        else:
+            new_indices = indices
+
+
+        target = indices[:, 1:]
+
+
+        logits =  self.transformer(new_indices[:, :-1])
+
+               
+
+        return logits, target
+
+    
+    def top_k_logits(self, logits, k):
+        v, ix = torch.topk(logits, k)
+        out = logits.clone()
+        out[out < v[..., [-1]]] = -float("inf")
+        return out
+
+
+   
+    def sample(self, x, steps, temp=1.0, top_k=100, seed=None, step_size=17, zeroing=False):
+        device = x.device
+        is_train = False
+        if self.transformer.training == True:
+            is_train = True    
+        self.transformer.eval()
+        block_size = self.block_size
+        generator = None
+        if seed is not None:
+            generator = torch.Generator(device).manual_seed(seed)
+        for k in range(steps):
+            if x.size(1) < block_size:
+                x_cond = x
+            else:
+                remain = step_size - (x.size(1) % step_size)
+                x_cond = x[:, -(block_size-remain):]  # crop context if needed
+                if zeroing:
+                    x_cond = x_cond.clone()
+                    x_cond[:, 0] = self.vqvae_vocab_size
+            logits = self.transformer(x_cond)
+            logits = logits[:, -1, :] / temp
+
+            if top_k is not None:
+                logits = self.top_k_logits(logits, top_k)
+
+            probs = F.softmax(logits, dim = -1)
+
+            ix = torch.multinomial(probs, num_samples=1, generator=generator)
+
+            x = torch.cat((x, ix), dim=1)
+
+        if is_train == True:
+            self.transformer.train()
+
+        return x
+    
+
+    def get_loc(self, ploc, dir):
+        if dir == 0:
+            loc = ploc
+        elif dir == 1:
+            loc = (ploc[0]+1, ploc[1])
+        elif dir == 2:
+            loc = (ploc[0]+1, ploc[1]+1)
+        elif dir == 3:
+            loc = (ploc[0], ploc[1]+1)
+        elif dir == 4:
+            loc = (ploc[0]-1, ploc[1]+1)
+        elif dir == 5:
+            loc = (ploc[0]-1, ploc[1])
+        elif dir == 6:
+            loc = (ploc[0]-1, ploc[1]-1)
+        elif dir == 7:
+            loc = (ploc[0], ploc[1]-1)
+        elif dir == 8:
+            loc = (ploc[0]+1, ploc[1]-1)
+        else:
+            raise NameError('Direction unknown')
+        return loc
+            
+    
+    def init_loc(self, x, step_size):
+        device = x.device
+        loc_dict = {}
+        loc = None
+        cached_loc = None
+        if x.shape[1] > 1:
+            steps = x.shape[1] -1
+            for k in range(steps):
+                if k % step_size == 0:
+                    dir = x[:,k].detach().item() - self.vqvae_vocab_size
+                    if dir == 0:
+                        loc = (0, 0) # init loc
+                    else:
+                        loc = self.get_loc(loc, dir) # getloc
+                    loc_dict[loc] = torch.empty(1,0).long().to(device)
+                    cached_loc = loc
+                else:
+                    ix = x[:,[k]]
+                    loc_dict[cached_loc]  = torch.cat((loc_dict[cached_loc], ix), dim = 1)
+        return loc_dict, loc
+    
+    def sample_memorized(self, x, steps, temp=1.0, top_k=100, seed=None, step_size=17, zeroing=False):
+        device = x.device
+        loc_dict, loc = self.init_loc(x, step_size)
+        is_train = False
+        if self.transformer.training == True:
+            is_train = True    
+        self.transformer.eval()
+        block_size = self.block_size
+        generator = None
+        if seed is not None:
+            generator = torch.Generator(device).manual_seed(seed)
+        is_visited = False
+        cache_counter = 0
+        # loc = None
+        for k in range(steps):
+            # check directionality
+            if k % step_size == 0:
+                dir = x[:,-1].detach().item() - self.vqvae_vocab_size
+                if dir == 0:
+                    is_visited = False
+                    loc = (0, 0) # init loc
+                    loc_dict[loc] = torch.empty(1,0).long().to(device)
+                else:
+                    loc = self.get_loc(loc, dir) # getloc
+                    if loc in loc_dict:
+                        is_visited = True
+                        cache_counter = 0
+                    else:
+                        is_visited = False
+                        loc_dict[loc] = torch.empty(1,0).long().to(device)
+
+
+            if x.size(1) < block_size:
+                x_cond = x
+            else:
+                remain = step_size - (x.size(1) % step_size)
+                x_cond = x[:, -(block_size-remain):]  # crop context if needed
+                if zeroing:
+                    x_cond = x_cond.clone()
+                    x_cond[:, 0] = self.vqvae_vocab_size
+
+            if is_visited == False:
+                logits = self.transformer(x_cond)
+                logits = logits[:, -1, :] / temp
+
+                if top_k is not None:
+                    logits = self.top_k_logits(logits, top_k)
+
+                probs = F.softmax(logits, dim = -1)
+                ix = torch.multinomial(probs, num_samples=1, generator=generator)
+                # print('this shouldnt')
+                loc_dict[loc] = torch.cat((loc_dict[loc], ix), dim = 1)
+            else:
+                if cache_counter == 15: #reaching end of latent code
+                    is_visited = False 
+                ix = loc_dict[loc][:,[cache_counter]]
+                # print(ix)
+                cache_counter += 1
+
+            x = torch.cat((x, ix), dim=1)
+
+
+        if is_train == True:
+            self.transformer.train()
+
+        return x
+
+
+    
+
+    

models/param.json

@@ -0,0 +1,22 @@
+{
+    "loc_vocab_size": 9,
+    "block_size": 255,
+    "batch_size": 4,
+    "seq_num": 8,
+    "seq_length": 15,
+    "block_seq_length": 15,
+    "p": 10.0,
+    "q": 0.001,
+    "loc_dim": 1,
+    "isovist_latent_dim": 16,
+    "latent_dim": 1024,
+    "heads": 16,
+    "N": 24,
+    "pkeep": 1.0,
+    "vqvae_num_hiddens": 512,
+    "vqvae_num_residual_layers": 4,
+    "vqvae_num_residual_hiddens":32,
+    "vqvae_latent_dim": 8,
+    "vqvae_commitment_cost": 0.25,
+    "vqvae_decay": 0.99
+}

models/vqvaegpt_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b50c99dcdf274b6936bbb51903e26a401e3b5dd3bed194f1f9a7bf3b4fa8a05
+size 1251118533

models/vqvaegpt_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:241ecccdbda134d226e2e765ab7278c16bc001a51ac34ea939fa89ead1fe8398
+size 1251118893

models/vqvaegpt_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2eba1d15b180caa8b5b7f69028cc17f1df69cfb58c2227eb3483739069302079
+size 1251118533

utils/isoutil.py

@@ -0,0 +1,669 @@
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.patches import Polygon
+from matplotlib.collections import PatchCollection
+
+
+
+def pol2car(rho, pi, xi, yi):
+    x = rho * np.cos(pi) + xi
+    y = rho * np.sin(pi) + yi
+    return (x, y)
+
+def car2pol(xi, yi):
+    rho = np.sqrt(xi**2 + yi**2)
+    phi = np.arctan2(yi, xi)
+    return (rho, phi)
+
+def car2polnorm(xi, yi):
+    rho = np.sqrt(xi**2 + yi**2)
+    phi = np.arctan2(yi, xi)
+    phi %= 2*np.pi
+    phi /= 2*np.pi
+    return (rho, phi)
+
+def plot_isovist(isovists, show_axis=False, s=0.1, figsize=(5,5)):
+    #transpose the matrix
+    # isovists = np.transpose(isovists, (isovists.ndim-1, isovists.ndim-2))
+    plt.switch_backend('agg')
+    fig = plt.figure(figsize=figsize)
+    points = []
+    res = np.pi/90
+    isovist = isovists
+    for j, rho in enumerate(isovist):
+        if rho < 1.0:
+            pt = pol2car(rho, j*res, 0, 0)
+            points.append(pt)
+    x = [i[0] for i in points]
+    y = [i[1] for i in points]
+    ax = fig.add_subplot(111)
+    ax.set_aspect('equal')
+    ax.set_xlim(-1,1)
+    ax.set_ylim(-1,1)
+    if not show_axis:
+        ax.axis('off')
+    ax.scatter(x, y, s, 'black')
+    return fig
+
+from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
+from matplotlib.figure import Figure
+
+
+def isovist_to_img(isovist, show_axis=False, s=0.1, figsize=(5,5)):
+    points = []
+    xy = (0, 0)
+    res = np.pi/90
+    isovist = isovist + 0.5
+    for j, rho in enumerate(isovist):
+            if rho <= 2.0:
+                pt = pol2car(rho, j*res, xy[0], xy[1])
+                points.append(pt)
+    x = [i[0] for i in points]
+    y = [i[1] for i in points]
+    fig = plt.figure(figsize=figsize)
+    canvas = FigureCanvas(fig)
+    ax = fig.add_subplot(111)
+    ax.set_aspect('equal')
+    ax.set_xlim(-1,1)
+    ax.set_ylim(-1,1)
+    if not show_axis:
+        ax.axis('off')
+    ax.scatter(x, y, s, 'black')
+
+    canvas.draw()
+    image = np.fromstring(canvas.tostring_rgb(), dtype='uint8')
+    return image
+
+def isovist_to_img_a(isovist, show_axis=False, s=0.1, figsize=(5,5)):
+    points = []
+    xy = (0, 0)
+    res = np.pi/128
+    isovist = isovist + 0.5
+    for j, rho in enumerate(isovist):
+            if rho <= 2.0:
+                pt = pol2car(rho, j*res, xy[0], xy[1])
+                points.append(pt)
+    x = [i[0] for i in points]
+    y = [i[1] for i in points]
+    fig = plt.figure(figsize=figsize)
+    canvas = FigureCanvas(fig)
+    ax = fig.add_subplot(111)
+    ax.set_aspect('equal')
+    ax.set_xlim(-1,1)
+    ax.set_ylim(-1,1)
+    if not show_axis:
+        ax.axis('off')
+    ax.scatter(x, y, s, 'black')
+
+    canvas.draw()
+    image = np.fromstring(canvas.tostring_rgb(), dtype='uint8')
+    return image
+
+def isovist_to_cartesian(isovist, x, y, scale):
+    points = []
+    xy = (x, y)
+    res = np.pi/90
+    isovist = isovist * scale
+    for j, rho in enumerate(isovist):
+        if rho <= scale:
+            pt = pol2car(rho, j*res, xy[0], xy[1])
+            points.append(pt)
+        else:
+            pt = pol2car(scale, j*res, xy[0], xy[1])
+            points.append(pt)
+    points = np.stack(points)
+    return(points)
+
+def isovist_to_cartesian_a(isovist, x, y, scale):
+    points = []
+    xy = (x, y)
+    res = np.pi/len(isovist)*2
+    isovist = isovist * scale
+    for j, rho in enumerate(isovist):
+        pt = pol2car(rho, j*res, xy[0], xy[1])
+        points.append(pt)
+    points = np.stack(points)
+    return(points)
+
+def isovist_to_cartesian_b(isovist, x, y):
+    points = []
+    xy = (x, y)
+    res = np.pi*2
+    isovist = isovist
+    for j, rho in isovist:
+        pt = pol2car(rho, j*res, xy[0], xy[1])
+        points.append(pt)
+    points = np.stack(points)
+    return(points)
+
+def isovist_to_cartesian_segment(isovist, x, y, scale):
+    points = []
+    segment = []
+    xy = (x, y)
+    res = np.pi/90
+    isovist = isovist * scale
+    p_rho = isovist[-1]
+    for j, rho in enumerate(isovist):
+        delta = abs(p_rho-rho)
+        if j == 0:
+            first_rho = rho
+        if rho < 0.98 * scale and delta < 0.05 * scale:
+            pt = pol2car(rho, j*res, xy[0], xy[1])
+            segment.append(pt)
+        else:
+            points.append(segment)
+            segment = []
+        p_rho = rho
+    if first_rho < 1.0 * scale and abs(rho-first_rho)< 0.05 * scale :
+        if len(points) > 0:
+            segment.extend(points[0])
+            points[0]=segment
+        else:
+            points.append(segment)
+    else:
+        points.append(segment)
+    segments = []
+    for i in range(len(points)):
+        if len(points[i])>0:
+            segment = np.stack(points[i])
+            segments.append(segment)
+    return(segments)
+
+def isovist_to_cartesian_segment_a(isovist, x, y, scale, max=0.98, min = 0.1, d=0.1):
+    points = []
+    segment = []
+    xy = (x, y)
+    res = np.pi/len(isovist)*2
+    isovist = isovist * scale
+    p_rho = isovist[-1]
+    for j, rho in enumerate(isovist):
+        delta = abs(p_rho-rho)
+        if j == 0:
+            first_rho = rho
+        if rho < max * scale and rho > min * scale and delta < d * scale:
+            pt = pol2car(rho, j*res, xy[0], xy[1])
+            segment.append(pt)
+        else:
+            points.append(segment)
+            segment = []
+        p_rho = rho
+    if first_rho < max * scale and  first_rho > min * scale and abs(rho-first_rho)< d * scale :
+        if len(points) > 0:
+            segment.extend(points[0])
+            points[0]=segment
+        else:
+            points.append(segment)
+    else:
+        points.append(segment)
+    segments = []
+    for i in range(len(points)):
+        if len(points[i])>0:
+            segment = np.stack(points[i])
+            segments.append(segment)
+    return(segments)
+
+
+def isovist_to_cartesian_segment_b(isovist, x, y):
+    points = []
+    segment = []
+    xy = (x, y)
+    res = np.pi*2
+    isovist = isovist
+    p_rho = isovist[-1, 1]
+    _i = 0
+    for j, rho in isovist:
+        delta = abs(p_rho-rho)
+        if _i == 0:
+            first_rho = rho
+        if rho < 0.98 and delta < 0.1 :
+            pt = pol2car(rho, j*res, xy[0], xy[1])
+            segment.append(pt)
+        else:
+            points.append(segment)
+            segment = []
+        p_rho = rho
+        _i += 1
+    if first_rho < 0.98  and abs(rho-first_rho)< 0.1:
+        if len(points) > 0:
+            segment.extend(points[0])
+            points[0]=segment
+        else:
+            points.append(segment)
+    else:
+        points.append(segment)
+    segments = []
+    for i in range(len(points)):
+        if len(points[i])>0:
+            segment = np.stack(points[i])
+            segments.append(segment)
+    return(segments)
+
+
+# plotting an isovist and return the numpy image
+def plot_isovist_numpy(k, text=None, figsize=(8,8)):
+    fig, ax = plt.subplots(1,1, figsize=figsize, dpi=300)
+
+    #plot isovist
+    xy = isovist_to_cartesian_a(k, 0, 0, 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('#dddddd')
+    p.set_edgecolor(None)
+    ax.add_collection(p)
+
+    # style
+    ax.set_aspect('equal')
+    lim = 1.2
+    ax.set_xlim(-lim,lim)
+    ax.set_ylim(-lim,lim)
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+    if text != None:
+        ax.set_title(text, size=5) # Title
+        fig.tight_layout()
+
+    # for plot with torchvision util
+    fig.canvas.draw()
+    data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    w, h = fig.canvas.get_width_height()
+    im = data.reshape((int(h), int(w), -1))
+    im = im.transpose((2, 0, 1))
+    plt.close()
+    return im
+
+
+
+# plotting isovist and boundary from  and return the numpy image
+def plot_isovist_boundary_numpy(isovist, boundary, figsize=(8,8)):
+    fig, ax = plt.subplots(1,1, figsize=figsize, dpi=300)
+
+    #plot isovist
+    xy = isovist_to_cartesian_a(isovist, 0, 0, 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('#eeeeee')
+    p.set_edgecolor(None)
+    ax.add_collection(p)
+
+
+    #plot assumed boundary
+    edge_patches = []
+    segments = isovist_to_cartesian_segment_a(boundary, 0, 0, 1.0)
+    for segment in segments:
+        polygon = Polygon(segment, False)
+        edge_patches.append(polygon)
+    p = PatchCollection(edge_patches)
+    p.set_facecolor('none')
+    p.set_edgecolor('#000000')
+    p.set_linewidth(0.5)
+    ax.add_collection(p)
+
+    # style
+    ax.set_aspect('equal')
+    lim = 1.2
+    ax.set_xlim(-lim,lim)
+    ax.set_ylim(-lim,lim)
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+
+    # for plot with torchvision util
+    fig.canvas.draw()
+    data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    w, h = fig.canvas.get_width_height()
+    im = data.reshape((int(h), int(w), -1))
+    im = im.transpose((2, 0, 1))
+    plt.close()
+    return im
+
+
+# plotting two isovists (fill and edge) and return the numpy image
+def plot_isovist_double_numpy(isovist1, isovist2, figsize=(8,8)):
+    fig, ax = plt.subplots(1,1, figsize=figsize, dpi=300)
+
+    #plot isovist1
+    xy = isovist_to_cartesian_a(isovist1, 0, 0, 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('#dddddd')
+    p.set_edgecolor(None)
+    ax.add_collection(p)
+
+    #plot isovist2 as boundary
+    xy = isovist_to_cartesian_a(isovist2, 0, 0, 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('none')
+    p.set_edgecolor('#000000')
+    p.set_linewidth(0.2)
+    ax.add_collection(p)
+
+    # style
+    ax.set_aspect('equal')
+    lim = 1.2
+    ax.set_xlim(-lim,lim)
+    ax.set_ylim(-lim,lim)
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+
+    # for plot with torchvision util
+    fig.canvas.draw()
+    data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    w, h = fig.canvas.get_width_height()
+    im = data.reshape((int(h), int(w), -1))
+    im = im.transpose((2, 0, 1))
+    plt.close()
+    return im
+
+
+# plotting two isovists (fill and edge) and return the numpy image
+def plot_isovist_triple_numpy(isovists, locs, figsize=(8,8)):
+    isovist1, isovist2, isovist3 = isovists
+    loc1, loc2, loc3 = locs
+
+    fig, ax = plt.subplots(1,1, figsize=figsize, dpi=300)
+
+    #plot isovist1
+    xy = isovist_to_cartesian_a(isovist1, loc1[0], loc1[1], 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('#ffdddd')
+    p.set_edgecolor(None)
+    ax.add_collection(p)
+
+    #plot isovist2
+    xy = isovist_to_cartesian_a(isovist2, loc2[0], loc2[1], 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('#ddddff')
+    p.set_edgecolor(None)
+    ax.add_collection(p)
+
+    #plot isovist3 as boundary
+    xy = isovist_to_cartesian_a(isovist3, 0, 0, 1.0)
+    polygon = Polygon(xy, True)
+    p = PatchCollection([polygon])
+    p.set_facecolor('none')
+    p.set_edgecolor('#000000')
+    p.set_linewidth(0.2)
+    ax.add_collection(p)
+
+    ax.scatter([x[0] for x in locs], [x[1] for x in locs], c='k', s=8, marker='+')
+
+    annotation = ['x1', 'x2', 'y']
+    for i, anno in enumerate(annotation):
+        ax.annotate(anno, (locs[i][0]+0.1, locs[i][1]), size=8)
+
+    # style
+    ax.set_aspect('equal')
+    lim = 1.5
+    ax.set_xlim(-lim,lim)
+    ax.set_ylim(-lim,lim)
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+
+    # for plot with torchvision util
+    fig.canvas.draw()
+    data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    w, h = fig.canvas.get_width_height()
+    im = data.reshape((int(h), int(w), -1))
+    im = im.transpose((2, 0, 1))
+    plt.close()
+    return im
+
+# showing isovist sequence
+def seq_show(locs, isovists, figsize=(8, 8)):
+    # walk trough the sequence
+    p_loc = np.array((0, 0))
+    b_segments = []
+    b_points = []
+    isovists_pts = []
+    res = np.pi/128
+    p_loc = np.array([0,0])
+    cartesian_locs = []
+    for loc, isovist in zip(locs, isovists):
+        rel_pos = np.asarray(pol2car(loc[0], loc[1]*2*np.pi, p_loc[0], p_loc[1]))
+        for j, rho in enumerate(isovist):
+            if rho < 0.98 :
+                pt = pol2car(rho, j*res, rel_pos[0], rel_pos[1])
+                b_points.append(pt)
+        segments = isovist_to_cartesian_segment_a(isovist, rel_pos[0], rel_pos[1], 1.0)
+        b_segments.extend(segments)
+        isovists_pts.append(isovist_to_cartesian_a(isovist, rel_pos[0], rel_pos[1], 1.0))
+        cartesian_locs.append(rel_pos)
+        p_loc = rel_pos
+
+    fig, ax = plt.subplots(1,1, figsize=figsize, dpi=96)
+
+    
+    # isovists
+    isovist_poly = []
+    for isovist_pts in isovists_pts:
+        isovist_poly.append(Polygon(isovist_pts, True))
+    r = PatchCollection(isovist_poly)
+    r.set_facecolor('#000000')
+    r.set_edgecolor(None)
+    r.set_alpha(0.02)
+    ax.add_collection(r)
+    
+
+    # isovist path
+    q = PatchCollection([Polygon(cartesian_locs, False)])
+    q.set_facecolor('none')
+    q.set_edgecolor('#cccccc')
+    q.set_linewidth(1.0)
+    q.set_linestyle('dashed')
+    ax.add_collection(q)
+    ax.scatter([x[0] for x in cartesian_locs], [x[1] for x in cartesian_locs], s = 6.0, c='red')
+
+    # boundaries
+    edge_patches = []
+    for segment in b_segments:
+        polygon = Polygon(segment, False)
+        edge_patches.append(polygon)
+    p = PatchCollection(edge_patches)
+    p.set_facecolor('none')
+    p.set_edgecolor('#000000')
+    p.set_linewidth(1.0)
+    ax.add_collection(p)
+    ax.scatter([x[0] for x in b_points], [x[1] for x in b_points], s = 0.05, c='k')
+
+
+    # style
+    ax.set_aspect('equal')
+    lim = 1.5
+    ax.set_xlim(-lim,lim)
+    ax.set_ylim(-lim,lim)
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+
+    return fig
+
+
+# plotting isovist sequence
+def plot_isovist_sequence(locs, isovists, figsize=(8,8)):
+    fig = seq_show(locs, isovists, figsize=figsize)
+
+    # for plot with torchvision util
+    fig.canvas.draw()
+    data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    w, h = fig.canvas.get_width_height()
+    im = data.reshape((int(h), int(w), -1))
+    im = im.transpose((2, 0, 1))
+    plt.close()
+    return im
+
+
+def index_to_loc_grid(idx, d):
+    if idx == 0:
+        return np.array((0., 0.), dtype=np.float32)
+    elif idx == 1:
+        return np.array((d, 0.), dtype=np.float32)
+    elif idx == 2:
+        return np.array((d, d), dtype=np.float32)
+    elif idx == 3:
+        return np.array((0., d), dtype=np.float32)
+    elif idx == 4:
+        return np.array((-d, d), dtype=np.float32)
+    elif idx == 5:
+        return np.array((-d, 0.), dtype=np.float32)
+    elif idx == 6:
+        return np.array((-d, -d), dtype=np.float32)
+    elif idx == 7:
+        return np.array((0., -d), dtype=np.float32)
+    elif idx == 8:
+        return np.array((d, -d), dtype=np.float32)
+    else:
+        raise NameError('Direction unknown')
+
+
+
+# showing isovist sequence grid
+def seq_show_grid(locs, isovists, d=0.2, figsize=(8, 8), center=False, lim=1.5, alpha=0.02, rad=0.9, b_width=1.0, calculate_lim=False):
+    # walk trough the sequence
+    p_loc = np.array((0, 0))
+    b_segments = []
+    b_points = []
+    isovists_pts = []
+    res = np.pi/128
+    cartesian_locs = []
+    for loc, isovist in zip(locs, isovists):
+        rel_pos = index_to_loc_grid(loc, d) + p_loc
+        for j, rho in enumerate(isovist):
+            if rho < rad :
+                pt = pol2car(rho, j*res, rel_pos[0], rel_pos[1])
+                b_points.append(pt)
+        segments = isovist_to_cartesian_segment_a(isovist, rel_pos[0], rel_pos[1], 1.0)
+        b_segments.extend(segments)
+        isovists_pts.append(isovist_to_cartesian_a(isovist, rel_pos[0], rel_pos[1], 1.0))
+        cartesian_locs.append(rel_pos)
+        p_loc = rel_pos
+
+    if len(b_points) > 0:
+        b_points = np.stack(b_points)
+    else:
+        b_points =[]
+    isovists_pts = np.stack(isovists_pts)
+    # b_segments = np.stack(b_segments)
+    cartesian_locs = np.stack(cartesian_locs)
+
+    # set graphic properties
+    isovist_path_width = 0.1
+    isovist_path_pt1 = 6.0
+    isovist_path_pt2 = 10.0
+    isovist_boundary_pt = 0.05
+
+    if center == True:
+
+        bbox = get_bbox(b_points)
+        center_pt = get_center_pts(bbox, np_array=True)
+        b_points = [ pt - center_pt for pt in b_points]
+        isovists_pts = [ pt - center_pt for pt in isovists_pts]
+        b_segments =  [ pt - center_pt for pt in b_segments]
+        cartesian_locs = [ pt - center_pt for pt in cartesian_locs]
+
+    # resize image
+    if calculate_lim == True:
+        if bbox is not None:
+            max = np.max(np.abs(bbox))
+        else:
+            max = 2.0
+        if max > 2.0:
+            lim = ((max // 0.5) + 1) * 0.5
+            isovist_path_width *= 2.0/lim
+            isovist_path_pt1 *= 2.0/lim
+            isovist_path_pt2 *= 2.0/lim
+            isovist_boundary_pt *= 2.0/lim
+
+
+    fig, ax = plt.subplots(1,1, figsize=figsize, dpi=96)
+
+    
+
+    # isovists
+    isovist_poly = []
+    for isovist_pts in isovists_pts:
+        isovist_poly.append(Polygon(isovist_pts, True))
+    r = PatchCollection(isovist_poly)
+    r.set_facecolor('#00aabb')
+    r.set_edgecolor(None)
+    r.set_alpha(alpha)
+    ax.add_collection(r)
+
+    
+
+    # isovist path
+    q = PatchCollection([Polygon(cartesian_locs, False)])
+    q.set_facecolor('none')
+    q.set_edgecolor('red')
+    q.set_linewidth(isovist_path_width)
+    # q.set_linestyle('dashed')
+    ax.add_collection(q)
+    
+    # start_pt
+    ax.scatter([x[0] for x in cartesian_locs[:1]], [x[1] for x in cartesian_locs[:1]], s = isovist_path_pt1, c='k', marker='s')
+
+    # sequence
+    ax.scatter([x[0] for x in cartesian_locs[1:-1]], [x[1] for x in cartesian_locs[1:-1]], s = isovist_path_pt1, c='red')
+
+    # end pt
+    ax.scatter([x[0] for x in cartesian_locs[-1:]], [x[1] for x in cartesian_locs[-1:]], s = isovist_path_pt2, c='k', marker='x')
+
+    # boundaries
+    edge_patches = []
+    for segment in b_segments:
+        if len(segment) > 5:
+            polygon = Polygon(segment, False)
+            edge_patches.append(polygon)
+    p = PatchCollection(edge_patches)
+    p.set_facecolor('none')
+    p.set_edgecolor('#000000')
+    p.set_linewidth(b_width)
+    ax.scatter([x[0] for x in b_points], [x[1] for x in b_points], s = isovist_boundary_pt, c='#000000',)
+    # ax.add_collection(p)
+
+
+    # style
+    ax.set_aspect('equal')
+    lim = lim
+    ax.set_xlim(-lim,lim)
+    ax.set_ylim(-lim,lim)
+    ax.set_xticks([])
+    ax.set_yticks([])
+    ax.axis('off')
+
+    return fig
+
+# plotting isovist sequence grid
+def plot_isovist_sequence_grid(locs, isovists, figsize=(8,8), center=False, lim=1.5, alpha=0.02, rad=0.9, b_width=1.0, calculate_lim=False):
+    fig = seq_show_grid(locs, isovists, figsize=figsize, center=center, lim=lim, alpha=alpha, rad=rad, b_width=b_width, calculate_lim=calculate_lim)
+    # for plot with torchvision util
+    fig.canvas.draw()
+    data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)
+    w, h = fig.canvas.get_width_height()
+    im = data.reshape((int(h), int(w), -1))
+    im = im.transpose((2, 0, 1))
+    plt.close()
+    return im
+
+def get_bbox(pts):
+    if len(pts) > 0:
+        if type(pts) is list:
+            pts = np.stack(pts)
+        bbox = np.min(pts[:, 0]), np.max(pts[:, 0]), np.min(pts[:, 1]), np.max(pts[:, 1])
+        return bbox
+    else:
+        return None
+
+def get_center_pts(bbox, np_array = False):
+    if bbox is not None:
+        center = 0.5*(bbox[0] + bbox[1]),  0.5*(bbox[2] + bbox[3])
+        if np_array:
+            center = np.asarray(center)
+    else:
+        center = np.asarray([0,0])
+    return center

utils/misc.py

@@ -0,0 +1,67 @@
+import json
+from os.path import join
+import numpy as np
+from PIL import Image
+from utils.isoutil import *
+import torch
+import torchvision
+import sys
+
+
+class MeanTracker(object):
+    def __init__(self, name):
+        self.values = []
+        self.name = name
+
+    def add(self, val):
+        self.values.append(float(val))
+
+    def mean(self):
+        return np.mean(self.values)
+
+    def flush(self):
+        mean = self.mean()
+        self.values = []
+        return self.name, mean
+
+def save_params(config, training_path):
+    save_dict_path = join(training_path, 'param.json')
+    with open(save_dict_path, 'w') as outfile:
+        json.dump(config,
+                   outfile,
+                   sort_keys=False,
+                   indent=4,
+                   separators=(',', ': '))
+
+def load_params(config_file):
+    with open(config_file, 'r') as f:
+        data = json.load(f)
+    return data
+
+
+def save_images(isovists, iter_num, title, sample_folder):
+    figs=[]
+    for i, x_ in enumerate(isovists):
+        x_ = np.squeeze(x_)
+        figs.append(plot_isovist_numpy(x_, figsize=(1,1)))
+    figs = torch.tensor(figs, dtype=torch.float)
+    nrow = int(np.sqrt(isovists.shape[0]))
+    im = torchvision.utils.make_grid(figs, normalize=True, range=(0, 255), nrow=nrow)
+    im = Image.fromarray(np.uint8(np.transpose(im.numpy(), (1, 2, 0))*255))
+    im.save(join(sample_folder, f'{title}_{iter_num:06}.jpg'))
+
+
+def imshow(img):
+    npimg = img.numpy()
+    plt.figure(figsize = (30,30))
+    plt.imshow(np.transpose(npimg, (1, 2, 0)))
+    plt.axis('off')
+    plt.show()
+
+
+def write(text):
+    sys.stdout.write('\n' + text)
+    if hasattr(sys.stdout, 'flush'):
+        sys.stdout.flush()
+
+