Upload 46 files

matdeeplearn/__init__.py

@@ -0,0 +1,3 @@
+from .models import *
+from .training import *
+from .process import *

matdeeplearn/models/__init__.py

@@ -0,0 +1,16 @@
+from .gcn import GCN
+from .mpnn import MPNN
+from .schnet import SchNet
+from .cgcnn import CGCNN
+from .megnet import MEGNet
+from .descriptor_nn import SOAP, SM
+
+__all__ = [
+    "GCN",
+    "MPNN",
+    "SchNet",
+    "CGCNN",
+    "MEGNet",
+    "SOAP",
+    "SM",
+]

matdeeplearn/models/cgcnn.py

@@ -0,0 +1,174 @@
+import torch
+from torch import Tensor
+import torch.nn.functional as F
+from torch.nn import Sequential, Linear, BatchNorm1d
+import torch_geometric
+from torch_geometric.nn import (
+    Set2Set,
+    global_mean_pool,
+    global_add_pool,
+    global_max_pool,
+    CGConv,
+)
+from torch_scatter import scatter_mean, scatter_add, scatter_max, scatter
+
+
+# CGCNN
+class CGCNN(torch.nn.Module):
+    def __init__(
+        self,
+        data,
+        dim1=64,
+        dim2=64,
+        pre_fc_count=1,
+        gc_count=3,
+        post_fc_count=1,
+        pool="global_mean_pool",
+        pool_order="early",
+        batch_norm="True",
+        batch_track_stats="True",
+        act="relu",
+        dropout_rate=0.0,
+        **kwargs
+    ):
+        super(CGCNN, self).__init__()
+        
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.pool = pool
+        self.act = act
+        self.pool_order = pool_order
+        self.dropout_rate = dropout_rate
+        
+        ##Determine gc dimension dimension
+        assert gc_count > 0, "Need at least 1 GC layer"        
+        if pre_fc_count == 0:
+            gc_dim = data.num_features
+        else:
+            gc_dim = dim1
+        ##Determine post_fc dimension
+        if pre_fc_count == 0:
+            post_fc_dim = data.num_features
+        else:
+            post_fc_dim = dim1
+        ##Determine output dimension length
+        if data[0].y.ndim == 0:
+            output_dim = 1
+        else:
+            output_dim = len(data[0].y[0])
+
+        ##Set up pre-GNN dense layers (NOTE: in v0.1 this is always set to 1 layer)
+        if pre_fc_count > 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+            for i in range(pre_fc_count):
+                if i == 0:
+                    lin = torch.nn.Linear(data.num_features, dim1)
+                    self.pre_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim1, dim1)
+                    self.pre_lin_list.append(lin)
+        elif pre_fc_count == 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+
+        ##Set up GNN layers
+        self.conv_list = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(gc_count):
+            conv = CGConv(
+                gc_dim, data.num_edge_features, aggr="mean", batch_norm=False
+            )
+            self.conv_list.append(conv)
+            ##Track running stats set to false can prevent some instabilities; this causes other issues with different val/test performance from loader size?
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(gc_dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)
+
+        ##Set up post-GNN dense layers (NOTE: in v0.1 there was a minimum of 2 dense layers, and fc_count(now post_fc_count) added to this number. In the current version, the minimum is zero)
+        if post_fc_count > 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            for i in range(post_fc_count):
+                if i == 0:
+                    ##Set2set pooling has doubled dimension
+                    if self.pool_order == "early" and self.pool == "set2set":
+                        lin = torch.nn.Linear(post_fc_dim * 2, dim2)
+                    else:
+                        lin = torch.nn.Linear(post_fc_dim, dim2)
+                    self.post_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim2, dim2)
+                    self.post_lin_list.append(lin)
+            self.lin_out = torch.nn.Linear(dim2, output_dim)
+
+        elif post_fc_count == 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            if self.pool_order == "early" and self.pool == "set2set":
+                self.lin_out = torch.nn.Linear(post_fc_dim*2, output_dim)
+            else:
+                self.lin_out = torch.nn.Linear(post_fc_dim, output_dim)   
+
+        ##Set up set2set pooling (if used)
+        ##Should processing_setps be a hypereparameter?
+        if self.pool_order == "early" and self.pool == "set2set":
+            self.set2set = Set2Set(post_fc_dim, processing_steps=3)
+        elif self.pool_order == "late" and self.pool == "set2set":
+            self.set2set = Set2Set(output_dim, processing_steps=3, num_layers=1)
+            # workaround for doubled dimension by set2set; if late pooling not reccomended to use set2set
+            self.lin_out_2 = torch.nn.Linear(output_dim * 2, output_dim)
+
+    def forward(self, data):
+
+        ##Pre-GNN dense layers
+        for i in range(0, len(self.pre_lin_list)):
+            if i == 0:
+                out = self.pre_lin_list[i](data.x)
+                out = getattr(F, self.act)(out)
+            else:
+                out = self.pre_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+
+        ##GNN layers
+        for i in range(0, len(self.conv_list)):
+            if len(self.pre_lin_list) == 0 and i == 0:
+                if self.batch_norm == "True":
+                    out = self.conv_list[i](data.x, data.edge_index, data.edge_attr)
+                    out = self.bn_list[i](out)
+                else:
+                    out = self.conv_list[i](data.x, data.edge_index, data.edge_attr)
+            else:
+                if self.batch_norm == "True":
+                    out = self.conv_list[i](out, data.edge_index, data.edge_attr)
+                    out = self.bn_list[i](out)
+                else:
+                    out = self.conv_list[i](out, data.edge_index, data.edge_attr)            
+            #out = getattr(F, self.act)(out)
+            out = F.dropout(out, p=self.dropout_rate, training=self.training)
+
+        ##Post-GNN dense layers
+        if self.pool_order == "early":
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+
+        elif self.pool_order == "late":
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+                out = self.lin_out_2(out)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+                
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out

matdeeplearn/models/descriptor_nn.py

@@ -0,0 +1,69 @@
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+from torch.nn import (
+    Sequential,
+    Linear,
+    ReLU,
+    GRU,
+    Embedding,
+    BatchNorm1d,
+    Dropout,
+    LayerNorm,
+)
+from torch_geometric.nn import (
+    Set2Set,
+    global_mean_pool,
+    global_add_pool,
+    global_max_pool,
+)
+
+
+# Sine matrix with neural network
+class SM(torch.nn.Module):
+    def __init__(self, data, dim1=64, fc_count=1,  **kwargs):
+        super(SM, self).__init__()
+        
+        self.lin1 = torch.nn.Linear(data[0].extra_features_SM.shape[1], dim1)
+
+        self.lin_list = torch.nn.ModuleList(
+            [torch.nn.Linear(dim1, dim1) for i in range(fc_count)]
+        )
+
+        self.lin2 = torch.nn.Linear(dim1, 1)
+
+    def forward(self, data):
+
+        out = F.relu(self.lin1(data.extra_features_SM))
+        for layer in self.lin_list:
+            out = F.relu(layer(out))
+        out = self.lin2(out)
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out
+
+
+# Smooth Overlap of Atomic Positions with neural network
+class SOAP(torch.nn.Module):
+    def __init__(self, data, dim1, fc_count,  **kwargs):
+        super(SOAP, self).__init__()
+        
+        self.lin1 = torch.nn.Linear(data[0].extra_features_SOAP.shape[1], dim1)
+
+        self.lin_list = torch.nn.ModuleList(
+            [torch.nn.Linear(dim1, dim1) for i in range(fc_count)]
+        )
+
+        self.lin2 = torch.nn.Linear(dim1, 1)
+
+    def forward(self, data):
+
+        out = F.relu(self.lin1(data.extra_features_SOAP))
+        for layer in self.lin_list:
+            out = F.relu(layer(out))
+        out = self.lin2(out)
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out

matdeeplearn/models/gcn.py

@@ -0,0 +1,173 @@
+import torch
+from torch import Tensor
+import torch.nn.functional as F
+from torch.nn import Sequential, Linear, BatchNorm1d
+import torch_geometric
+from torch_geometric.nn import (
+    Set2Set,
+    global_mean_pool,
+    global_add_pool,
+    global_max_pool,
+    GCNConv,
+)
+from torch_scatter import scatter_mean, scatter_add, scatter_max, scatter
+
+
+# CGCNN
+class GCN(torch.nn.Module):
+    def __init__(
+        self,
+        data,
+        dim1=64,
+        dim2=64,
+        pre_fc_count=1,
+        gc_count=3,
+        post_fc_count=1,
+        pool="global_mean_pool",
+        pool_order="early",
+        batch_norm="True",
+        batch_track_stats="True",
+        act="relu",
+        dropout_rate=0.0,
+        **kwargs
+    ):
+        super(GCN, self).__init__()
+        
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.pool = pool
+        self.act = act
+        self.pool_order = pool_order
+        self.dropout_rate = dropout_rate
+        
+        ##Determine gc dimension dimension
+        assert gc_count > 0, "Need at least 1 GC layer"
+        if pre_fc_count == 0:
+            gc_dim = data.num_features
+        else:
+            gc_dim = dim1
+        ##Determine post_fc dimension
+        if pre_fc_count == 0:
+            post_fc_dim = data.num_features
+        else:
+            post_fc_dim = dim1
+        ##Determine output dimension length
+        if data[0].y.ndim == 0:
+            output_dim = 1
+        else:
+            output_dim = len(data[0].y[0])
+
+        ##Set up pre-GNN dense layers (NOTE: in v0.1 this is always set to 1 layer)
+        if pre_fc_count > 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+            for i in range(pre_fc_count):
+                if i == 0:
+                    lin = torch.nn.Linear(data.num_features, dim1)
+                    self.pre_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim1, dim1)
+                    self.pre_lin_list.append(lin)
+        elif pre_fc_count == 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+
+        ##Set up GNN layers
+        self.conv_list = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(gc_count):
+            conv = GCNConv(
+                gc_dim, gc_dim, improved=True, add_self_loops=False
+            )
+            self.conv_list.append(conv)
+            ##Track running stats set to false can prevent some instabilities; this causes other issues with different val/test performance from loader size?
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(gc_dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)
+
+        ##Set up post-GNN dense layers (NOTE: in v0.1 there was a minimum of 2 dense layers, and fc_count(now post_fc_count) added to this number. In the current version, the minimum is zero)
+        if post_fc_count > 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            for i in range(post_fc_count):
+                if i == 0:
+                    ##Set2set pooling has doubled dimension
+                    if self.pool_order == "early" and self.pool == "set2set":
+                        lin = torch.nn.Linear(post_fc_dim * 2, dim2)
+                    else:
+                        lin = torch.nn.Linear(post_fc_dim, dim2)
+                    self.post_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim2, dim2)
+                    self.post_lin_list.append(lin)
+            self.lin_out = torch.nn.Linear(dim2, output_dim)
+
+        elif post_fc_count == 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            if self.pool_order == "early" and self.pool == "set2set":
+                self.lin_out = torch.nn.Linear(post_fc_dim*2, output_dim)
+            else:
+                self.lin_out = torch.nn.Linear(post_fc_dim, output_dim)   
+
+        ##Set up set2set pooling (if used)
+        if self.pool_order == "early" and self.pool == "set2set":
+            self.set2set = Set2Set(post_fc_dim, processing_steps=3)
+        elif self.pool_order == "late" and self.pool == "set2set":
+            self.set2set = Set2Set(output_dim, processing_steps=3, num_layers=1)
+            # workaround for doubled dimension by set2set; if late pooling not reccomended to use set2set
+            self.lin_out_2 = torch.nn.Linear(output_dim * 2, output_dim)
+
+    def forward(self, data):
+
+        ##Pre-GNN dense layers
+        for i in range(0, len(self.pre_lin_list)):
+            if i == 0:
+                out = self.pre_lin_list[i](data.x)
+                out = getattr(F, self.act)(out)
+            else:
+                out = self.pre_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+
+        ##GNN layers
+        for i in range(0, len(self.conv_list)):
+            if len(self.pre_lin_list) == 0 and i == 0:
+                if self.batch_norm == "True":
+                    out = self.conv_list[i](data.x, data.edge_index, data.edge_weight)
+                    out = self.bn_list[i](out)
+                else:
+                    out = self.conv_list[i](data.x, data.edge_index, data.edge_weight)
+            else:
+                if self.batch_norm == "True":
+                    out = self.conv_list[i](out, data.edge_index, data.edge_weight)
+                    out = self.bn_list[i](out)
+                else:
+                    out = self.conv_list[i](out, data.edge_index, data.edge_weight)            
+            out = getattr(F, self.act)(out)
+            out = F.dropout(out, p=self.dropout_rate, training=self.training)
+
+        ##Post-GNN dense layers
+        if self.pool_order == "early":
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+
+        elif self.pool_order == "late":
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+                out = self.lin_out_2(out)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+                
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out

matdeeplearn/models/megnet.py

@@ -0,0 +1,371 @@
+import torch
+from torch import Tensor
+import torch.nn.functional as F
+from torch.nn import Sequential, Linear, ReLU, BatchNorm1d
+import torch_geometric
+from torch_geometric.nn import (
+    Set2Set,
+    global_mean_pool,
+    global_add_pool,
+    global_max_pool,
+    MetaLayer,
+)
+from torch_scatter import scatter_mean, scatter_add, scatter_max, scatter
+
+# Megnet
+class Megnet_EdgeModel(torch.nn.Module):
+    def __init__(self, dim, act, batch_norm, batch_track_stats, dropout_rate, fc_layers=2):
+        super(Megnet_EdgeModel, self).__init__()
+        self.act=act
+        self.fc_layers = fc_layers
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.dropout_rate = dropout_rate
+                
+        self.edge_mlp = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(self.fc_layers + 1):
+            if i == 0:
+                lin = torch.nn.Linear(dim * 4, dim)
+                self.edge_mlp.append(lin)       
+            else:      
+                lin = torch.nn.Linear(dim, dim)
+                self.edge_mlp.append(lin) 
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)            
+
+    def forward(self, src, dest, edge_attr, u, batch):
+        comb = torch.cat([src, dest, edge_attr, u[batch]], dim=1)
+        for i in range(0, len(self.edge_mlp)):
+            if i == 0:
+                out = self.edge_mlp[i](comb)
+                out = getattr(F, self.act)(out)  
+                if self.batch_norm == "True":
+                    out = self.bn_list[i](out)
+                out = F.dropout(out, p=self.dropout_rate, training=self.training)
+            else:    
+                out = self.edge_mlp[i](out)
+                out = getattr(F, self.act)(out)    
+                if self.batch_norm == "True":
+                    out = self.bn_list[i](out)
+                out = F.dropout(out, p=self.dropout_rate, training=self.training)   
+        return out
+
+
+class Megnet_NodeModel(torch.nn.Module):
+    def __init__(self, dim, act, batch_norm, batch_track_stats, dropout_rate, fc_layers=2):
+        super(Megnet_NodeModel, self).__init__()
+        self.act=act
+        self.fc_layers = fc_layers
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.dropout_rate = dropout_rate
+                
+        self.node_mlp = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(self.fc_layers + 1):
+            if i == 0:
+                lin = torch.nn.Linear(dim * 3, dim)
+                self.node_mlp.append(lin)       
+            else:      
+                lin = torch.nn.Linear(dim, dim)
+                self.node_mlp.append(lin) 
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)    
+
+    def forward(self, x, edge_index, edge_attr, u, batch):
+        # row, col = edge_index
+        v_e = scatter_mean(edge_attr, edge_index[0, :], dim=0)
+        comb = torch.cat([x, v_e, u[batch]], dim=1)
+        for i in range(0, len(self.node_mlp)):
+            if i == 0:
+                out = self.node_mlp[i](comb)
+                out = getattr(F, self.act)(out)  
+                if self.batch_norm == "True":
+                    out = self.bn_list[i](out)
+                out = F.dropout(out, p=self.dropout_rate, training=self.training)
+            else:    
+                out = self.node_mlp[i](out)
+                out = getattr(F, self.act)(out)    
+                if self.batch_norm == "True":
+                    out = self.bn_list[i](out)
+                out = F.dropout(out, p=self.dropout_rate, training=self.training)   
+        return out
+
+
+class Megnet_GlobalModel(torch.nn.Module):
+    def __init__(self, dim, act, batch_norm, batch_track_stats, dropout_rate, fc_layers=2):
+        super(Megnet_GlobalModel, self).__init__()
+        self.act=act
+        self.fc_layers = fc_layers
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.dropout_rate = dropout_rate
+                
+        self.global_mlp = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(self.fc_layers + 1):
+            if i == 0:
+                lin = torch.nn.Linear(dim * 3, dim)
+                self.global_mlp.append(lin)       
+            else:      
+                lin = torch.nn.Linear(dim, dim)
+                self.global_mlp.append(lin) 
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)  
+
+    def forward(self, x, edge_index, edge_attr, u, batch):
+        u_e = scatter_mean(edge_attr, edge_index[0, :], dim=0)
+        u_e = scatter_mean(u_e, batch, dim=0)
+        u_v = scatter_mean(x, batch, dim=0)
+        comb = torch.cat([u_e, u_v, u], dim=1)
+        for i in range(0, len(self.global_mlp)):
+            if i == 0:
+                out = self.global_mlp[i](comb)
+                out = getattr(F, self.act)(out)  
+                if self.batch_norm == "True":
+                    out = self.bn_list[i](out)
+                out = F.dropout(out, p=self.dropout_rate, training=self.training)
+            else:    
+                out = self.global_mlp[i](out)
+                out = getattr(F, self.act)(out)    
+                if self.batch_norm == "True":
+                    out = self.bn_list[i](out)
+                out = F.dropout(out, p=self.dropout_rate, training=self.training)   
+        return out
+
+
+class MEGNet(torch.nn.Module):
+    def __init__(
+        self,
+        data,
+        dim1=64,
+        dim2=64,
+        dim3=64,
+        pre_fc_count=1,
+        gc_count=3,
+        gc_fc_count=2,
+        post_fc_count=1,
+        pool="global_mean_pool",
+        pool_order="early",
+        batch_norm="True",
+        batch_track_stats="True",
+        act="relu",
+        dropout_rate=0.0,
+        **kwargs
+    ):
+        super(MEGNet, self).__init__()
+        
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.pool = pool
+        if pool == "global_mean_pool":
+            self.pool_reduce="mean"
+        elif pool== "global_max_pool":
+            self.pool_reduce="max" 
+        elif pool== "global_sum_pool":
+            self.pool_reduce="sum"             
+        self.act = act
+        self.pool_order = pool_order
+        self.dropout_rate = dropout_rate
+        
+        ##Determine gc dimension dimension
+        assert gc_count > 0, "Need at least 1 GC layer"
+        if pre_fc_count == 0:
+            gc_dim = data.num_features
+        else:
+            gc_dim = dim1
+        ##Determine post_fc dimension
+        post_fc_dim = dim3
+        ##Determine output dimension length
+        if data[0].y.ndim == 0:
+            output_dim = 1
+        else:
+            output_dim = len(data[0].y[0])
+
+        ##Set up pre-GNN dense layers (NOTE: in v0.1 this is always set to 1 layer)
+        if pre_fc_count > 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+            for i in range(pre_fc_count):
+                if i == 0:
+                    lin = torch.nn.Linear(data.num_features, dim1)
+                    self.pre_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim1, dim1)
+                    self.pre_lin_list.append(lin)
+        elif pre_fc_count == 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+
+        ##Set up GNN layers
+        self.e_embed_list = torch.nn.ModuleList()
+        self.x_embed_list = torch.nn.ModuleList()
+        self.u_embed_list = torch.nn.ModuleList()   
+        self.conv_list = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(gc_count):
+            if i == 0:
+                e_embed = Sequential(
+                    Linear(data.num_edge_features, dim3), ReLU(), Linear(dim3, dim3), ReLU()
+                )
+                x_embed = Sequential(
+                    Linear(gc_dim, dim3), ReLU(), Linear(dim3, dim3), ReLU()
+                )
+                u_embed = Sequential(
+                    Linear((data[0].u.shape[1]), dim3), ReLU(), Linear(dim3, dim3), ReLU()
+                )
+                self.e_embed_list.append(e_embed)
+                self.x_embed_list.append(x_embed)
+                self.u_embed_list.append(u_embed)
+                self.conv_list.append(
+                    MetaLayer(
+                        Megnet_EdgeModel(dim3, self.act, self.batch_norm, self.batch_track_stats, self.dropout_rate, gc_fc_count),
+                        Megnet_NodeModel(dim3, self.act, self.batch_norm, self.batch_track_stats, self.dropout_rate, gc_fc_count),
+                        Megnet_GlobalModel(dim3, self.act, self.batch_norm, self.batch_track_stats, self.dropout_rate, gc_fc_count),
+                    )
+                )
+            elif i > 0:
+                e_embed = Sequential(Linear(dim3, dim3), ReLU(), Linear(dim3, dim3), ReLU())
+                x_embed = Sequential(Linear(dim3, dim3), ReLU(), Linear(dim3, dim3), ReLU())
+                u_embed = Sequential(Linear(dim3, dim3), ReLU(), Linear(dim3, dim3), ReLU())
+                self.e_embed_list.append(e_embed)
+                self.x_embed_list.append(x_embed)
+                self.u_embed_list.append(u_embed)
+                self.conv_list.append(
+                    MetaLayer(
+                        Megnet_EdgeModel(dim3, self.act, self.batch_norm, self.batch_track_stats, self.dropout_rate, gc_fc_count),
+                        Megnet_NodeModel(dim3, self.act, self.batch_norm, self.batch_track_stats, self.dropout_rate, gc_fc_count),
+                        Megnet_GlobalModel(dim3, self.act, self.batch_norm, self.batch_track_stats, self.dropout_rate, gc_fc_count),
+                    )
+                )
+
+        ##Set up post-GNN dense layers (NOTE: in v0.1 there was a minimum of 2 dense layers, and fc_count(now post_fc_count) added to this number. In the current version, the minimum is zero)
+        if post_fc_count > 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            for i in range(post_fc_count):
+                if i == 0:
+                    ##Set2set pooling has doubled dimension
+                    if self.pool_order == "early" and self.pool == "set2set":
+                        lin = torch.nn.Linear(post_fc_dim * 5, dim2)
+                    elif self.pool_order == "early" and self.pool != "set2set":
+                        lin = torch.nn.Linear(post_fc_dim * 3, dim2)
+                    elif self.pool_order == "late":
+                        lin = torch.nn.Linear(post_fc_dim, dim2)
+                    self.post_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim2, dim2)
+                    self.post_lin_list.append(lin)
+            self.lin_out = torch.nn.Linear(dim2, output_dim)
+
+        elif post_fc_count == 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            if self.pool_order == "early" and self.pool == "set2set":
+                self.lin_out = torch.nn.Linear(post_fc_dim * 5, output_dim)
+            elif self.pool_order == "early" and self.pool != "set2set":
+                self.lin_out = torch.nn.Linear(post_fc_dim * 3, output_dim)              
+            else:
+                self.lin_out = torch.nn.Linear(post_fc_dim, output_dim)   
+
+        ##Set up set2set pooling (if used)
+        if self.pool_order == "early" and self.pool == "set2set":
+            self.set2set_x = Set2Set(post_fc_dim, processing_steps=3)
+            self.set2set_e = Set2Set(post_fc_dim, processing_steps=3)
+        elif self.pool_order == "late" and self.pool == "set2set":
+            self.set2set_x = Set2Set(output_dim, processing_steps=3, num_layers=1)
+            # workaround for doubled dimension by set2set; if late pooling not reccomended to use set2set
+            self.lin_out_2 = torch.nn.Linear(output_dim * 2, output_dim)
+
+    def forward(self, data):
+
+        ##Pre-GNN dense layers
+        for i in range(0, len(self.pre_lin_list)):
+            if i == 0:
+                out = self.pre_lin_list[i](data.x)
+                out = getattr(F, self.act)(out)
+            else:
+                out = self.pre_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+
+        ##GNN layers        
+        for i in range(0, len(self.conv_list)):
+            if i == 0:
+                if len(self.pre_lin_list) == 0:
+                    e_temp = self.e_embed_list[i](data.edge_attr)
+                    x_temp = self.x_embed_list[i](data.x)
+                    u_temp = self.u_embed_list[i](data.u)
+                    x_out, e_out, u_out = self.conv_list[i](
+                        x_temp, data.edge_index, e_temp, u_temp, data.batch
+                    )
+                    x = torch.add(x_out, x_temp)
+                    e = torch.add(e_out, e_temp)
+                    u = torch.add(u_out, u_temp)
+                else:
+                    e_temp = self.e_embed_list[i](data.edge_attr)
+                    x_temp = self.x_embed_list[i](out)
+                    u_temp = self.u_embed_list[i](data.u)
+                    x_out, e_out, u_out = self.conv_list[i](
+                        x_temp, data.edge_index, e_temp, u_temp, data.batch
+                    )
+                    x = torch.add(x_out, x_temp)
+                    e = torch.add(e_out, e_temp)
+                    u = torch.add(u_out, u_temp)
+                    
+            elif i > 0:
+                e_temp = self.e_embed_list[i](e)
+                x_temp = self.x_embed_list[i](x)
+                u_temp = self.u_embed_list[i](u)
+                x_out, e_out, u_out = self.conv_list[i](
+                    x_temp, data.edge_index, e_temp, u_temp, data.batch
+                )
+                x = torch.add(x_out, x)
+                e = torch.add(e_out, e)
+                u = torch.add(u_out, u)          
+
+        ##Post-GNN dense layers
+        if self.pool_order == "early":
+            if self.pool == "set2set":
+                x_pool = self.set2set_x(x, data.batch)
+                e = scatter(e, data.edge_index[0, :], dim=0, reduce="mean")
+                e_pool = self.set2set_e(e, data.batch)
+                out = torch.cat([x_pool, e_pool, u], dim=1)                
+            else:
+                x_pool = scatter(x, data.batch, dim=0, reduce=self.pool_reduce)
+                e_pool = scatter(e, data.edge_index[0, :], dim=0, reduce=self.pool_reduce)
+                e_pool = scatter(e_pool, data.batch, dim=0, reduce=self.pool_reduce)
+                out = torch.cat([x_pool, e_pool, u], dim=1)
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+
+        ##currently only uses node features for late pooling
+        elif self.pool_order == "late":
+            out = x
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+            if self.pool == "set2set":
+                out = self.set2set_x(out, data.batch)
+                out = self.lin_out_2(out)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+                
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out

matdeeplearn/models/mpnn.py

@@ -0,0 +1,188 @@
+import torch
+from torch import Tensor
+import torch.nn.functional as F
+from torch.nn import Sequential, Linear, ReLU, BatchNorm1d, GRU
+import torch_geometric
+from torch_geometric.nn import (
+    Set2Set,
+    global_mean_pool,
+    global_add_pool,
+    global_max_pool,
+    NNConv,
+)
+from torch_scatter import scatter_mean, scatter_add, scatter_max, scatter
+
+
+# CGCNN
+class MPNN(torch.nn.Module):
+    def __init__(
+        self,
+        data,
+        dim1=64,
+        dim2=64,
+        dim3=64,
+        pre_fc_count=1,
+        gc_count=3,
+        post_fc_count=1,
+        pool="global_mean_pool",
+        pool_order="early",
+        batch_norm="True",
+        batch_track_stats="True",
+        act="relu",
+        dropout_rate=0.0,
+        **kwargs
+    ):
+        super(MPNN, self).__init__()
+
+        
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.pool = pool
+        self.act = act
+        self.pool_order = pool_order
+        self.dropout_rate = dropout_rate
+        
+        ##Determine gc dimension dimension
+        assert gc_count > 0, "Need at least 1 GC layer"        
+        if pre_fc_count == 0:
+            gc_dim = data.num_features
+        else:
+            gc_dim = dim1
+        ##Determine post_fc dimension
+        if pre_fc_count == 0:
+            post_fc_dim = data.num_features
+        else:
+            post_fc_dim = dim1
+        ##Determine output dimension length
+        if data[0].y.ndim == 0:
+            output_dim = 1
+        else:
+            output_dim = len(data[0].y[0])
+
+        ##Set up pre-GNN dense layers (NOTE: in v0.1 this is always set to 1 layer)
+        if pre_fc_count > 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+            for i in range(pre_fc_count):
+                if i == 0:
+                    lin = torch.nn.Linear(data.num_features, dim1)
+                    self.pre_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim1, dim1)
+                    self.pre_lin_list.append(lin)
+        elif pre_fc_count == 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+
+        ##Set up GNN layers
+        self.conv_list = torch.nn.ModuleList()
+        self.gru_list = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(gc_count):
+            nn = Sequential(
+                Linear(data.num_edge_features, dim3), ReLU(), Linear(dim3, gc_dim * gc_dim)
+            )
+            conv = NNConv(
+                gc_dim, gc_dim, nn, aggr="mean"
+            )            
+            self.conv_list.append(conv)
+            gru = GRU(gc_dim, gc_dim)
+            self.gru_list.append(gru)
+
+            ##Track running stats set to false can prevent some instabilities; this causes other issues with different val/test performance from loader size?
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(gc_dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)
+
+        ##Set up post-GNN dense layers (NOTE: in v0.1 there was a minimum of 2 dense layers, and fc_count(now post_fc_count) added to this number. In the current version, the minimum is zero)
+        if post_fc_count > 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            for i in range(post_fc_count):
+                if i == 0:
+                    ##Set2set pooling has doubled dimension
+                    if self.pool_order == "early" and self.pool == "set2set":
+                        lin = torch.nn.Linear(post_fc_dim * 2, dim2)
+                    else:
+                        lin = torch.nn.Linear(post_fc_dim, dim2)
+                    self.post_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim2, dim2)
+                    self.post_lin_list.append(lin)
+            self.lin_out = torch.nn.Linear(dim2, output_dim)
+
+        elif post_fc_count == 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            if self.pool_order == "early" and self.pool == "set2set":
+                self.lin_out = torch.nn.Linear(post_fc_dim*2, output_dim)
+            else:
+                self.lin_out = torch.nn.Linear(post_fc_dim, output_dim)   
+
+        ##Set up set2set pooling (if used)
+        if self.pool_order == "early" and self.pool == "set2set":
+            self.set2set = Set2Set(post_fc_dim, processing_steps=3)
+        elif self.pool_order == "late" and self.pool == "set2set":
+            self.set2set = Set2Set(output_dim, processing_steps=3, num_layers=1)
+            # workaround for doubled dimension by set2set; if late pooling not reccomended to use set2set
+            self.lin_out_2 = torch.nn.Linear(output_dim * 2, output_dim)
+
+    def forward(self, data):
+
+        ##Pre-GNN dense layers
+        for i in range(0, len(self.pre_lin_list)):
+            if i == 0:
+                out = self.pre_lin_list[i](data.x)
+                out = getattr(F, self.act)(out)
+            else:
+                out = self.pre_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+
+        ##GNN layers
+        if len(self.pre_lin_list) == 0:
+            h = data.x.unsqueeze(0)    
+        else:
+            h = out.unsqueeze(0)                
+        for i in range(0, len(self.conv_list)):
+            if len(self.pre_lin_list) == 0 and i == 0:
+                if self.batch_norm == "True":
+                    m = self.conv_list[i](data.x, data.edge_index, data.edge_attr)
+                    m = self.bn_list[i](m)
+                else:
+                    m = self.conv_list[i](data.x, data.edge_index, data.edge_attr)
+            else:
+                if self.batch_norm == "True":
+                    m = self.conv_list[i](out, data.edge_index, data.edge_attr)
+                    m = self.bn_list[i](m)
+                else:
+                    m = self.conv_list[i](out, data.edge_index, data.edge_attr)            
+            m = getattr(F, self.act)(m)          
+            m = F.dropout(m, p=self.dropout_rate, training=self.training)
+            out, h = self.gru_list[i](m.unsqueeze(0), h)
+            out = out.squeeze(0)                
+
+        ##Post-GNN dense layers
+        if self.pool_order == "early":
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+
+        elif self.pool_order == "late":
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+                out = self.lin_out_2(out)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+                
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out

matdeeplearn/models/schnet.py

@@ -0,0 +1,172 @@
+import torch
+from torch import Tensor
+import torch.nn.functional as F
+from torch.nn import Sequential, Linear, BatchNorm1d
+import torch_geometric
+from torch_geometric.nn import (
+    Set2Set,
+    global_mean_pool,
+    global_add_pool,
+    global_max_pool,
+)
+from torch_scatter import scatter_mean, scatter_add, scatter_max, scatter
+from torch_geometric.nn.models.schnet import InteractionBlock
+
+# Schnet
+class SchNet(torch.nn.Module):
+    def __init__(
+        self,
+        data,
+        dim1=64,
+        dim2=64,
+        dim3=64,
+        cutoff=8,
+        pre_fc_count=1,
+        gc_count=3,
+        post_fc_count=1,
+        pool="global_mean_pool",
+        pool_order="early",
+        batch_norm="True",
+        batch_track_stats="True",
+        act="relu",
+        dropout_rate=0.0,
+        **kwargs
+    ):
+        super(SchNet, self).__init__()
+        
+        if batch_track_stats == "False":
+            self.batch_track_stats = False 
+        else:
+            self.batch_track_stats = True 
+        self.batch_norm = batch_norm
+        self.pool = pool
+        self.act = act
+        self.pool_order = pool_order
+        self.dropout_rate = dropout_rate
+        
+        ##Determine gc dimension dimension
+        assert gc_count > 0, "Need at least 1 GC layer"        
+        if pre_fc_count == 0:
+            gc_dim = data.num_features
+        else:
+            gc_dim = dim1
+        ##Determine post_fc dimension
+        if pre_fc_count == 0:
+            post_fc_dim = data.num_features
+        else:
+            post_fc_dim = dim1
+        ##Determine output dimension length
+        if data[0].y.ndim == 0:
+            output_dim = 1
+        else:
+            output_dim = len(data[0].y[0])
+
+        ##Set up pre-GNN dense layers (NOTE: in v0.1 this is always set to 1 layer)
+        if pre_fc_count > 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+            for i in range(pre_fc_count):
+                if i == 0:
+                    lin = torch.nn.Linear(data.num_features, dim1)
+                    self.pre_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim1, dim1)
+                    self.pre_lin_list.append(lin)
+        elif pre_fc_count == 0:
+            self.pre_lin_list = torch.nn.ModuleList()
+
+        ##Set up GNN layers       
+        self.conv_list = torch.nn.ModuleList()
+        self.bn_list = torch.nn.ModuleList()
+        for i in range(gc_count):
+            conv = InteractionBlock(gc_dim, data.num_edge_features, dim3, cutoff)
+            self.conv_list.append(conv)
+            ##Track running stats set to false can prevent some instabilities; this causes other issues with different val/test performance from loader size?
+            if self.batch_norm == "True":
+                bn = BatchNorm1d(gc_dim, track_running_stats=self.batch_track_stats)
+                self.bn_list.append(bn)
+
+        ##Set up post-GNN dense layers (NOTE: in v0.1 there was a minimum of 2 dense layers, and fc_count(now post_fc_count) added to this number. In the current version, the minimum is zero)
+        if post_fc_count > 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            for i in range(post_fc_count):
+                if i == 0:
+                    ##Set2set pooling has doubled dimension
+                    if self.pool_order == "early" and self.pool == "set2set":
+                        lin = torch.nn.Linear(post_fc_dim * 2, dim2)
+                    else:
+                        lin = torch.nn.Linear(post_fc_dim, dim2)
+                    self.post_lin_list.append(lin)
+                else:
+                    lin = torch.nn.Linear(dim2, dim2)
+                    self.post_lin_list.append(lin)
+            self.lin_out = torch.nn.Linear(dim2, output_dim)
+
+        elif post_fc_count == 0:
+            self.post_lin_list = torch.nn.ModuleList()
+            if self.pool_order == "early" and self.pool == "set2set":
+                self.lin_out = torch.nn.Linear(post_fc_dim*2, output_dim)
+            else:
+                self.lin_out = torch.nn.Linear(post_fc_dim, output_dim)   
+
+        ##Set up set2set pooling (if used)
+        if self.pool_order == "early" and self.pool == "set2set":
+            self.set2set = Set2Set(post_fc_dim, processing_steps=3)
+        elif self.pool_order == "late" and self.pool == "set2set":
+            self.set2set = Set2Set(output_dim, processing_steps=3, num_layers=1)
+            # workaround for doubled dimension by set2set; if late pooling not reccomended to use set2set
+            self.lin_out_2 = torch.nn.Linear(output_dim * 2, output_dim)
+
+    def forward(self, data):
+
+        ##Pre-GNN dense layers
+        for i in range(0, len(self.pre_lin_list)):
+            if i == 0:
+                out = self.pre_lin_list[i](data.x)
+                out = getattr(F, self.act)(out)
+            else:
+                out = self.pre_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+
+        ##GNN layers
+        for i in range(0, len(self.conv_list)):
+            if len(self.pre_lin_list) == 0 and i == 0:
+                if self.batch_norm == "True":
+                    out = data.x + self.conv_list[i](data.x, data.edge_index, data.edge_weight, data.edge_attr)
+                    out = self.bn_list[i](out)
+                else:
+                    out = data.x + self.conv_list[i](data.x, data.edge_index, data.edge_weight, data.edge_attr)
+            else:
+                if self.batch_norm == "True":
+                    out = out + self.conv_list[i](out, data.edge_index, data.edge_weight, data.edge_attr)
+                    out = self.bn_list[i](out)
+                else:
+                    out = out + self.conv_list[i](out, data.edge_index, data.edge_weight, data.edge_attr)            
+            #out = getattr(F, self.act)(out)
+            out = F.dropout(out, p=self.dropout_rate, training=self.training)
+
+        ##Post-GNN dense layers
+        if self.pool_order == "early":
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+
+        elif self.pool_order == "late":
+            for i in range(0, len(self.post_lin_list)):
+                out = self.post_lin_list[i](out)
+                out = getattr(F, self.act)(out)
+            out = self.lin_out(out)
+            if self.pool == "set2set":
+                out = self.set2set(out, data.batch)
+                out = self.lin_out_2(out)
+            else:
+                out = getattr(torch_geometric.nn, self.pool)(out, data.batch)
+                
+        if out.shape[1] == 1:
+            return out.view(-1)
+        else:
+            return out

matdeeplearn/models/utils.py

@@ -0,0 +1,23 @@
+import torch
+
+# Prints model summary
+def model_summary(model):
+    model_params_list = list(model.named_parameters())
+    print("--------------------------------------------------------------------------")
+    line_new = "{:>30}  {:>20} {:>20}".format(
+        "Layer.Parameter", "Param Tensor Shape", "Param #"
+    )
+    print(line_new)
+    print("--------------------------------------------------------------------------")
+    for elem in model_params_list:
+        p_name = elem[0]
+        p_shape = list(elem[1].size())
+        p_count = torch.tensor(elem[1].size()).prod().item()
+        line_new = "{:>30}  {:>20} {:>20}".format(p_name, str(p_shape), str(p_count))
+        print(line_new)
+    print("--------------------------------------------------------------------------")
+    total_params = sum([param.nelement() for param in model.parameters()])
+    print("Total params:", total_params)
+    num_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    print("Trainable params:", num_trainable_params)
+    print("Non-trainable params:", total_params - num_trainable_params)

matdeeplearn/process/__init__.py

@@ -0,0 +1 @@
+from .process import *

matdeeplearn/process/dictionary_blank.json

@@ -0,0 +1 @@
+{"1": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "2": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "3": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "4": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "5": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "6": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "7": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "8": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "9": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "10": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "11": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "12": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "13": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "14": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "15": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "16": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "17": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "18": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "19": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "20": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "21": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "22": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "23": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "24": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "25": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "26": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "27": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "28": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "29": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "30": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "31": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "32": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "33": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "34": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "35": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "36": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "37": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "38": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "39": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "40": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "41": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "42": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "43": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "44": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "45": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "46": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "47": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "48": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "49": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "50": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "51": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "52": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "53": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "54": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "55": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "56": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "57": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "58": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "59": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "60": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "61": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "62": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "63": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "64": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "65": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "66": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "67": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "68": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "69": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "70": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "71": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "72": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "73": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "74": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "75": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "76": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "77": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "78": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "79": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "80": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "81": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "82": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "83": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "84": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "85": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "86": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "87": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "88": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "89": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "90": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "91": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "92": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "93": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "94": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "95": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "96": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "97": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "98": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "99": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "100": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}

matdeeplearn/process/dictionary_default.json

@@ -0,0 +1 @@
+{"1": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "2": [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "3": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "4": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "5": [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "6": [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "7": [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "8": [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "9": [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "10": [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "11": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "12": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "13": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "14": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "15": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "16": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "17": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "18": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "19": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "20": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "21": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "22": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "23": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "24": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "25": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "26": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "27": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "28": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "29": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "30": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "31": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "32": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "33": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "34": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "35": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "36": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "37": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "38": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "39": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "40": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "41": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "42": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "43": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "44": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "45": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "46": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "47": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "48": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "49": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "50": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "51": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "52": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "53": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "54": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "55": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "56": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "57": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "58": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "59": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "60": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "61": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "62": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "63": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "64": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "65": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "66": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "67": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "68": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "69": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "70": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "71": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "72": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "73": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "74": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "75": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "76": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "77": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "78": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "79": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "80": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "81": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "82": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "83": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "84": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "85": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "86": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "87": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "88": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "89": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "90": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "91": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0], "92": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], "93": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "94": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], "95": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], "96": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], "97": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "98": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], "99": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], "100": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]}

matdeeplearn/process/process.py

@@ -0,0 +1,703 @@
+import os
+import sys
+import time
+import csv
+import json
+import warnings
+import numpy as np
+import ase
+import glob
+from ase import io
+from scipy.stats import rankdata
+from scipy import interpolate
+
+##torch imports
+import torch
+import torch.nn.functional as F
+from torch_geometric.data import DataLoader, Dataset, Data, InMemoryDataset
+from torch_geometric.utils import dense_to_sparse, degree, add_self_loops
+import torch_geometric.transforms as T
+from torch_geometric.utils import degree
+
+################################################################################
+# Data splitting
+################################################################################
+
+##basic train, val, test split
+def split_data(
+    dataset,
+    train_ratio,
+    val_ratio,
+    test_ratio,
+    seed=np.random.randint(1, 1e6),
+    save=False,
+):
+    dataset_size = len(dataset)
+    if (train_ratio + val_ratio + test_ratio) <= 1:
+        train_length = int(dataset_size * train_ratio)
+        val_length = int(dataset_size * val_ratio)
+        test_length = int(dataset_size * test_ratio)
+        unused_length = dataset_size - train_length - val_length - test_length
+        (
+            train_dataset,
+            val_dataset,
+            test_dataset,
+            unused_dataset,
+        ) = torch.utils.data.random_split(
+            dataset,
+            [train_length, val_length, test_length, unused_length],
+            generator=torch.Generator().manual_seed(seed),
+        )
+        print(
+            "train length:",
+            train_length,
+            "val length:",
+            val_length,
+            "test length:",
+            test_length,
+            "unused length:",
+            unused_length,
+            "seed :",
+            seed,
+        )
+        return train_dataset, val_dataset, test_dataset
+    else:
+        print("invalid ratios")
+
+
+##Basic CV split
+def split_data_CV(dataset, num_folds=5, seed=np.random.randint(1, 1e6), save=False):
+    dataset_size = len(dataset)
+    fold_length = int(dataset_size / num_folds)
+    unused_length = dataset_size - fold_length * num_folds
+    folds = [fold_length for i in range(num_folds)]
+    folds.append(unused_length)
+    cv_dataset = torch.utils.data.random_split(
+        dataset, folds, generator=torch.Generator().manual_seed(seed)
+    )
+    print("fold length :", fold_length, "unused length:", unused_length, "seed", seed)
+    return cv_dataset[0:num_folds]
+
+
+################################################################################
+# Pytorch datasets
+################################################################################
+
+##Fetch dataset; processes the raw data if specified
+def get_dataset(data_path, target_index, reprocess="False", processing_args=None):
+    if processing_args == None:
+        processed_path = "processed"
+    else:
+        processed_path = processing_args.get("processed_path", "processed")
+
+    transforms = GetY(index=target_index)
+
+    if os.path.exists(data_path) == False:
+        print("Data not found in:", data_path)
+        sys.exit()
+
+    if reprocess == "True":
+        os.system("rm -rf " + os.path.join(data_path, processed_path))
+        process_data(data_path, processed_path, processing_args)
+
+    if os.path.exists(os.path.join(data_path, processed_path, "data.pt")) == True:
+        dataset = StructureDataset(
+            data_path,
+            processed_path,
+            transforms,
+        )
+    elif os.path.exists(os.path.join(data_path, processed_path, "data0.pt")) == True:
+        dataset = StructureDataset_large(
+            data_path,
+            processed_path,
+            transforms,
+        )
+    else:
+        process_data(data_path, processed_path, processing_args)
+        if os.path.exists(os.path.join(data_path, processed_path, "data.pt")) == True:
+            dataset = StructureDataset(
+                data_path,
+                processed_path,
+                transforms,
+            )
+        elif os.path.exists(os.path.join(data_path, processed_path, "data0.pt")) == True:
+            dataset = StructureDataset_large(
+                data_path,
+                processed_path,
+                transforms,
+            )        
+    return dataset
+
+
+##Dataset class from pytorch/pytorch geometric; inmemory case
+class StructureDataset(InMemoryDataset):
+    def __init__(
+        self, data_path, processed_path="processed", transform=None, pre_transform=None
+    ):
+        self.data_path = data_path
+        self.processed_path = processed_path
+        super(StructureDataset, self).__init__(data_path, transform, pre_transform)
+        self.data, self.slices = torch.load(self.processed_paths[0])
+
+    @property
+    def raw_file_names(self):
+        return []
+
+    @property
+    def processed_dir(self):
+        return os.path.join(self.data_path, self.processed_path)
+
+    @property
+    def processed_file_names(self):
+        file_names = ["data.pt"]
+        return file_names
+
+
+##Dataset class from pytorch/pytorch geometric
+class StructureDataset_large(Dataset):
+    def __init__(
+        self, data_path, processed_path="processed", transform=None, pre_transform=None
+    ):
+        self.data_path = data_path
+        self.processed_path = processed_path
+        super(StructureDataset_large, self).__init__(
+            data_path, transform, pre_transform
+        )
+
+    @property
+    def raw_file_names(self):
+        return []
+
+    @property
+    def processed_dir(self):
+        return os.path.join(self.data_path, self.processed_path)
+
+    @property
+    def processed_file_names(self):
+        # file_names = ["data.pt"]
+        file_names = []
+        for file_name in glob.glob(self.processed_dir + "/data*.pt"):
+            file_names.append(os.path.basename(file_name))
+        # print(file_names)
+        return file_names
+
+    def len(self):
+        return len(self.processed_file_names)
+
+    def get(self, idx):
+        data = torch.load(os.path.join(self.processed_dir, "data_{}.pt".format(idx)))
+        return data
+
+
+################################################################################
+#  Processing
+################################################################################
+
+
+def process_data(data_path, processed_path, processing_args):
+
+    ##Begin processing data
+    print("Processing data to: " + os.path.join(data_path, processed_path))
+    assert os.path.exists(data_path), "Data path not found in " + data_path
+
+    ##Load dictionary
+    if processing_args["dictionary_source"] != "generated":
+        if processing_args["dictionary_source"] == "default":
+            print("Using default dictionary.")
+            atom_dictionary = get_dictionary(
+                os.path.join(
+                    os.path.dirname(os.path.realpath(__file__)),
+                    "dictionary_default.json",
+                )
+            )
+        elif processing_args["dictionary_source"] == "blank":
+            print(
+                "Using blank dictionary. Warning: only do this if you know what you are doing"
+            )
+            atom_dictionary = get_dictionary(
+                os.path.join(
+                    os.path.dirname(os.path.realpath(__file__)), "dictionary_blank.json"
+                )
+            )
+        else:
+            dictionary_file_path = os.path.join(
+                data_path, processing_args["dictionary_path"]
+            )
+            if os.path.exists(dictionary_file_path) == False:
+                print("Atom dictionary not found, exiting program...")
+                sys.exit()
+            else:
+                print("Loading atom dictionary from file.")
+                atom_dictionary = get_dictionary(dictionary_file_path)
+
+    ##Load targets
+    target_property_file = os.path.join(data_path, processing_args["target_path"])
+    assert os.path.exists(target_property_file), (
+        "targets not found in " + target_property_file
+    )
+    with open(target_property_file) as f:
+        reader = csv.reader(f)
+        target_data = [row for row in reader]
+
+    ##Read db file if specified
+    ase_crystal_list = []
+    if processing_args["data_format"] == "db":
+        db = ase.db.connect(os.path.join(data_path, "data.db"))
+        row_count = 0
+        # target_data=[]
+        for row in db.select():
+            # target_data.append([str(row_count), row.get('target')])
+            ase_temp = row.toatoms()
+            ase_crystal_list.append(ase_temp)
+            row_count = row_count + 1
+            if row_count % 500 == 0:
+                print("db processed: ", row_count)
+
+    ##Process structure files and create structure graphs
+    data_list = []
+    for index in range(0, len(target_data)):
+
+        structure_id = target_data[index][0]
+        data = Data()
+
+        ##Read in structure file using ase
+        if processing_args["data_format"] != "db":
+            ase_crystal = ase.io.read(
+                os.path.join(
+                    data_path, structure_id + "." + processing_args["data_format"]
+                )
+            )
+            data.ase = ase_crystal
+        else:
+            ase_crystal = ase_crystal_list[index]
+            data.ase = ase_crystal
+
+        ##Compile structure sizes (# of atoms) and elemental compositions
+        if index == 0:
+            length = [len(ase_crystal)]
+            elements = [list(set(ase_crystal.get_chemical_symbols()))]
+        else:
+            length.append(len(ase_crystal))
+            elements.append(list(set(ase_crystal.get_chemical_symbols())))
+
+        ##Obtain distance matrix with ase
+        distance_matrix = ase_crystal.get_all_distances(mic=True)
+
+        ##Create sparse graph from distance matrix
+        distance_matrix_trimmed = threshold_sort(
+            distance_matrix,
+            processing_args["graph_max_radius"],
+            processing_args["graph_max_neighbors"],
+            adj=False,
+        )
+
+        distance_matrix_trimmed = torch.Tensor(distance_matrix_trimmed)
+        out = dense_to_sparse(distance_matrix_trimmed)
+        edge_index = out[0]
+        edge_weight = out[1]
+
+        self_loops = True
+        if self_loops == True:
+            edge_index, edge_weight = add_self_loops(
+                edge_index, edge_weight, num_nodes=len(ase_crystal), fill_value=0
+            )
+            data.edge_index = edge_index
+            data.edge_weight = edge_weight
+
+            distance_matrix_mask = (
+                distance_matrix_trimmed.fill_diagonal_(1) != 0
+            ).int()
+        elif self_loops == False:
+            data.edge_index = edge_index
+            data.edge_weight = edge_weight
+
+            distance_matrix_mask = (distance_matrix_trimmed != 0).int()
+
+        data.edge_descriptor = {}
+        data.edge_descriptor["distance"] = edge_weight
+        data.edge_descriptor["mask"] = distance_matrix_mask
+
+        target = target_data[index][1:]
+        y = torch.Tensor(np.array([target], dtype=np.float32))
+        data.y = y
+
+        # pos = torch.Tensor(ase_crystal.get_positions())
+        # data.pos = pos
+        z = torch.LongTensor(ase_crystal.get_atomic_numbers())
+        data.z = z
+
+        ###placeholder for state feature
+        u = np.zeros((3))
+        u = torch.Tensor(u[np.newaxis, ...])
+        data.u = u
+
+        data.structure_id = [[structure_id] * len(data.y)]
+
+        if processing_args["verbose"] == "True" and (
+            (index + 1) % 500 == 0 or (index + 1) == len(target_data)
+        ):
+            print("Data processed: ", index + 1, "out of", len(target_data))
+            # if index == 0:
+            # print(data)
+            # print(data.edge_weight, data.edge_attr[0])
+
+        data_list.append(data)
+
+    ##
+    n_atoms_max = max(length)
+    species = list(set(sum(elements, [])))
+    species.sort()
+    num_species = len(species)
+    if processing_args["verbose"] == "True":
+        print(
+            "Max structure size: ",
+            n_atoms_max,
+            "Max number of elements: ",
+            num_species,
+        )
+        print("Unique species:", species)
+    crystal_length = len(ase_crystal)
+    data.length = torch.LongTensor([crystal_length])
+
+    ##Generate node features
+    if processing_args["dictionary_source"] != "generated":
+        ##Atom features(node features) from atom dictionary file
+        for index in range(0, len(data_list)):
+            atom_fea = np.vstack(
+                [
+                    atom_dictionary[str(data_list[index].ase.get_atomic_numbers()[i])]
+                    for i in range(len(data_list[index].ase))
+                ]
+            ).astype(float)
+            data_list[index].x = torch.Tensor(atom_fea)
+    elif processing_args["dictionary_source"] == "generated":
+        ##Generates one-hot node features rather than using dict file
+        from sklearn.preprocessing import LabelBinarizer
+
+        lb = LabelBinarizer()
+        lb.fit(species)
+        for index in range(0, len(data_list)):
+            data_list[index].x = torch.Tensor(
+                lb.transform(data_list[index].ase.get_chemical_symbols())
+            )
+
+    ##Adds node degree to node features (appears to improve performance)
+    for index in range(0, len(data_list)):
+        data_list[index] = OneHotDegree(
+            data_list[index], processing_args["graph_max_neighbors"] + 1
+        )
+
+    ##Get graphs based on voronoi connectivity; todo: also get voronoi features
+    ##avoid use for the time being until a good approach is found
+    processing_args["voronoi"] = "False"
+    if processing_args["voronoi"] == "True":
+        from pymatgen.core.structure import Structure
+        from pymatgen.analysis.structure_analyzer import VoronoiConnectivity
+        from pymatgen.io.ase import AseAtomsAdaptor
+
+        Converter = AseAtomsAdaptor()
+
+        for index in range(0, len(data_list)):
+            pymatgen_crystal = Converter.get_structure(data_list[index].ase)
+            # double check if cutoff distance does anything
+            Voronoi = VoronoiConnectivity(
+                pymatgen_crystal, cutoff=processing_args["graph_max_radius"]
+            )
+            connections = Voronoi.max_connectivity
+
+            distance_matrix_voronoi = threshold_sort(
+                connections,
+                9999,
+                processing_args["graph_max_neighbors"],
+                reverse=True,
+                adj=False,
+            )
+            distance_matrix_voronoi = torch.Tensor(distance_matrix_voronoi)
+
+            out = dense_to_sparse(distance_matrix_voronoi)
+            edge_index_voronoi = out[0]
+            edge_weight_voronoi = out[1]
+
+            edge_attr_voronoi = distance_gaussian(edge_weight_voronoi)
+            edge_attr_voronoi = edge_attr_voronoi.float()
+
+            data_list[index].edge_index_voronoi = edge_index_voronoi
+            data_list[index].edge_weight_voronoi = edge_weight_voronoi
+            data_list[index].edge_attr_voronoi = edge_attr_voronoi
+            if index % 500 == 0:
+                print("Voronoi data processed: ", index)
+
+    ##makes SOAP and SM features from dscribe
+    if processing_args["SOAP_descriptor"] == "True":
+        if True in data_list[0].ase.pbc:
+            periodicity = True
+        else:
+            periodicity = False
+
+        from dscribe.descriptors import SOAP
+            
+        make_feature_SOAP = SOAP(
+            species=species,
+            rcut=processing_args["SOAP_rcut"],
+            nmax=processing_args["SOAP_nmax"],
+            lmax=processing_args["SOAP_lmax"],
+            sigma=processing_args["SOAP_sigma"],
+            periodic=periodicity,
+            sparse=False,
+            average="inner",
+            rbf="gto",
+            crossover=False,
+        )
+        for index in range(0, len(data_list)):
+            features_SOAP = make_feature_SOAP.create(data_list[index].ase)
+            data_list[index].extra_features_SOAP = torch.Tensor(features_SOAP)
+            if processing_args["verbose"] == "True" and index % 500 == 0:
+                if index == 0:
+                    print(
+                        "SOAP length: ",
+                        features_SOAP.shape,
+                    )
+                print("SOAP descriptor processed: ", index)
+
+    elif processing_args["SM_descriptor"] == "True":
+        if True in data_list[0].ase.pbc:
+            periodicity = True
+        else:
+            periodicity = False
+
+        from dscribe.descriptors import SineMatrix, CoulombMatrix
+        
+        if periodicity == True:
+            make_feature_SM = SineMatrix(
+                n_atoms_max=n_atoms_max,
+                permutation="eigenspectrum",
+                sparse=False,
+                flatten=True,
+            )
+        else:
+            make_feature_SM = CoulombMatrix(
+                n_atoms_max=n_atoms_max,
+                permutation="eigenspectrum",
+                sparse=False,
+                flatten=True,
+            )
+            
+        for index in range(0, len(data_list)):
+            features_SM = make_feature_SM.create(data_list[index].ase)
+            data_list[index].extra_features_SM = torch.Tensor(features_SM)
+            if processing_args["verbose"] == "True" and index % 500 == 0:
+                if index == 0:
+                    print(
+                        "SM length: ",
+                        features_SM.shape,
+                    )
+                print("SM descriptor processed: ", index)
+
+    ##Generate edge features
+    if processing_args["edge_features"] == "True":
+
+        ##Distance descriptor using a Gaussian basis
+        distance_gaussian = GaussianSmearing(
+            0, 1, processing_args["graph_edge_length"], 0.2
+        )
+        # print(GetRanges(data_list, 'distance'))
+        NormalizeEdge(data_list, "distance")
+        # print(GetRanges(data_list, 'distance'))
+        for index in range(0, len(data_list)):
+            data_list[index].edge_attr = distance_gaussian(
+                data_list[index].edge_descriptor["distance"]
+            )
+            if processing_args["verbose"] == "True" and (
+                (index + 1) % 500 == 0 or (index + 1) == len(target_data)
+            ):
+                print("Edge processed: ", index + 1, "out of", len(target_data))
+
+    Cleanup(data_list, ["ase", "edge_descriptor"])
+
+    if os.path.isdir(os.path.join(data_path, processed_path)) == False:
+        os.mkdir(os.path.join(data_path, processed_path))
+
+    ##Save processed dataset to file
+    if processing_args["dataset_type"] == "inmemory":
+        data, slices = InMemoryDataset.collate(data_list)
+        torch.save((data, slices), os.path.join(data_path, processed_path, "data.pt"))
+
+    elif processing_args["dataset_type"] == "large":
+        for i in range(0, len(data_list)):
+            torch.save(
+                data_list[i],
+                os.path.join(
+                    os.path.join(data_path, processed_path), "data_{}.pt".format(i)
+                ),
+            )
+
+
+################################################################################
+#  Processing sub-functions
+################################################################################
+
+##Selects edges with distance threshold and limited number of neighbors
+def threshold_sort(matrix, threshold, neighbors, reverse=False, adj=False):
+    mask = matrix > threshold
+    distance_matrix_trimmed = np.ma.array(matrix, mask=mask)
+    if reverse == False:
+        distance_matrix_trimmed = rankdata(
+            distance_matrix_trimmed, method="ordinal", axis=1
+        )
+    elif reverse == True:
+        distance_matrix_trimmed = rankdata(
+            distance_matrix_trimmed * -1, method="ordinal", axis=1
+        )
+    distance_matrix_trimmed = np.nan_to_num(
+        np.where(mask, np.nan, distance_matrix_trimmed)
+    )
+    distance_matrix_trimmed[distance_matrix_trimmed > neighbors + 1] = 0
+
+    if adj == False:
+        distance_matrix_trimmed = np.where(
+            distance_matrix_trimmed == 0, distance_matrix_trimmed, matrix
+        )
+        return distance_matrix_trimmed
+    elif adj == True:
+        adj_list = np.zeros((matrix.shape[0], neighbors + 1))
+        adj_attr = np.zeros((matrix.shape[0], neighbors + 1))
+        for i in range(0, matrix.shape[0]):
+            temp = np.where(distance_matrix_trimmed[i] != 0)[0]
+            adj_list[i, :] = np.pad(
+                temp,
+                pad_width=(0, neighbors + 1 - len(temp)),
+                mode="constant",
+                constant_values=0,
+            )
+            adj_attr[i, :] = matrix[i, adj_list[i, :].astype(int)]
+        distance_matrix_trimmed = np.where(
+            distance_matrix_trimmed == 0, distance_matrix_trimmed, matrix
+        )
+        return distance_matrix_trimmed, adj_list, adj_attr
+
+
+##Slightly edited version from pytorch geometric to create edge from gaussian basis
+class GaussianSmearing(torch.nn.Module):
+    def __init__(self, start=0.0, stop=5.0, resolution=50, width=0.05, **kwargs):
+        super(GaussianSmearing, self).__init__()
+        offset = torch.linspace(start, stop, resolution)
+        # self.coeff = -0.5 / (offset[1] - offset[0]).item() ** 2
+        self.coeff = -0.5 / ((stop - start) * width) ** 2
+        self.register_buffer("offset", offset)
+
+    def forward(self, dist):
+        dist = dist.unsqueeze(-1) - self.offset.view(1, -1)
+        return torch.exp(self.coeff * torch.pow(dist, 2))
+
+
+##Obtain node degree in one-hot representation
+def OneHotDegree(data, max_degree, in_degree=False, cat=True):
+    idx, x = data.edge_index[1 if in_degree else 0], data.x
+    deg = degree(idx, data.num_nodes, dtype=torch.long)
+    deg = F.one_hot(deg, num_classes=max_degree + 1).to(torch.float)
+
+    if x is not None and cat:
+        x = x.view(-1, 1) if x.dim() == 1 else x
+        data.x = torch.cat([x, deg.to(x.dtype)], dim=-1)
+    else:
+        data.x = deg
+
+    return data
+
+
+##Obtain dictionary file for elemental features
+def get_dictionary(dictionary_file):
+    with open(dictionary_file) as f:
+        atom_dictionary = json.load(f)
+    return atom_dictionary
+
+
+##Deletes unnecessary data due to slow dataloader
+def Cleanup(data_list, entries):
+    for data in data_list:
+        for entry in entries:
+            try:
+                delattr(data, entry)
+            except Exception:
+                pass
+
+
+##Get min/max ranges for normalized edges
+def GetRanges(dataset, descriptor_label):
+    mean = 0.0
+    std = 0.0
+    for index in range(0, len(dataset)):
+        if len(dataset[index].edge_descriptor[descriptor_label]) > 0:
+            if index == 0:
+                feature_max = dataset[index].edge_descriptor[descriptor_label].max()
+                feature_min = dataset[index].edge_descriptor[descriptor_label].min()
+            mean += dataset[index].edge_descriptor[descriptor_label].mean()
+            std += dataset[index].edge_descriptor[descriptor_label].std()
+            if dataset[index].edge_descriptor[descriptor_label].max() > feature_max:
+                feature_max = dataset[index].edge_descriptor[descriptor_label].max()
+            if dataset[index].edge_descriptor[descriptor_label].min() < feature_min:
+                feature_min = dataset[index].edge_descriptor[descriptor_label].min()
+
+    mean = mean / len(dataset)
+    std = std / len(dataset)
+    return mean, std, feature_min, feature_max
+
+
+##Normalizes edges
+def NormalizeEdge(dataset, descriptor_label):
+    mean, std, feature_min, feature_max = GetRanges(dataset, descriptor_label)
+
+    for data in dataset:
+        data.edge_descriptor[descriptor_label] = (
+            data.edge_descriptor[descriptor_label] - feature_min
+        ) / (feature_max - feature_min)
+
+
+# WIP
+def SM_Edge(dataset):
+    from dscribe.descriptors import (
+        CoulombMatrix,
+        SOAP,
+        MBTR,
+        EwaldSumMatrix,
+        SineMatrix,
+    )
+
+    count = 0
+    for data in dataset:
+        n_atoms_max = len(data.ase)
+        make_feature_SM = SineMatrix(
+            n_atoms_max=n_atoms_max,
+            permutation="none",
+            sparse=False,
+            flatten=False,
+        )
+        features_SM = make_feature_SM.create(data.ase)
+        features_SM_trimmed = np.where(data.mask == 0, data.mask, features_SM)
+        features_SM_trimmed = torch.Tensor(features_SM_trimmed)
+        out = dense_to_sparse(features_SM_trimmed)
+        edge_index = out[0]
+        edge_weight = out[1]
+        data.edge_descriptor["SM"] = edge_weight
+
+        if count % 500 == 0:
+            print("SM data processed: ", count)
+        count = count + 1
+
+    return dataset
+
+
+################################################################################
+#  Transforms
+################################################################################
+
+##Get specified y index from data.y
+class GetY(object):
+    def __init__(self, index=0):
+        self.index = index
+
+    def __call__(self, data):
+        # Specify target.
+        if self.index != -1:
+            data.y = data.y[0][self.index]
+        return data

matdeeplearn/training/__init__.py

@@ -0,0 +1 @@
+from .training import *

matdeeplearn/training/training.py

@@ -0,0 +1,1290 @@
+##General imports
+import csv
+import os
+import time
+from datetime import datetime
+import shutil
+import copy
+import numpy as np
+from functools import partial
+import platform
+
+##Torch imports
+import torch.nn.functional as F
+import torch
+from torch_geometric.data import DataLoader, Dataset
+from torch_geometric.nn import DataParallel
+import torch_geometric.transforms as T
+from torch.utils.data.distributed import DistributedSampler
+from torch.nn.parallel import DistributedDataParallel
+import torch.distributed as dist
+import torch.multiprocessing as mp
+
+##Matdeeplearn imports
+from matdeeplearn import models
+import matdeeplearn.process as process
+import matdeeplearn.training as training
+from matdeeplearn.models.utils import model_summary
+
+################################################################################
+#  Training functions
+################################################################################
+
+##Train step, runs model in train mode
+def train(model, optimizer, loader, loss_method, rank):
+    model.train()
+    loss_all = 0
+    count = 0
+    for data in loader:
+        data = data.to(rank)
+        optimizer.zero_grad()
+        output = model(data)
+        # print(data.y.shape, output.shape)
+        loss = getattr(F, loss_method)(output, data.y)
+        loss.backward()
+        loss_all += loss.detach() * output.size(0)
+
+        # clip = 10
+        # torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
+
+        optimizer.step()
+        count = count + output.size(0)
+
+    loss_all = loss_all / count
+    return loss_all
+
+
+##Evaluation step, runs model in eval mode
+def evaluate(loader, model, loss_method, rank, out=False):
+    model.eval()
+    loss_all = 0
+    count = 0
+    for data in loader:
+        data = data.to(rank)
+        with torch.no_grad():
+            output = model(data)
+            loss = getattr(F, loss_method)(output, data.y)
+            loss_all += loss * output.size(0)
+            if out == True:
+                if count == 0:
+                    ids = [item for sublist in data.structure_id for item in sublist]
+                    ids = [item for sublist in ids for item in sublist]
+                    predict = output.data.cpu().numpy()
+                    target = data.y.cpu().numpy()
+                else:
+                    ids_temp = [
+                        item for sublist in data.structure_id for item in sublist
+                    ]
+                    ids_temp = [item for sublist in ids_temp for item in sublist]
+                    ids = ids + ids_temp
+                    predict = np.concatenate(
+                        (predict, output.data.cpu().numpy()), axis=0
+                    )
+                    target = np.concatenate((target, data.y.cpu().numpy()), axis=0)
+            count = count + output.size(0)
+
+    loss_all = loss_all / count
+
+    if out == True:
+        test_out = np.column_stack((ids, target, predict))
+        return loss_all, test_out
+    elif out == False:
+        return loss_all
+
+
+##Model trainer
+def trainer(
+    rank,
+    world_size,
+    model,
+    optimizer,
+    scheduler,
+    loss,
+    train_loader,
+    val_loader,
+    train_sampler,
+    epochs,
+    verbosity,
+    filename = "my_model_temp.pth",
+):
+
+    train_error = val_error = test_error = epoch_time = float("NaN")
+    train_start = time.time()
+    best_val_error = 1e10
+    model_best = model
+    ##Start training over epochs loop
+    for epoch in range(1, epochs + 1):
+
+        lr = scheduler.optimizer.param_groups[0]["lr"]
+        if rank not in ("cpu", "cuda"):
+            train_sampler.set_epoch(epoch)
+        ##Train model
+        train_error = train(model, optimizer, train_loader, loss, rank=rank)
+        if rank not in ("cpu", "cuda"):
+            torch.distributed.reduce(train_error, dst=0)
+            train_error = train_error / world_size
+
+        ##Get validation performance
+        if rank not in ("cpu", "cuda"):
+            dist.barrier()
+        if val_loader != None and rank in (0, "cpu", "cuda"):
+            if rank not in ("cpu", "cuda"):
+                val_error = evaluate(
+                    val_loader, model.module, loss, rank=rank, out=False
+                )
+            else:
+                val_error = evaluate(val_loader, model, loss, rank=rank, out=False)
+
+        ##Train loop timings
+        epoch_time = time.time() - train_start
+        train_start = time.time()
+
+        ##remember the best val error and save model and checkpoint        
+        if val_loader != None and rank in (0, "cpu", "cuda"):
+            if val_error == float("NaN") or val_error < best_val_error:
+                if rank not in ("cpu", "cuda"):
+                    model_best = copy.deepcopy(model.module)
+                    torch.save(
+                        {
+                            "state_dict": model.state_dict(),
+                            "optimizer_state_dict": optimizer.state_dict(),
+                            "scheduler_state_dict": scheduler.state_dict(),
+                            "full_model": model,
+                        },
+                        filename,
+                    )
+                else:
+                    model_best = copy.deepcopy(model)
+                    torch.save(
+                        {
+                            "state_dict": model.state_dict(),
+                            "optimizer_state_dict": optimizer.state_dict(),
+                            "scheduler_state_dict": scheduler.state_dict(),
+                            "full_model": model,
+                        },
+                        filename,
+                    )
+            best_val_error = min(val_error, best_val_error)
+        elif val_loader == None and rank in (0, "cpu", "cuda"):
+            if rank not in ("cpu", "cuda"):
+                model_best = copy.deepcopy(model.module)
+                torch.save(
+                    {
+                        "state_dict": model.state_dict(),
+                        "optimizer_state_dict": optimizer.state_dict(),
+                        "scheduler_state_dict": scheduler.state_dict(),
+                        "full_model": model,
+                    },
+                    filename,
+                )
+            else:
+                model_best = copy.deepcopy(model)
+                torch.save(
+                    {
+                        "state_dict": model.state_dict(),
+                        "optimizer_state_dict": optimizer.state_dict(),
+                        "scheduler_state_dict": scheduler.state_dict(),
+                        "full_model": model,
+                    },
+                    filename,
+                )
+
+        ##scheduler on train error
+        scheduler.step(train_error)
+
+        ##Print performance
+        if epoch % verbosity == 0:
+            if rank in (0, "cpu", "cuda"):
+                print(
+                    "Epoch: {:04d}, Learning Rate: {:.6f}, Training Error: {:.5f}, Val Error: {:.5f}, Time per epoch (s): {:.5f}".format(
+                        epoch, lr, train_error, val_error, epoch_time
+                    )
+                )
+
+    if rank not in ("cpu", "cuda"):
+        dist.barrier()
+
+    return model_best
+
+
+##Write results to csv file
+def write_results(output, filename):
+    shape = output.shape
+    with open(filename, "w") as f:
+        csvwriter = csv.writer(f)
+        for i in range(0, len(output)):
+            if i == 0:
+                csvwriter.writerow(
+                    ["ids"]
+                    + ["target"] * int((shape[1] - 1) / 2)
+                    + ["prediction"] * int((shape[1] - 1) / 2)
+                )
+            elif i > 0:
+                csvwriter.writerow(output[i - 1, :])
+
+
+##Pytorch ddp setup
+def ddp_setup(rank, world_size):
+    if rank in ("cpu", "cuda"):
+        return
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = "12355"
+    if platform.system() == 'Windows':
+        dist.init_process_group("gloo", rank=rank, world_size=world_size)    
+    else:
+        dist.init_process_group("nccl", rank=rank, world_size=world_size)
+    torch.backends.cudnn.enabled = False
+    torch.backends.cudnn.benchmark = True
+
+
+##Pytorch model setup
+def model_setup(
+    rank,
+    model_name,
+    model_params,
+    dataset,
+    load_model=False,
+    model_path=None,
+    print_model=True,
+):
+    model = getattr(models, model_name)(
+        data=dataset, **(model_params if model_params is not None else {})
+    ).to(rank)
+    if load_model == "True":
+        assert os.path.exists(model_path), "Saved model not found"
+        if str(rank) in ("cpu"):
+            saved = torch.load(model_path, map_location=torch.device("cpu"))
+        else:
+            saved = torch.load(model_path)
+        model.load_state_dict(saved["model_state_dict"])
+        # optimizer.load_state_dict(saved['optimizer_state_dict'])
+
+    # DDP
+    if rank not in ("cpu", "cuda"):
+        model = DistributedDataParallel(
+            model, device_ids=[rank], find_unused_parameters=True
+        )
+        # model = DistributedDataParallel(model, device_ids=[rank], find_unused_parameters=False)
+    if print_model == True and rank in (0, "cpu", "cuda"):
+        model_summary(model)
+    return model
+
+
+##Pytorch loader setup
+def loader_setup(
+    train_ratio,
+    val_ratio,
+    test_ratio,
+    batch_size,
+    dataset,
+    rank,
+    seed,
+    world_size=0,
+    num_workers=0,
+):
+    ##Split datasets
+    train_dataset, val_dataset, test_dataset = process.split_data(
+        dataset, train_ratio, val_ratio, test_ratio, seed
+    )
+
+    ##DDP
+    if rank not in ("cpu", "cuda"):
+        train_sampler = DistributedSampler(
+            train_dataset, num_replicas=world_size, rank=rank
+        )
+    elif rank in ("cpu", "cuda"):
+        train_sampler = None
+
+    ##Load data
+    train_loader = val_loader = test_loader = None
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size=batch_size,
+        shuffle=(train_sampler is None),
+        num_workers=num_workers,
+        pin_memory=True,
+        sampler=train_sampler,
+    )
+    # may scale down batch size if memory is an issue
+    if rank in (0, "cpu", "cuda"):
+        if len(val_dataset) > 0:
+            val_loader = DataLoader(
+                val_dataset,
+                batch_size=batch_size,
+                shuffle=False,
+                num_workers=num_workers,
+                pin_memory=True,
+            )
+        if len(test_dataset) > 0:
+            test_loader = DataLoader(
+                test_dataset,
+                batch_size=batch_size,
+                shuffle=False,
+                num_workers=num_workers,
+                pin_memory=True,
+            )
+    return (
+        train_loader,
+        val_loader,
+        test_loader,
+        train_sampler,
+        train_dataset,
+        val_dataset,
+        test_dataset,
+    )
+
+
+def loader_setup_CV(index, batch_size, dataset, rank, world_size=0, num_workers=0):
+    ##Split datasets
+    train_dataset = [x for i, x in enumerate(dataset) if i != index]
+    train_dataset = torch.utils.data.ConcatDataset(train_dataset)
+    test_dataset = dataset[index]
+
+    ##DDP
+    if rank not in ("cpu", "cuda"):
+        train_sampler = DistributedSampler(
+            train_dataset, num_replicas=world_size, rank=rank
+        )
+    elif rank in ("cpu", "cuda"):
+        train_sampler = None
+
+    train_loader = val_loader = test_loader = None
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size=batch_size,
+        shuffle=(train_sampler is None),
+        num_workers=num_workers,
+        pin_memory=True,
+        sampler=train_sampler,
+    )
+    if rank in (0, "cpu", "cuda"):
+        test_loader = DataLoader(
+            test_dataset,
+            batch_size=batch_size,
+            shuffle=False,
+            num_workers=num_workers,
+            pin_memory=True,
+        )
+
+    return train_loader, test_loader, train_sampler, train_dataset, test_dataset
+
+
+################################################################################
+#  Trainers
+################################################################################
+
+###Regular training with train, val, test split
+def train_regular(
+    rank,
+    world_size,
+    data_path,
+    job_parameters=None,
+    training_parameters=None,
+    model_parameters=None,
+):
+    ##DDP
+    ddp_setup(rank, world_size)
+    ##some issues with DDP learning rate
+    if rank not in ("cpu", "cuda"):
+        model_parameters["lr"] = model_parameters["lr"] * world_size
+
+    ##Get dataset
+    dataset = process.get_dataset(data_path, training_parameters["target_index"], False)
+
+    if rank not in ("cpu", "cuda"):
+        dist.barrier()
+
+    ##Set up loader
+    (
+        train_loader,
+        val_loader,
+        test_loader,
+        train_sampler,
+        train_dataset,
+        _,
+        _,
+    ) = loader_setup(
+        training_parameters["train_ratio"],
+        training_parameters["val_ratio"],
+        training_parameters["test_ratio"],
+        model_parameters["batch_size"],
+        dataset,
+        rank,
+        job_parameters["seed"],
+        world_size,
+    )
+
+    ##Set up model
+    model = model_setup(
+        rank,
+        model_parameters["model"],
+        model_parameters,
+        dataset,
+        job_parameters["load_model"],
+        job_parameters["model_path"],
+        model_parameters.get("print_model", True),
+    )
+
+    ##Set-up optimizer & scheduler
+    optimizer = getattr(torch.optim, model_parameters["optimizer"])(
+        model.parameters(),
+        lr=model_parameters["lr"],
+        **model_parameters["optimizer_args"]
+    )
+    scheduler = getattr(torch.optim.lr_scheduler, model_parameters["scheduler"])(
+        optimizer, **model_parameters["scheduler_args"]
+    )
+
+    ##Start training
+    model = trainer(
+        rank,
+        world_size,
+        model,
+        optimizer,
+        scheduler,
+        training_parameters["loss"],
+        train_loader,
+        val_loader,
+        train_sampler,
+        model_parameters["epochs"],
+        training_parameters["verbosity"],
+        "my_model_temp.pth",
+    )
+
+    if rank in (0, "cpu", "cuda"):
+
+        train_error = val_error = test_error = float("NaN")
+
+        ##workaround to get training output in DDP mode
+        ##outputs are slightly different, could be due to dropout or batchnorm?
+        train_loader = DataLoader(
+            train_dataset,
+            batch_size=model_parameters["batch_size"],
+            shuffle=False,
+            num_workers=0,
+            pin_memory=True,
+        )
+
+        ##Get train error in eval mode
+        train_error, train_out = evaluate(
+            train_loader, model, training_parameters["loss"], rank, out=True
+        )
+        print("Train Error: {:.5f}".format(train_error))
+
+        ##Get val error
+        if val_loader != None:
+            val_error, val_out = evaluate(
+                val_loader, model, training_parameters["loss"], rank, out=True
+            )
+            print("Val Error: {:.5f}".format(val_error))
+
+        ##Get test error
+        if test_loader != None:
+            test_error, test_out = evaluate(
+                test_loader, model, training_parameters["loss"], rank, out=True
+            )
+            print("Test Error: {:.5f}".format(test_error))
+
+        ##Save model
+        if job_parameters["save_model"] == "True":
+
+            if rank not in ("cpu", "cuda"):
+                torch.save(
+                    {
+                        "model_state_dict": model.state_dict(),
+                        "optimizer_state_dict": optimizer.state_dict(),
+                        "scheduler_state_dict": scheduler.state_dict(),
+                        "full_model": model,
+                    },
+                    job_parameters["model_path"],
+                )
+            else:
+                torch.save(
+                    {
+                        "model_state_dict": model.state_dict(),
+                        "optimizer_state_dict": optimizer.state_dict(),
+                        "scheduler_state_dict": scheduler.state_dict(),
+                        "full_model": model,
+                    },
+                    job_parameters["model_path"],
+                )
+
+        ##Write outputs
+        if job_parameters["write_output"] == "True":
+
+            write_results(
+                train_out, str(job_parameters["job_name"]) + "_train_outputs.csv"
+            )
+            if val_loader != None:
+                write_results(
+                    val_out, str(job_parameters["job_name"]) + "_val_outputs.csv"
+                )
+            if test_loader != None:
+                write_results(
+                    test_out, str(job_parameters["job_name"]) + "_test_outputs.csv"
+                )
+
+        if rank not in ("cpu", "cuda"):
+            dist.destroy_process_group()
+
+        ##Write out model performance to file
+        error_values = np.array((train_error.cpu(), val_error.cpu(), test_error.cpu()))
+        if job_parameters.get("write_error") == "True":
+            np.savetxt(
+                job_parameters["job_name"] + "_errorvalues.csv",
+                error_values[np.newaxis, ...],
+                delimiter=",",
+            )
+
+        return error_values
+
+
+###Predict using a saved movel
+def predict(dataset, loss, job_parameters=None):
+
+    rank = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    ##Loads predict dataset in one go, care needed for large datasets)
+    loader = DataLoader(
+        dataset,
+        batch_size=128,
+        shuffle=False,
+        num_workers=0,
+        pin_memory=True,
+    )
+
+    ##Load saved model
+    assert os.path.exists(job_parameters["model_path"]), "Saved model not found"
+    if str(rank) == "cpu":
+        saved = torch.load(
+            job_parameters["model_path"], map_location=torch.device("cpu")
+        )
+    else:
+        saved = torch.load(
+            job_parameters["model_path"], map_location=torch.device("cuda")
+        )
+    model = saved["full_model"]
+    model = model.to(rank)
+    model_summary(model)
+
+    ##Get predictions
+    time_start = time.time()
+    test_error, test_out = evaluate(loader, model, loss, rank, out=True)
+    elapsed_time = time.time() - time_start
+
+    print("Evaluation time (s): {:.5f}".format(elapsed_time))
+
+    ##Write output
+    if job_parameters["write_output"] == "True":
+        write_results(
+            test_out, str(job_parameters["job_name"]) + "_predicted_outputs.csv"
+        )
+
+    return test_error
+
+
+###n-fold cross validation
+def train_CV(
+    rank,
+    world_size,
+    data_path,
+    job_parameters=None,
+    training_parameters=None,
+    model_parameters=None,
+):
+
+    job_parameters["load_model"] = "False"
+    job_parameters["save_model"] = "False"
+    job_parameters["model_path"] = None
+    ##DDP
+    ddp_setup(rank, world_size)
+    ##some issues with DDP learning rate
+    if rank not in ("cpu", "cuda"):
+        model_parameters["lr"] = model_parameters["lr"] * world_size
+
+    ##Get dataset
+    dataset = process.get_dataset(data_path, training_parameters["target_index"], False)
+
+    ##Split datasets
+    cv_dataset = process.split_data_CV(
+        dataset, num_folds=job_parameters["cv_folds"], seed=job_parameters["seed"]
+    )
+    cv_error = 0
+
+    for index in range(0, len(cv_dataset)):
+
+        ##Set up model
+        if index == 0:
+            model = model_setup(
+                rank,
+                model_parameters["model"],
+                model_parameters,
+                dataset,
+                job_parameters["load_model"],
+                job_parameters["model_path"],
+                print_model=True,
+            )
+        else:
+            model = model_setup(
+                rank,
+                model_parameters["model"],
+                model_parameters,
+                dataset,
+                job_parameters["load_model"],
+                job_parameters["model_path"],
+                print_model=False,
+            )
+
+        ##Set-up optimizer & scheduler
+        optimizer = getattr(torch.optim, model_parameters["optimizer"])(
+            model.parameters(),
+            lr=model_parameters["lr"],
+            **model_parameters["optimizer_args"]
+        )
+        scheduler = getattr(torch.optim.lr_scheduler, model_parameters["scheduler"])(
+            optimizer, **model_parameters["scheduler_args"]
+        )
+
+        ##Set up loader
+        train_loader, test_loader, train_sampler, train_dataset, _ = loader_setup_CV(
+            index, model_parameters["batch_size"], cv_dataset, rank, world_size
+        )
+
+        ##Start training
+        model = trainer(
+            rank,
+            world_size,
+            model,
+            optimizer,
+            scheduler,
+            training_parameters["loss"],
+            train_loader,
+            None,
+            train_sampler,
+            model_parameters["epochs"],
+            training_parameters["verbosity"],
+            "my_model_temp.pth",
+        )
+
+        if rank not in ("cpu", "cuda"):
+            dist.barrier()
+
+        if rank in (0, "cpu", "cuda"):
+
+            train_loader = DataLoader(
+                train_dataset,
+                batch_size=model_parameters["batch_size"],
+                shuffle=False,
+                num_workers=0,
+                pin_memory=True,
+            )
+
+            ##Get train error
+            train_error, train_out = evaluate(
+                train_loader, model, training_parameters["loss"], rank, out=True
+            )
+            print("Train Error: {:.5f}".format(train_error))
+
+            ##Get test error
+            test_error, test_out = evaluate(
+                test_loader, model, training_parameters["loss"], rank, out=True
+            )
+            print("Test Error: {:.5f}".format(test_error))
+
+            cv_error = cv_error + test_error
+
+            if index == 0:
+                total_rows = test_out
+            else:
+                total_rows = np.vstack((total_rows, test_out))
+
+    ##Write output
+    if rank in (0, "cpu", "cuda"):
+        if job_parameters["write_output"] == "True":
+            if test_loader != None:
+                write_results(
+                    total_rows, str(job_parameters["job_name"]) + "_CV_outputs.csv"
+                )
+
+        cv_error = cv_error / len(cv_dataset)
+        print("CV Error: {:.5f}".format(cv_error))
+
+    if rank not in ("cpu", "cuda"):
+        dist.destroy_process_group()
+
+    return cv_error
+
+
+### Repeat training for n times
+def train_repeat(
+    data_path,
+    job_parameters=None,
+    training_parameters=None,
+    model_parameters=None,
+):
+
+    world_size = torch.cuda.device_count()
+    job_name = job_parameters["job_name"]
+    model_path = job_parameters["model_path"]
+    job_parameters["write_error"] = "True"
+    job_parameters["load_model"] = "False"
+    job_parameters["save_model"] = "False"
+    ##Loop over number of repeated trials
+    for i in range(0, job_parameters["repeat_trials"]):
+
+        ##new seed each time for different data split
+        job_parameters["seed"] = np.random.randint(1, 1e6)
+
+        if i == 0:
+            model_parameters["print_model"] = True
+        else:
+            model_parameters["print_model"] = False
+
+        job_parameters["job_name"] = job_name + str(i)
+        job_parameters["model_path"] = str(i) + "_" + model_path
+
+        if world_size == 0:
+            print("Running on CPU - this will be slow")
+            training.train_regular(
+                "cpu",
+                world_size,
+                data_path,
+                job_parameters,
+                training_parameters,
+                model_parameters,
+            )
+        elif world_size > 0:
+            if job_parameters["parallel"] == "True":
+                print("Running on", world_size, "GPUs")
+                mp.spawn(
+                    training.train_regular,
+                    args=(
+                        world_size,
+                        data_path,
+                        job_parameters,
+                        training_parameters,
+                        model_parameters,
+                    ),
+                    nprocs=world_size,
+                    join=True,
+                )
+            if job_parameters["parallel"] == "False":
+                print("Running on one GPU")
+                training.train_regular(
+                    "cuda",
+                    world_size,
+                    data_path,
+                    job_parameters,
+                    training_parameters,
+                    model_parameters,
+                )
+
+    ##Compile error metrics from individual trials
+    print("Individual training finished.")
+    print("Compiling metrics from individual trials...")
+    error_values = np.zeros((job_parameters["repeat_trials"], 3))
+    for i in range(0, job_parameters["repeat_trials"]):
+        filename = job_name + str(i) + "_errorvalues.csv"
+        error_values[i] = np.genfromtxt(filename, delimiter=",")
+    mean_values = [
+        np.mean(error_values[:, 0]),
+        np.mean(error_values[:, 1]),
+        np.mean(error_values[:, 2]),
+    ]
+    std_values = [
+        np.std(error_values[:, 0]),
+        np.std(error_values[:, 1]),
+        np.std(error_values[:, 2]),
+    ]
+
+    ##Print error
+    print(
+        "Training Error Avg: {:.3f}, Training Standard Dev: {:.3f}".format(
+            mean_values[0], std_values[0]
+        )
+    )
+    print(
+        "Val Error Avg: {:.3f}, Val Standard Dev: {:.3f}".format(
+            mean_values[1], std_values[1]
+        )
+    )
+    print(
+        "Test Error Avg: {:.3f}, Test Standard Dev: {:.3f}".format(
+            mean_values[2], std_values[2]
+        )
+    )
+
+    ##Write error metrics
+    if job_parameters["write_output"] == "True":
+        with open(job_name + "_all_errorvalues.csv", "w") as f:
+            csvwriter = csv.writer(f)
+            csvwriter.writerow(
+                [
+                    "",
+                    "Training",
+                    "Validation",
+                    "Test",
+                ]
+            )
+            for i in range(0, len(error_values)):
+                csvwriter.writerow(
+                    [
+                        "Trial " + str(i),
+                        error_values[i, 0],
+                        error_values[i, 1],
+                        error_values[i, 2],
+                    ]
+                )
+            csvwriter.writerow(["Mean", mean_values[0], mean_values[1], mean_values[2]])
+            csvwriter.writerow(["Std", std_values[0], std_values[1], std_values[2]])
+    elif job_parameters["write_output"] == "False":
+        for i in range(0, job_parameters["repeat_trials"]):
+            filename = job_name + str(i) + "_errorvalues.csv"
+            os.remove(filename)
+
+
+###Hyperparameter optimization
+# trainable function for ray tune (no parallel, max 1 GPU per job)
+def tune_trainable(config, checkpoint_dir=None, data_path=None):
+
+    # imports
+    from ray import tune
+
+    print("Hyperparameter trial start")
+    hyper_args = config["hyper_args"]
+    job_parameters = config["job_parameters"]
+    processing_parameters = config["processing_parameters"]
+    training_parameters = config["training_parameters"]
+    model_parameters = config["model_parameters"]
+
+    ##Merge hyperparameter parameters with constant parameters, with precedence over hyperparameter ones
+    ##Omit training and job parameters as they should not be part of hyperparameter opt, in theory
+    model_parameters = {**model_parameters, **hyper_args}
+    processing_parameters = {**processing_parameters, **hyper_args}
+
+    ##Assume 1 gpu or 1 cpu per trial, no functionality for parallel yet
+    world_size = 1
+    rank = "cpu"
+    if torch.cuda.is_available():
+        rank = "cuda"
+
+    ##Reprocess data in a separate directory to prevent conflict
+    if job_parameters["reprocess"] == "True":
+        time = datetime.now()
+        processing_parameters["processed_path"] = time.strftime("%H%M%S%f")
+        processing_parameters["verbose"] = "False"
+    data_path = os.path.dirname(
+        os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+    )
+    data_path = os.path.join(data_path, processing_parameters["data_path"])
+    data_path = os.path.normpath(data_path)
+    print("Data path", data_path)
+
+    ##Set up dataset
+    dataset = process.get_dataset(
+        data_path,
+        training_parameters["target_index"],
+        job_parameters["reprocess"],
+        processing_parameters,
+    )
+
+    ##Set up loader
+    (
+        train_loader,
+        val_loader,
+        test_loader,
+        train_sampler,
+        train_dataset,
+        _,
+        _,
+    ) = loader_setup(
+        training_parameters["train_ratio"],
+        training_parameters["val_ratio"],
+        training_parameters["test_ratio"],
+        model_parameters["batch_size"],
+        dataset,
+        rank,
+        job_parameters["seed"],
+        world_size,
+    )
+
+    ##Set up model
+    model = model_setup(
+        rank,
+        model_parameters["model"],
+        model_parameters,
+        dataset,
+        False,
+        None,
+        False,
+    )
+
+    ##Set-up optimizer & scheduler
+    optimizer = getattr(torch.optim, model_parameters["optimizer"])(
+        model.parameters(),
+        lr=model_parameters["lr"],
+        **model_parameters["optimizer_args"]
+    )
+    scheduler = getattr(torch.optim.lr_scheduler, model_parameters["scheduler"])(
+        optimizer, **model_parameters["scheduler_args"]
+    )
+
+    ##Load checkpoint
+    if checkpoint_dir:
+        model_state, optimizer_state, scheduler_state = torch.load(
+            os.path.join(checkpoint_dir, "checkpoint")
+        )
+        model.load_state_dict(model_state)
+        optimizer.load_state_dict(optimizer_state)
+        scheduler.load_state_dict(scheduler_state)
+
+    ##Training loop
+    for epoch in range(1, model_parameters["epochs"] + 1):
+        lr = scheduler.optimizer.param_groups[0]["lr"]
+        train_error = train(
+            model, optimizer, train_loader, training_parameters["loss"], rank=rank
+        )
+
+        val_error = evaluate(
+            val_loader, model, training_parameters["loss"], rank=rank, out=False
+        )
+
+        ##Delete processed data
+        if epoch == model_parameters["epochs"]:
+            if (
+                job_parameters["reprocess"] == "True"
+                and job_parameters["hyper_delete_processed"] == "True"
+            ):
+                shutil.rmtree(
+                    os.path.join(data_path, processing_parameters["processed_path"])
+                )
+            print("Finished Training")
+
+        ##Update to tune
+        if epoch % job_parameters["hyper_iter"] == 0:
+            with tune.checkpoint_dir(step=epoch) as checkpoint_dir:
+                path = os.path.join(checkpoint_dir, "checkpoint")
+                torch.save(
+                    (
+                        model.state_dict(),
+                        optimizer.state_dict(),
+                        scheduler.state_dict(),
+                    ),
+                    path,
+                )
+            ##Somehow tune does not recognize value without *1
+            tune.report(loss=val_error.cpu().numpy() * 1)
+            # tune.report(loss=val_error)
+
+
+# Tune setup
+def tune_setup(
+    hyper_args,
+    job_parameters,
+    processing_parameters,
+    training_parameters,
+    model_parameters,
+):
+
+    # imports
+    import ray
+    from ray import tune
+    from ray.tune.schedulers import ASHAScheduler
+    from ray.tune.suggest.hyperopt import HyperOptSearch
+    from ray.tune.suggest import ConcurrencyLimiter
+    from ray.tune import CLIReporter
+
+    ray.init()
+    data_path = "_"
+    local_dir = "ray_results"
+    # currently no support for paralleization per trial
+    gpus_per_trial = 1
+
+    ##Set up search algo
+    search_algo = HyperOptSearch(metric="loss", mode="min", n_initial_points=5)
+    search_algo = ConcurrencyLimiter(
+        search_algo, max_concurrent=job_parameters["hyper_concurrency"]
+    )
+
+    ##Resume run
+    if os.path.exists(local_dir + "/" + job_parameters["job_name"]) and os.path.isdir(
+        local_dir + "/" + job_parameters["job_name"]
+    ):
+        if job_parameters["hyper_resume"] == "False":
+            resume = False
+        elif job_parameters["hyper_resume"] == "True":
+            resume = True
+        # else:
+        #    resume = "PROMPT"
+    else:
+        resume = False
+
+    ##Print out hyperparameters
+    parameter_columns = [
+        element for element in hyper_args.keys() if element not in "global"
+    ]
+    parameter_columns = ["hyper_args"]
+    reporter = CLIReporter(
+        max_progress_rows=20, max_error_rows=5, parameter_columns=parameter_columns
+    )
+
+    ##Run tune
+    tune_result = tune.run(
+        partial(tune_trainable, data_path=data_path),
+        resources_per_trial={"cpu": 1, "gpu": gpus_per_trial},
+        config={
+            "hyper_args": hyper_args,
+            "job_parameters": job_parameters,
+            "processing_parameters": processing_parameters,
+            "training_parameters": training_parameters,
+            "model_parameters": model_parameters,
+        },
+        num_samples=job_parameters["hyper_trials"],
+        # scheduler=scheduler,
+        search_alg=search_algo,
+        local_dir=local_dir,
+        progress_reporter=reporter,
+        verbose=job_parameters["hyper_verbosity"],
+        resume=resume,
+        log_to_file=True,
+        name=job_parameters["job_name"],
+        max_failures=4,
+        raise_on_failed_trial=False,
+        # keep_checkpoints_num=job_parameters["hyper_keep_checkpoints_num"],
+        # checkpoint_score_attr="min-loss",
+        stop={
+            "training_iteration": model_parameters["epochs"]
+            // job_parameters["hyper_iter"]
+        },
+    )
+
+    ##Get best trial
+    best_trial = tune_result.get_best_trial("loss", "min", "all")
+    # best_trial = tune_result.get_best_trial("loss", "min", "last")
+
+    return best_trial
+
+
+###Simple ensemble using averages
+def train_ensemble(
+    data_path,
+    job_parameters=None,
+    training_parameters=None,
+    model_parameters=None,
+):
+
+    world_size = torch.cuda.device_count()
+    job_name = job_parameters["job_name"]
+    write_output = job_parameters["write_output"]
+    model_path = job_parameters["model_path"]
+    job_parameters["write_error"] = "True"
+    job_parameters["write_output"] = "True"
+    job_parameters["load_model"] = "False"
+    ##Loop over number of repeated trials
+    for i in range(0, len(job_parameters["ensemble_list"])):
+        job_parameters["job_name"] = job_name + str(i)
+        job_parameters["model_path"] = (
+            str(i) + "_" + job_parameters["ensemble_list"][i] + "_" + model_path
+        )
+
+        if world_size == 0:
+            print("Running on CPU - this will be slow")
+            training.train_regular(
+                "cpu",
+                world_size,
+                data_path,
+                job_parameters,
+                training_parameters,
+                model_parameters[job_parameters["ensemble_list"][i]],
+            )
+        elif world_size > 0:
+            if job_parameters["parallel"] == "True":
+                print("Running on", world_size, "GPUs")
+                mp.spawn(
+                    training.train_regular,
+                    args=(
+                        world_size,
+                        data_path,
+                        job_parameters,
+                        training_parameters,
+                        model_parameters[job_parameters["ensemble_list"][i]],
+                    ),
+                    nprocs=world_size,
+                    join=True,
+                )
+            if job_parameters["parallel"] == "False":
+                print("Running on one GPU")
+                training.train_regular(
+                    "cuda",
+                    world_size,
+                    data_path,
+                    job_parameters,
+                    training_parameters,
+                    model_parameters[job_parameters["ensemble_list"][i]],
+                )
+
+    ##Compile error metrics from individual models
+    print("Individual training finished.")
+    print("Compiling metrics from individual models...")
+    error_values = np.zeros((len(job_parameters["ensemble_list"]), 3))
+    for i in range(0, len(job_parameters["ensemble_list"])):
+        filename = job_name + str(i) + "_errorvalues.csv"
+        error_values[i] = np.genfromtxt(filename, delimiter=",")
+    mean_values = [
+        np.mean(error_values[:, 0]),
+        np.mean(error_values[:, 1]),
+        np.mean(error_values[:, 2]),
+    ]
+    std_values = [
+        np.std(error_values[:, 0]),
+        np.std(error_values[:, 1]),
+        np.std(error_values[:, 2]),
+    ]
+
+    # average ensembling, takes the mean of the predictions
+    for i in range(0, len(job_parameters["ensemble_list"])):
+        filename = job_name + str(i) + "_test_outputs.csv"
+        test_out = np.genfromtxt(filename, delimiter=",", skip_header=1)
+        if i == 0:
+            test_total = test_out
+        elif i > 0:
+            test_total = np.column_stack((test_total, test_out[:, 2]))
+
+    ensemble_test = np.mean(np.array(test_total[:, 2:]).astype(np.float), axis=1)
+    ensemble_test_error = getattr(F, training_parameters["loss"])(
+        torch.tensor(ensemble_test),
+        torch.tensor(test_total[:, 1].astype(np.float)),
+    )
+    test_total = np.column_stack((test_total, ensemble_test))
+    
+    ##Print performance
+    for i in range(0, len(job_parameters["ensemble_list"])):
+        print(
+            job_parameters["ensemble_list"][i]
+            + " Test Error: {:.5f}".format(error_values[i, 2])
+        )
+    print(
+        "Test Error Avg: {:.3f}, Test Standard Dev: {:.3f}".format(
+            mean_values[2], std_values[2]
+        )
+    )
+    print("Ensemble Error: {:.5f}".format(ensemble_test_error))
+    
+    ##Write output
+    if write_output == "True" or write_output == "Partial":
+        with open(
+            str(job_name) + "_test_ensemble_outputs.csv", "w"
+        ) as f:
+            csvwriter = csv.writer(f)
+            for i in range(0, len(test_total) + 1):
+                if i == 0:
+                    csvwriter.writerow(
+                        [
+                            "ids",
+                            "target",
+                        ]
+                        + job_parameters["ensemble_list"]
+                        + ["ensemble"]
+                    )
+                elif i > 0:
+                    csvwriter.writerow(test_total[i - 1, :])
+    if write_output == "False" or write_output == "Partial":
+        for i in range(0, len(job_parameters["ensemble_list"])):
+            filename = job_name + str(i) + "_errorvalues.csv"
+            os.remove(filename)
+            filename = job_name + str(i) + "_test_outputs.csv"
+            os.remove(filename)
+
+##Obtains features from graph in a trained model and analysis with tsne
+def analysis(
+    dataset,
+    model_path,
+    tsne_args,
+):
+
+    # imports
+    from sklearn.decomposition import PCA
+    from sklearn.manifold import TSNE
+    import matplotlib.pyplot as plt
+
+    rank = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    inputs = []
+
+    def hook(module, input, output):
+        inputs.append(input)
+
+    assert os.path.exists(model_path), "saved model not found"
+    if str(rank) == "cpu":
+        saved = torch.load(model_path, map_location=torch.device("cpu"))
+    else:
+        saved = torch.load(model_path, map_location=torch.device("cuda"))
+    model = saved["full_model"]
+    model_summary(model)
+
+    print(dataset)
+
+    loader = DataLoader(
+        dataset,
+        batch_size=512,
+        shuffle=False,
+        num_workers=0,
+        pin_memory=True,
+    )
+
+    model.eval()
+    ##Grabs the input of the first linear layer after the GNN
+    model.post_lin_list[0].register_forward_hook(hook)
+    for data in loader:
+        with torch.no_grad():
+            data = data.to(rank)
+            output = model(data)
+
+    inputs = [i for sub in inputs for i in sub]
+    inputs = torch.cat(inputs)
+    inputs = inputs.cpu().numpy()
+    print("Number of samples: ", inputs.shape[0])
+    print("Number of features: ", inputs.shape[1])
+
+    # only works for when targets has one index
+    targets = dataset.data.y.numpy()
+
+    # pca = PCA(n_components=2)
+    # pca_out=pca.fit_transform(inputs)
+    # print(pca_out.shape)
+    # np.savetxt('pca.csv', pca_out, delimiter=',')
+    # plt.scatter(pca_out[:,1],pca_out[:,0],c=targets,s=15)
+    # plt.colorbar()
+    # plt.show()
+    # plt.clf()
+
+    ##Start t-SNE analysis
+    tsne = TSNE(**tsne_args)
+    tsne_out = tsne.fit_transform(inputs)
+    rows = zip(
+        dataset.data.structure_id,
+        list(dataset.data.y.numpy()),
+        list(tsne_out[:, 0]),
+        list(tsne_out[:, 1]),
+    )
+
+    with open("tsne_output.csv", "w") as csv_file:
+        writer = csv.writer(csv_file, delimiter=",")
+        for row in rows:
+            writer.writerow(row)
+
+    fig, ax = plt.subplots()
+    main = plt.scatter(tsne_out[:, 1], tsne_out[:, 0], c=targets, s=3)
+    ax.set_xticklabels([])
+    ax.set_yticklabels([])
+    ax.set_xticks([])
+    ax.set_yticks([])
+    cbar = plt.colorbar(main, ax=ax)
+    stdev = np.std(targets)
+    cbar.mappable.set_clim(
+        np.mean(targets) - 2 * np.std(targets), np.mean(targets) + 2 * np.std(targets)
+    )
+    # cbar.ax.tick_params(labelsize=50)
+    # cbar.ax.tick_params(size=40)
+    plt.savefig("tsne_output.png", format="png", dpi=600)
+    plt.show()