Best model using transformer-encoder layers with K562 data

K562/TE/K562-TE.config

@@ -0,0 +1,30 @@
+UseCustomLoss = 0
+Tasks = K562
+TaskWeights = 1
+Verbose = 2
+NumConvLayers = 4
+KernelSizes = 8,16,32,64
+NumKernels = 1024,512,256,128
+MaxTrain = 3000000
+MaxTest = 999999999
+ShouldTest=1
+Epochs = 200
+
+RevComp = 0
+BatchSize = 128
+EarlyStop = 10
+DropoutRate = 0.5
+LearningRate = 0.002
+ConvResidualSkip = 0
+ConvDropout = 1
+ConvPad = same
+DilationFactor = 1
+ConvPoolSize = 1
+GlobalAvePool = 1
+GlobalMaxPool = 0
+NumDense = 0
+DenseSizes = 0
+NumAttentionLayers = 1
+AttentionHeads = 8
+AttentionKeyDim = 64
+AttentionResidualSkip = 0

K562/TE/K562_TE-6.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c90486f1753a082c66751c8b09427601a5e3282a32b1366e0b162b7e9bdda79
+size 59285096

K562/TE/K562_TE-6.json

@@ -0,0 +1 @@
+{"class_name": "Functional", "config": {"name": "model", "trainable": true, "layers": [{"module": "keras.layers", "class_name": "InputLayer", "config": {"batch_input_shape": [null, 300, 4], "dtype": "float32", "sparse": false, "ragged": false, "name": "input_1"}, "registered_name": null, "name": "input_1", "inbound_nodes": []}, {"module": "keras.layers", "class_name": "Conv1D", "config": {"name": "conv1d", "trainable": true, "dtype": "float32", "filters": 1024, "kernel_size": [8], "strides": [1], "padding": "same", "data_format": "channels_last", "dilation_rate": [1], "groups": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 4]}, "name": "conv1d", "inbound_nodes": [[["input_1", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "BatchNormalization", "config": {"name": "batch_normalization", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "gamma_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "moving_mean_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "moving_variance_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 1024]}, "name": "batch_normalization", "inbound_nodes": [[["conv1d", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Activation", "config": {"name": "activation", "trainable": true, "dtype": "float32", "activation": "relu"}, "registered_name": null, "build_config": {"input_shape": [null, 300, 1024]}, "name": "activation", "inbound_nodes": [[["batch_normalization", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Dropout", "config": {"name": "dropout", "trainable": true, "dtype": "float32", "rate": 0.5, "noise_shape": null, "seed": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 1024]}, "name": "dropout", "inbound_nodes": [[["activation", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "dtype": "float32", "filters": 512, "kernel_size": [16], "strides": [1], "padding": "same", "data_format": "channels_last", "dilation_rate": [1], "groups": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 1024]}, "name": "conv1d_1", "inbound_nodes": [[["dropout", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "BatchNormalization", "config": {"name": "batch_normalization_1", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "gamma_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "moving_mean_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "moving_variance_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 512]}, "name": "batch_normalization_1", "inbound_nodes": [[["conv1d_1", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Activation", "config": {"name": "activation_1", "trainable": true, "dtype": "float32", "activation": "relu"}, "registered_name": null, "build_config": {"input_shape": [null, 300, 512]}, "name": "activation_1", "inbound_nodes": [[["batch_normalization_1", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Dropout", "config": {"name": "dropout_1", "trainable": true, "dtype": "float32", "rate": 0.5, "noise_shape": null, "seed": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 512]}, "name": "dropout_1", "inbound_nodes": [[["activation_1", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Conv1D", "config": {"name": "conv1d_2", "trainable": true, "dtype": "float32", "filters": 256, "kernel_size": [32], "strides": [1], "padding": "same", "data_format": "channels_last", "dilation_rate": [1], "groups": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 512]}, "name": "conv1d_2", "inbound_nodes": [[["dropout_1", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "BatchNormalization", "config": {"name": "batch_normalization_2", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "gamma_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "moving_mean_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "moving_variance_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 256]}, "name": "batch_normalization_2", "inbound_nodes": [[["conv1d_2", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Activation", "config": {"name": "activation_2", "trainable": true, "dtype": "float32", "activation": "relu"}, "registered_name": null, "build_config": {"input_shape": [null, 300, 256]}, "name": "activation_2", "inbound_nodes": [[["batch_normalization_2", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Dropout", "config": {"name": "dropout_2", "trainable": true, "dtype": "float32", "rate": 0.5, "noise_shape": null, "seed": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 256]}, "name": "dropout_2", "inbound_nodes": [[["activation_2", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Conv1D", "config": {"name": "conv1d_3", "trainable": true, "dtype": "float32", "filters": 128, "kernel_size": [64], "strides": [1], "padding": "same", "data_format": "channels_last", "dilation_rate": [1], "groups": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 256]}, "name": "conv1d_3", "inbound_nodes": [[["dropout_2", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "BatchNormalization", "config": {"name": "batch_normalization_3", "trainable": true, "dtype": "float32", "axis": [2], "momentum": 0.99, "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "gamma_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "moving_mean_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "moving_variance_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 128]}, "name": "batch_normalization_3", "inbound_nodes": [[["conv1d_3", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Activation", "config": {"name": "activation_3", "trainable": true, "dtype": "float32", "activation": "relu"}, "registered_name": null, "build_config": {"input_shape": [null, 300, 128]}, "name": "activation_3", "inbound_nodes": [[["batch_normalization_3", 0, 0, {}]]]}, {"module": "keras_nlp.src.layers.modeling.rotary_embedding", "class_name": "RotaryEmbedding", "config": {"name": "rotary_embedding", "trainable": true, "dtype": "float32", "max_wavelength": 10000, "scaling_factor": 1.0, "sequence_axis": 1, "feature_axis": -1}, "registered_name": "keras_nlp>RotaryEmbedding", "name": "rotary_embedding", "inbound_nodes": [[["activation_3", 0, 0, {}]]]}, {"module": "keras.src.layers.core.tf_op_layer", "class_name": "TFOpLambda", "config": {"name": "tf.__operators__.add", "trainable": true, "dtype": "float32", "function": "__operators__.add"}, "registered_name": "TFOpLambda", "build_config": {"input_shape": [null, 300, 128]}, "name": "tf.__operators__.add", "inbound_nodes": [["activation_3", 0, 0, {"y": ["rotary_embedding", 0, 0], "name": null}]]}, {"module": "keras.layers", "class_name": "LayerNormalization", "config": {"name": "layernorm_0", "trainable": true, "dtype": "float32", "axis": [2], "epsilon": 0.001, "center": true, "scale": true, "beta_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "gamma_initializer": {"module": "keras.initializers", "class_name": "Ones", "config": {}, "registered_name": null}, "beta_regularizer": null, "gamma_regularizer": null, "beta_constraint": null, "gamma_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 128]}, "name": "layernorm_0", "inbound_nodes": [[["tf.__operators__.add", 0, 0, {}]]]}, {"module": "keras_nlp.src.layers.modeling.transformer_encoder", "class_name": "TransformerEncoder", "config": {"name": "transformer_encoder_0", "trainable": true, "dtype": "float32", "intermediate_dim": 64, "num_heads": 8, "dropout": 0.5, "activation": "relu", "layer_norm_epsilon": 1e-05, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "normalize_first": false}, "registered_name": "keras_nlp>TransformerEncoder", "build_config": {"input_shape": [null, 300, 128]}, "name": "transformer_encoder_0", "inbound_nodes": [[["layernorm_0", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Dropout", "config": {"name": "dropout_0", "trainable": true, "dtype": "float32", "rate": 0.5, "noise_shape": null, "seed": null}, "registered_name": null, "build_config": {"input_shape": [null, 300, 128]}, "name": "dropout_0", "inbound_nodes": [[["transformer_encoder_0", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "AveragePooling1D", "config": {"name": "average_pooling1d", "trainable": true, "dtype": "float32", "strides": [300], "pool_size": [300], "padding": "valid", "data_format": "channels_last"}, "registered_name": null, "build_config": {"input_shape": [null, 300, 128]}, "name": "average_pooling1d", "inbound_nodes": [[["dropout_0", 0, 0, {}]]]}, {"module": "keras.layers", "class_name": "Dense", "config": {"name": "K562", "trainable": true, "dtype": "float32", "units": 1, "activation": "linear", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [null, 1, 128]}, "name": "K562", "inbound_nodes": [[["average_pooling1d", 0, 0, {}]]]}], "input_layers": [["input_1", 0, 0]], "output_layers": [["K562", 0, 0]]}, "keras_version": "2.15.0", "backend": "tensorflow"}