Initial commit

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 pythia-2.8b-deduped_resid_pre_layer_11 filter=lfs diff=lfs merge=lfs -text
 pythia-2.8b-deduped_resid_pre_layer_24 filter=lfs diff=lfs merge=lfs -text
+devout-surf-31/hyperparameters.yaml filter=lfs diff=lfs merge=lfs -text
+giddy-morning-30/hyperparameters.yaml filter=lfs diff=lfs merge=lfs -text

chocolate-snow-25/hyperparameters.yaml

@@ -0,0 +1,71 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+b_dec_init_method: zeros
+cached_activations_path: null
+checkpoint_path: ../outputs/checkpoints
+clip_grad_norm: false
+context_size: 256
+custom_loss: null
+d_in: 2560
+d_out: null
+dataset: Skylion007/openwebtext
+dense_loss_coefficient: 0
+device: cuda
+different_output: false
+dtype: float32
+epsilon_l0_approx: 0.2
+eval_frequency: 500
+expansion_factor: 8
+feature_reinit_scale: 0.2
+feature_resampling_method: null
+flatten_activations_over_layer: false
+flatten_activations_over_layer_output: false
+from_pretrained_path: null
+hook_point: blocks.11.hook_resid_pre
+hook_point_head_index: null
+hook_point_head_index_output: null
+hook_point_layer: 11
+hook_point_layer_output: null
+hook_point_output: null
+initial_decoder_norm: 0.1
+initialise_encoder_to_decoder_transpose: false
+is_dataset_tokenized: false
+l0_coefficient: 9.0e-05
+l0_warmup: false
+l0_warmup_steps: 1000
+l1_coefficient: 0
+l1_warmup: false
+l1_warmup_steps: 1000
+log_to_wandb: true
+loop_dataset: false
+lr: 0.0004
+lr_scheduler_name: constant
+lr_warm_up_steps: 500
+max_resample_step: 100000
+max_sparsity_target: 1
+min_sparsity_for_resample: 0
+min_sparsity_target: 8.0e-06
+model_name: EleutherAI/pythia-2.8b-deduped
+mse_loss_coefficient: 1
+mse_loss_type: centered
+multiple_runs: false
+n_batches_in_store_buffer: 128
+n_checkpoints: 80
+n_running_sparsity: 300
+normalise_initial_decoder_weights: false
+normalise_w_dec: true
+resample_batches: 128
+resample_frequency: 25000
+scale_input_norm: false
+seed: 42
+sparse_loss_coefficient: 1.0e-06
+sparsity_log_frequency: 5000
+store_batch_size: 8
+subtract_b_dec_from_inputs: false
+total_training_steps: 200000
+train_batch_size: 4096
+use_cached_activations: false
+use_gated_sparse_autoencoder: false
+wandb_log_frequency: 10
+wandb_project: test_gemma_2b
+weight_l1_by_decoder_norms: false

devout-surf-31/hyperparameters.yaml

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bec3cd5394d19ad235f88ce3825b5ab897b9d1658fdb5bc7a6c91be67846c962
+size 27635242

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f8fe015f01a26d8d42c9e9e5b68837a55ecae1a1011954d04d6b381c5fd2b88
+size 438472512

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_0_log_feature_sparsity.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39e928e0a196df623d04e8e33489e53389f7abbcbc14e16dad56088326f68e2
+size 83755

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_10240000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f81d154e481d330ee7decaaa149b413d4870e0c6c96351bccd8d10d31d48eb
+size 438472632

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_20480000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a41f70df434f664b7eaba3895e6a2549e72b868fd94548b45cc558d1a05a6183
+size 438472632

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_20480000_log_feature_sparsity.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fab99adf5ad7d7a43ac05e4f3eddddf9254fc57f118513820ce467da36aed71
+size 83790

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_30720000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79b9abb147dd062cc7fdb8fe4608d112b5557a6ee83c55162eb2f5d5c8ec8904
+size 438472632

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_40960000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb46e4364ecdf4ce0380f25ec26170f44180b7c3161b7de1f586cf31a291ce6f
+size 438472632

devout-surf-31/sparse_autoencoder_EleutherAI/pythia-2.8b-deduped_blocks.11.hook_resid_pre_s20480_40960000_log_feature_sparsity.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9ed250b3bce8607715d178a4075c38a2b611261095d6c9963a531749d593953
+size 83790

giddy-morning-30/hyperparameters.yaml

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bec3cd5394d19ad235f88ce3825b5ab897b9d1658fdb5bc7a6c91be67846c962
+size 27635242