Adds 300M resid_5_pre SAE trained on the pile

fragrant-disco-34/hyperparameters.yaml

@@ -0,0 +1,72 @@
+adam_beta1: 0.9
+adam_beta2: 0.999
+b_dec_init_method: zeros
+cached_activations_path: null
+checkpoint_path: ./outputs/checkpoints
+clip_grad_norm: true
+context_size: 256
+custom_loss: null
+d_in: 2560
+d_out: null
+dataset: monology/pile-uncopyrighted
+dense_loss_coefficient: 0
+device: cuda
+different_output: false
+dtype: float32
+epsilon_l0_approx: 0.5
+eval_frequency: 500
+expansion_factor: 8
+feature_reinit_scale: 0.2
+feature_resampling_method: null
+fine_tune_dataset: false
+flatten_activations_over_layer: false
+flatten_activations_over_layer_output: false
+from_pretrained_path: null
+hook_point: blocks.5.hook_resid_pre
+hook_point_head_index: null
+hook_point_head_index_output: null
+hook_point_layer: 5
+hook_point_layer_output: null
+hook_point_output: null
+initial_decoder_norm: 0.1
+initialise_encoder_to_decoder_transpose: false
+is_dataset_tokenized: false
+l0_coefficient: 0
+l0_warmup: false
+l0_warmup_steps: 1000
+l1_coefficient: 5.5
+l1_warmup: true
+l1_warmup_steps: 10000
+log_to_wandb: true
+loop_dataset: false
+lr: 0.0001
+lr_scheduler_name: constant_with_warmup
+lr_warm_up_steps: 1000
+max_resample_step: 100000
+max_sparsity_target: 1
+min_sparsity_for_resample: 0
+min_sparsity_target: 0
+model_name: EleutherAI/pythia-2.8b-deduped
+mse_loss_coefficient: 1
+mse_loss_type: standard
+multiple_runs: false
+n_batches_in_store_buffer: 128
+n_checkpoints: 80
+n_running_sparsity: 100
+normalise_initial_decoder_weights: false
+normalise_w_dec: false
+resample_batches: 128
+resample_frequency: 25000
+scale_input_norm: false
+seed: 42
+sparse_loss_coefficient: 0
+sparsity_log_frequency: 5000
+store_batch_size: 8
+subtract_b_dec_from_inputs: false
+total_training_steps: 73242
+train_batch_size: 4096
+use_cached_activations: false
+use_gated_sparse_autoencoder: false
+wandb_log_frequency: 10
+wandb_project: test_pythia-mlp
+weight_l1_by_decoder_norms: true

fragrant-disco-34/sparse_autoencoder/pythia-2.8b-deduped_blocks.0.hook_mlp_out_s20480_300M_tokens_thepile_L0-16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cace92f94e4310515a813e0076c8aa94ce1aeef5f54c1491cbd185e0f40a0fb0
+size 419527672