Upload 11 files

benchmarks.md

@@ -0,0 +1,79 @@
+|                           Tasks                            |Version|Filter|n-shot|    Metric     |   | Value  |   |Stderr|
+|------------------------------------------------------------|------:|------|-----:|---------------|---|-------:|---|------|
+|arc_easy                                                    |      1|none  |     0|acc            |↑  |  0.3439|±  |0.0097|
+|                                                            |       |none  |     0|acc_norm       |↑  |  0.3346|±  |0.0097|
+|blimp                                                       |      2|none  |      |acc            |↑  |  0.6349|±  |0.0016|
+| - blimp_adjunct_island                                     |      1|none  |     0|acc            |↑  |  0.7990|±  |0.0127|
+| - blimp_anaphor_gender_agreement                           |      1|none  |     0|acc            |↑  |  0.3290|±  |0.0149|
+| - blimp_anaphor_number_agreement                           |      1|none  |     0|acc            |↑  |  0.6330|±  |0.0152|
+| - blimp_animate_subject_passive                            |      1|none  |     0|acc            |↑  |  0.6550|±  |0.0150|
+| - blimp_animate_subject_trans                              |      1|none  |     0|acc            |↑  |  0.8180|±  |0.0122|
+| - blimp_causative                                          |      1|none  |     0|acc            |↑  |  0.4900|±  |0.0158|
+| - blimp_complex_NP_island                                  |      1|none  |     0|acc            |↑  |  0.5300|±  |0.0158|
+| - blimp_coordinate_structure_constraint_complex_left_branch|      1|none  |     0|acc            |↑  |  0.2990|±  |0.0145|
+| - blimp_coordinate_structure_constraint_object_extraction  |      1|none  |     0|acc            |↑  |  0.7550|±  |0.0136|
+| - blimp_determiner_noun_agreement_1                        |      1|none  |     0|acc            |↑  |  0.7910|±  |0.0129|
+| - blimp_determiner_noun_agreement_2                        |      1|none  |     0|acc            |↑  |  0.8640|±  |0.0108|
+| - blimp_determiner_noun_agreement_irregular_1              |      1|none  |     0|acc            |↑  |  0.7020|±  |0.0145|
+| - blimp_determiner_noun_agreement_irregular_2              |      1|none  |     0|acc            |↑  |  0.8460|±  |0.0114|
+| - blimp_determiner_noun_agreement_with_adj_2               |      1|none  |     0|acc            |↑  |  0.7370|±  |0.0139|
+| - blimp_determiner_noun_agreement_with_adj_irregular_1     |      1|none  |     0|acc            |↑  |  0.5780|±  |0.0156|
+| - blimp_determiner_noun_agreement_with_adj_irregular_2     |      1|none  |     0|acc            |↑  |  0.7300|±  |0.0140|
+| - blimp_determiner_noun_agreement_with_adjective_1         |      1|none  |     0|acc            |↑  |  0.7060|±  |0.0144|
+| - blimp_distractor_agreement_relational_noun               |      1|none  |     0|acc            |↑  |  0.2630|±  |0.0139|
+| - blimp_distractor_agreement_relative_clause               |      1|none  |     0|acc            |↑  |  0.2060|±  |0.0128|
+| - blimp_drop_argument                                      |      1|none  |     0|acc            |↑  |  0.7110|±  |0.0143|
+| - blimp_ellipsis_n_bar_1                                   |      1|none  |     0|acc            |↑  |  0.5800|±  |0.0156|
+| - blimp_ellipsis_n_bar_2                                   |      1|none  |     0|acc            |↑  |  0.7490|±  |0.0137|
+| - blimp_existential_there_object_raising                   |      1|none  |     0|acc            |↑  |  0.7470|±  |0.0138|
+| - blimp_existential_there_quantifiers_1                    |      1|none  |     0|acc            |↑  |  0.8450|±  |0.0115|
+| - blimp_existential_there_quantifiers_2                    |      1|none  |     0|acc            |↑  |  0.2720|±  |0.0141|
+| - blimp_existential_there_subject_raising                  |      1|none  |     0|acc            |↑  |  0.6560|±  |0.0150|
+| - blimp_expletive_it_object_raising                        |      1|none  |     0|acc            |↑  |  0.6820|±  |0.0147|
+| - blimp_inchoative                                         |      1|none  |     0|acc            |↑  |  0.4210|±  |0.0156|
+| - blimp_intransitive                                       |      1|none  |     0|acc            |↑  |  0.5750|±  |0.0156|
+| - blimp_irregular_past_participle_adjectives               |      1|none  |     0|acc            |↑  |  0.9240|±  |0.0084|
+| - blimp_irregular_past_participle_verbs                    |      1|none  |     0|acc            |↑  |  0.6800|±  |0.0148|
+| - blimp_irregular_plural_subject_verb_agreement_1          |      1|none  |     0|acc            |↑  |  0.7100|±  |0.0144|
+| - blimp_irregular_plural_subject_verb_agreement_2          |      1|none  |     0|acc            |↑  |  0.8520|±  |0.0112|
+| - blimp_left_branch_island_echo_question                   |      1|none  |     0|acc            |↑  |  0.8390|±  |0.0116|
+| - blimp_left_branch_island_simple_question                 |      1|none  |     0|acc            |↑  |  0.3810|±  |0.0154|
+| - blimp_matrix_question_npi_licensor_present               |      1|none  |     0|acc            |↑  |  0.0060|±  |0.0024|
+| - blimp_npi_present_1                                      |      1|none  |     0|acc            |↑  |  0.5420|±  |0.0158|
+| - blimp_npi_present_2                                      |      1|none  |     0|acc            |↑  |  0.5250|±  |0.0158|
+| - blimp_only_npi_licensor_present                          |      1|none  |     0|acc            |↑  |  0.3710|±  |0.0153|
+| - blimp_only_npi_scope                                     |      1|none  |     0|acc            |↑  |  0.4090|±  |0.0156|
+| - blimp_passive_1                                          |      1|none  |     0|acc            |↑  |  0.7980|±  |0.0127|
+| - blimp_passive_2                                          |      1|none  |     0|acc            |↑  |  0.7770|±  |0.0132|
+| - blimp_principle_A_c_command                              |      1|none  |     0|acc            |↑  |  0.6410|±  |0.0152|
+| - blimp_principle_A_case_1                                 |      1|none  |     0|acc            |↑  |  1.0000|±  |     0|
+| - blimp_principle_A_case_2                                 |      1|none  |     0|acc            |↑  |  0.7200|±  |0.0142|
+| - blimp_principle_A_domain_1                               |      1|none  |     0|acc            |↑  |  0.7350|±  |0.0140|
+| - blimp_principle_A_domain_2                               |      1|none  |     0|acc            |↑  |  0.6190|±  |0.0154|
+| - blimp_principle_A_domain_3                               |      1|none  |     0|acc            |↑  |  0.5460|±  |0.0158|
+| - blimp_principle_A_reconstruction                         |      1|none  |     0|acc            |↑  |  0.4780|±  |0.0158|
+| - blimp_regular_plural_subject_verb_agreement_1            |      1|none  |     0|acc            |↑  |  0.7920|±  |0.0128|
+| - blimp_regular_plural_subject_verb_agreement_2            |      1|none  |     0|acc            |↑  |  0.7970|±  |0.0127|
+| - blimp_sentential_negation_npi_licensor_present           |      1|none  |     0|acc            |↑  |  1.0000|±  |     0|
+| - blimp_sentential_negation_npi_scope                      |      1|none  |     0|acc            |↑  |  0.6700|±  |0.0149|
+| - blimp_sentential_subject_island                          |      1|none  |     0|acc            |↑  |  0.4350|±  |0.0157|
+| - blimp_superlative_quantifiers_1                          |      1|none  |     0|acc            |↑  |  0.9270|±  |0.0082|
+| - blimp_superlative_quantifiers_2                          |      1|none  |     0|acc            |↑  |  0.5460|±  |0.0158|
+| - blimp_tough_vs_raising_1                                 |      1|none  |     0|acc            |↑  |  0.4410|±  |0.0157|
+| - blimp_tough_vs_raising_2                                 |      1|none  |     0|acc            |↑  |  0.6850|±  |0.0147|
+| - blimp_transitive                                         |      1|none  |     0|acc            |↑  |  0.7490|±  |0.0137|
+| - blimp_wh_island                                          |      1|none  |     0|acc            |↑  |  0.4360|±  |0.0157|
+| - blimp_wh_questions_object_gap                            |      1|none  |     0|acc            |↑  |  0.5880|±  |0.0156|
+| - blimp_wh_questions_subject_gap                           |      1|none  |     0|acc            |↑  |  0.8800|±  |0.0103|
+| - blimp_wh_questions_subject_gap_long_distance             |      1|none  |     0|acc            |↑  |  0.9460|±  |0.0072|
+| - blimp_wh_vs_that_no_gap                                  |      1|none  |     0|acc            |↑  |  0.9810|±  |0.0043|
+| - blimp_wh_vs_that_no_gap_long_distance                    |      1|none  |     0|acc            |↑  |  0.9880|±  |0.0034|
+| - blimp_wh_vs_that_with_gap                                |      1|none  |     0|acc            |↑  |  0.1180|±  |0.0102|
+| - blimp_wh_vs_that_with_gap_long_distance                  |      1|none  |     0|acc            |↑  |  0.0340|±  |0.0057|
+|wikitext                                                    |      2|none  |     0|bits_per_byte  |↓  |  1.4123|±  |   N/A|
+|                                                            |       |none  |     0|byte_perplexity|↓  |  2.6617|±  |   N/A|
+|                                                            |       |none  |     0|word_perplexity|↓  |187.7215|±  |   N/A|
+
+|Groups|Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|------|------:|------|------|------|---|-----:|---|-----:|
+|blimp |      2|none  |      |acc   |↑  |0.6349|±  |0.0016|

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 48,
+  "hidden_act": "silu",
+  "hidden_size": 192,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 1024,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 4,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 500000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.8.1",
+  "use_cache": false,
+  "vocab_size": 16384
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 1,
+  "transformers_version": "5.8.1",
+  "use_cache": true
+}

inference.py

@@ -0,0 +1,33 @@
+print("[*] Loading libraries...")
+import torch
+from transformers import LlamaForCausalLM, PreTrainedTokenizerFast
+
+model_path = "./Supra-Mini-v5-8M-FINAL"
+
+print("[*] Loading tokenizer...")
+tokenizer = PreTrainedTokenizerFast.from_pretrained(model_path)
+
+print("[*] Loading model...")
+model = LlamaForCausalLM.from_pretrained(model_path)
+model.eval()
+
+prompt = "The main concept of physics is " # "Artificial intelligence is " | "The main concept of physics is " | "Once upon a time, "
+print(f"[*] Prompt: {prompt!r}")
+
+inputs = tokenizer(prompt, return_tensors="pt")
+
+with torch.no_grad():
+    outputs = model.generate(
+        input_ids=inputs["input_ids"],
+        attention_mask=inputs["attention_mask"],
+        max_new_tokens=256,
+        do_sample=True,
+        temperature=0.5,
+        top_p=0.9,
+        top_k=25,
+        repetition_penalty=1.3,
+        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+    )
+
+print("[*] Output:", tokenizer.decode(outputs[0], skip_special_tokens=True))

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d1deeaccf5b8fad50dfd4506c92980f1bff6d612290982d2be6bfc68bd32a0
+size 31478384

samples_BASE.txt

@@ -0,0 +1,16 @@
+[*] Prompt: 'Once upon a time, '
+[*] Output: Once upon a time, .. ...
+- "The word 'pot' is not used in the past.") (A.D., A.D.)
+- "It's like a thing of course that we're going to think about it. The phrase "these" means what you see and how much are there?").
+- "You have no idea where I'm so, but if my son has any thought or understanding of his name, he will be able to understand him by saying something more than one day. This way they can tell us when you've got me at home. That is why I want your child to know which words are most important for them: "If you get this language from another person, then you'll find yourself in the same place as you read it," says Mike McNamara, who was born with an English friend, Jennifer Batharinee, who had been diagnosed with dementia during her lifetime. He said he would learn how to say things such as "a lot of things," and "you don't really need to do anything else." It may seem simple because he didn't feel good before he went out and asked whether he could make sense of it. But he wanted to take advantage of the fact
+
+[*] Prompt: 'Artificial intelligence is '
+[*] Output: Artificial intelligence is _______.
+I was a scientist at the University of Cambridge in London, and I had been looking for ways to get them back into the first time. It’s not just about learning how to use it but also about the world around us.
+The study has led to more than 150 studies showing that people who are able to learn new things like computers, laptops, and smartphones can be found in many different types of computer systems. The researchers have demonstrated that this type of technology works well with a wide range of computing applications. They believe they need to know what they want to do, such as a laptop or iPad, which means you might find themselves doing everything on your own. This could mean we would take up to 3 times an hour if we didn't see any problems, including anything from something else. So now there is no idea how much data will go through so far. We don’t understand why these algorithms aren’t actually working on their devices. And if you think about it, then here are some of the most important things we can do to help make our decisions.
+What does it say? What happens when someone comes across the internet? Why doesn’t they really feel like the Internet? How should you connect
+
+[*] Prompt: 'The main concept of physics is '
+[*] Output: The main concept of physics is _______.
+This principle, which means that the universe has no energy; it should be a matter of time in its own right and therefore to be able to achieve its maximum potential. It must have been used as an alternative for quantum mechanics but not only in terms of power but also in the form of electrical systems such as magnetic fields or electric currents. The theory would include some processes like this: "We are using mechanical machines." In fact, we need to work with other types of computers, including those who do so by their computer. This is because they can actually use them to generate electricity from our devices. Thus, if you want to learn how to manufacture electronic circuits, then you will see a very simple solution - the way you think about what happens when you're trying to build up on your device.
+So, why does that sound? Because there's something new than a few different things, and I'm sure that you don't know exactly where it was. But, just let me know if there were many ways to create the same thing. You might say that it could make sense to you how much one is doing at all, even though it isn't going to happen! So, if you'd ask yourself to write down the problem

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

train.py

@@ -0,0 +1,188 @@
+"""
+© SupraLabs 2026 - Official pretraining code for Supra Mini v5 8M
+"""
+
+import os
+os.environ["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+
+print("[*] Loading libraries...")
+import torch
+import math
+import numpy as np
+from datasets import load_dataset
+from tokenizers import ByteLevelBPETokenizer
+from transformers import (
+    LlamaConfig,
+    LlamaForCausalLM,
+    PreTrainedTokenizerFast,
+    Trainer,
+    TrainingArguments,
+)
+from torch.utils.data import Dataset
+from tqdm import tqdm
+
+print("[*] Loading tokenizer...")
+fast_tokenizer = ByteLevelBPETokenizer(
+    "./custom_llama_tokenizer-vocab.json",
+    "./custom_llama_tokenizer-merges.txt"
+)
+tokenizer = PreTrainedTokenizerFast(
+    tokenizer_object=fast_tokenizer,
+    bos_token="<s>",
+    eos_token="</s>",
+    unk_token="<unk>",
+    pad_token="<pad>",
+)
+
+TOKEN_BIN     = "./tokens.bin"
+TARGET_TOKENS = 5_000_000_000
+SEQ_LEN       = 1024
+BATCH_TEXTS   = 1000
+FLUSH_EVERY   = 1_000_000
+
+
+def build_token_bin(fast_tokenizer, path=TOKEN_BIN, target_tokens=TARGET_TOKENS):
+    if os.path.exists(path) and os.path.getsize(path) >= target_tokens * 2:
+        print(f"[=] Reusing existing token file: {path}")
+        return
+
+    print(f"[*] Streaming + tokenizing {target_tokens:,} tokens → {path}")
+    mm = np.memmap(path, dtype=np.uint16, mode="w+", shape=(target_tokens,))
+
+    dataset = load_dataset(
+        "HuggingFaceFW/fineweb-edu", "sample-10BT",
+        split="train", streaming=True
+    )
+
+    written = 0
+    buf = []
+    texts = []
+    pbar = tqdm(total=target_tokens, desc="[*] Gathering tokens", unit="tok")
+
+    def flush_buf():
+        nonlocal written, buf
+        if not buf:
+            return False
+        n = min(len(buf), target_tokens - written)
+        mm[written:written + n] = np.asarray(buf[:n], dtype=np.uint16)
+        written += n
+        pbar.update(n)
+        del buf[:n]
+        return written >= target_tokens
+
+    for example in dataset:
+        texts.append(example["text"])
+        if len(texts) >= BATCH_TEXTS:
+            encs = fast_tokenizer.encode_batch(texts)
+            texts.clear()
+            for e in encs:
+                buf.extend(e.ids)
+            if len(buf) >= FLUSH_EVERY:
+                if flush_buf():
+                    break
+
+    if written < target_tokens and texts:
+        encs = fast_tokenizer.encode_batch(texts)
+        for e in encs:
+            buf.extend(e.ids)
+    if written < target_tokens:
+        flush_buf()
+
+    pbar.close()
+    mm.flush()
+    del mm
+    print(f"[+] Wrote {written:,} tokens to {path} "
+          f"({os.path.getsize(path)/1e6:.1f} MB)")
+
+
+class MemmapDataset(Dataset):
+    def __init__(self, path, total_tokens, seq_len=SEQ_LEN):
+        self.path     = path
+        self.seq_len  = seq_len
+        self.n_chunks = total_tokens // seq_len
+        self._data    = None  # lazy open (Multiprocessing-safe)
+
+    @property
+    def data(self):
+        if self._data is None:
+            self._data = np.memmap(
+                self.path, dtype=np.uint16, mode="r",
+                shape=(self.n_chunks * self.seq_len,)
+            )
+        return self._data
+
+    def __len__(self):
+        return self.n_chunks
+
+    def __getitem__(self, idx):
+        s   = idx * self.seq_len
+        arr = np.asarray(self.data[s:s + self.seq_len], dtype=np.int64)
+        ids = torch.from_numpy(arr)
+        return {"input_ids": ids, "labels": ids.clone()}
+
+
+def collate_fn(batch):
+    input_ids = torch.stack([b["input_ids"] for b in batch])
+    labels    = torch.stack([b["labels"]    for b in batch])
+    return {"input_ids": input_ids, "labels": labels}
+
+
+print(f"[*] Preparing {TARGET_TOKENS:,} tokens (streaming, memmap-backed)...")
+build_token_bin(fast_tokenizer, TOKEN_BIN, TARGET_TOKENS)
+dataset = MemmapDataset(TOKEN_BIN, TARGET_TOKENS, seq_len=SEQ_LEN)
+print(f"[+] Dataset ready: {len(dataset):,} chunks of {SEQ_LEN} tokens")
+
+print("[*] Setting up model...")
+config = LlamaConfig(
+    vocab_size=16384,
+    hidden_size=192,
+    intermediate_size=768,
+    num_hidden_layers=8,
+    num_attention_heads=4,
+    num_key_value_heads=4,
+    max_position_embeddings=1024,
+    rope_theta=500000,
+    tie_word_embeddings=True,
+    pad_token_id=tokenizer.pad_token_id,
+    bos_token_id=tokenizer.bos_token_id,
+    eos_token_id=tokenizer.eos_token_id,
+)
+model = LlamaForCausalLM(config)
+print(f"[*] Model parameters: {model.num_parameters():,}")
+
+print("[*] Defining training arguments...")
+training_args = TrainingArguments(
+    output_dir="./Supra-Mini-v5-8M",
+    num_train_epochs=2,
+    per_device_train_batch_size=64,
+    gradient_accumulation_steps=16,
+    save_steps=500,
+    save_total_limit=2,
+    logging_steps=100,
+    weight_decay=0.01,
+    fp16=False,
+    bf16=True,
+    push_to_hub=False,
+    report_to="none",
+    dataloader_num_workers=os.cpu_count() // 2,
+    dataloader_pin_memory=True,
+    learning_rate=2e-4,
+    lr_scheduler_type="cosine",
+    warmup_ratio=0.05,
+    max_grad_norm=1.0,
+    torch_compile=True,
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=dataset,
+    data_collator=collate_fn,
+)
+
+print("[*] Starting training...")
+trainer.train()
+trainer.save_model("./Supra-Mini-v5-8M-FINAL")
+tokenizer.save_pretrained("./Supra-Mini-v5-8M-FINAL")
+print("[*] Training finished.")

training.log

@@ -0,0 +1,130 @@
+[*] Loading libraries...
+[*] Loading tokenizer...
+[*] Preparing 5,000,000,000 tokens (streaming, memmap-backed)...
+[=] Reusing existing token file: ./tokens.bin
+[+] Dataset ready: 4,882,812 chunks of 1024 tokens
+[*] Setting up model...
+[*] Model parameters: 7,867,584
+[*] Defining training arguments...
+[transformers] warmup_ratio is deprecated and will be removed in v5.2. Use `warmup_steps` instead.
+[*] Starting training...
+  0%|                                                  | 0/9538 [00:00<?, ?it/s]W0515 19:05:07.214000 41242 torch/_inductor/utils.py:1731] [0/0] Not enough SMs to use max_autotune_gemm mode
+{'loss': '9.401', 'grad_norm': '1.098', 'learning_rate': '4.151e-05', 'epoch': '0.02097'}
+{'loss': '8.45', 'grad_norm': '1.012', 'learning_rate': '8.344e-05', 'epoch': '0.04194'}
+{'loss': '7.463', 'grad_norm': '1.007', 'learning_rate': '0.0001254', 'epoch': '0.06291'}
+{'loss': '6.763', 'grad_norm': '1.135', 'learning_rate': '0.0001673', 'epoch': '0.08389'}
+{'loss': '6.296', 'grad_norm': '0.8444', 'learning_rate': '0.0002', 'epoch': '0.1049'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 45.64it/s]
+{'loss': '5.963', 'grad_norm': '1.129', 'learning_rate': '0.0001999', 'epoch': '0.1258'}
+{'loss': '5.732', 'grad_norm': '1.432', 'learning_rate': '0.0001997', 'epoch': '0.1468'}
+{'loss': '5.555', 'grad_norm': '1.714', 'learning_rate': '0.0001994', 'epoch': '0.1678'}
+{'loss': '5.407', 'grad_norm': '1.082', 'learning_rate': '0.0001989', 'epoch': '0.1887'}
+{'loss': '5.281', 'grad_norm': '1.087', 'learning_rate': '0.0001984', 'epoch': '0.2097'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 51.79it/s]
+{'loss': '5.176', 'grad_norm': '1.031', 'learning_rate': '0.0001977', 'epoch': '0.2307'}
+{'loss': '5.081', 'grad_norm': '1.037', 'learning_rate': '0.0001969', 'epoch': '0.2517'}
+{'loss': '4.997', 'grad_norm': '1.259', 'learning_rate': '0.000196', 'epoch': '0.2726'}
+{'loss': '4.919', 'grad_norm': '1.149', 'learning_rate': '0.0001949', 'epoch': '0.2936'}
+{'loss': '4.848', 'grad_norm': '1.25', 'learning_rate': '0.0001938', 'epoch': '0.3146'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 53.17it/s]
+{'loss': '4.782', 'grad_norm': '1.465', 'learning_rate': '0.0001925', 'epoch': '0.3355'}
+{'loss': '4.717', 'grad_norm': '1.792', 'learning_rate': '0.0001912', 'epoch': '0.3565'}
+{'loss': '4.656', 'grad_norm': '1.379', 'learning_rate': '0.0001897', 'epoch': '0.3775'}
+{'loss': '4.598', 'grad_norm': '1.669', 'learning_rate': '0.0001881', 'epoch': '0.3985'}
+{'loss': '4.55', 'grad_norm': '1.305', 'learning_rate': '0.0001864', 'epoch': '0.4194'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 53.48it/s]
+{'loss': '4.508', 'grad_norm': '1.443', 'learning_rate': '0.0001846', 'epoch': '0.4404'}
+{'loss': '4.47', 'grad_norm': '1.677', 'learning_rate': '0.0001827', 'epoch': '0.4614'}
+{'loss': '4.437', 'grad_norm': '1.25', 'learning_rate': '0.0001807', 'epoch': '0.4823'}
+{'loss': '4.403', 'grad_norm': '1.595', 'learning_rate': '0.0001786', 'epoch': '0.5033'}
+{'loss': '4.375', 'grad_norm': '1.593', 'learning_rate': '0.0001764', 'epoch': '0.5243'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 53.04it/s]
+{'loss': '4.35', 'grad_norm': '1.411', 'learning_rate': '0.0001741', 'epoch': '0.5453'}
+{'loss': '4.328', 'grad_norm': '2.014', 'learning_rate': '0.0001718', 'epoch': '0.5662'}
+{'loss': '4.303', 'grad_norm': '1.523', 'learning_rate': '0.0001693', 'epoch': '0.5872'}
+{'loss': '4.285', 'grad_norm': '1.343', 'learning_rate': '0.0001668', 'epoch': '0.6082'}
+{'loss': '4.264', 'grad_norm': '1.376', 'learning_rate': '0.0001641', 'epoch': '0.6291'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 53.00it/s]
+{'loss': '4.247', 'grad_norm': '1.62', 'learning_rate': '0.0001614', 'epoch': '0.6501'}
+{'loss': '4.232', 'grad_norm': '1.243', 'learning_rate': '0.0001587', 'epoch': '0.6711'}
+{'loss': '4.214', 'grad_norm': '1.226', 'learning_rate': '0.0001558', 'epoch': '0.6921'}
+{'loss': '4.199', 'grad_norm': '1.243', 'learning_rate': '0.0001529', 'epoch': '0.713'}
+{'loss': '4.186', 'grad_norm': '1.813', 'learning_rate': '0.0001499', 'epoch': '0.734'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 52.44it/s]
+{'loss': '4.171', 'grad_norm': '1.488', 'learning_rate': '0.0001469', 'epoch': '0.755'}
+{'loss': '4.158', 'grad_norm': '1.535', 'learning_rate': '0.0001438', 'epoch': '0.7759'}
+{'loss': '4.148', 'grad_norm': '1.208', 'learning_rate': '0.0001407', 'epoch': '0.7969'}
+{'loss': '4.136', 'grad_norm': '1.366', 'learning_rate': '0.0001375', 'epoch': '0.8179'}
+{'loss': '4.125', 'grad_norm': '1.289', 'learning_rate': '0.0001343', 'epoch': '0.8389'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 52.97it/s]
+{'loss': '4.115', 'grad_norm': '1.306', 'learning_rate': '0.000131', 'epoch': '0.8598'}
+{'loss': '4.104', 'grad_norm': '1.121', 'learning_rate': '0.0001277', 'epoch': '0.8808'}
+{'loss': '4.096', 'grad_norm': '1.608', 'learning_rate': '0.0001243', 'epoch': '0.9018'}
+{'loss': '4.089', 'grad_norm': '1.192', 'learning_rate': '0.0001209', 'epoch': '0.9227'}
+{'loss': '4.078', 'grad_norm': '1.291', 'learning_rate': '0.0001175', 'epoch': '0.9437'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 53.15it/s]
+{'loss': '4.073', 'grad_norm': '1.054', 'learning_rate': '0.0001141', 'epoch': '0.9647'}
+{'loss': '4.066', 'grad_norm': '1.141', 'learning_rate': '0.0001107', 'epoch': '0.9857'}
+{'loss': '4.057', 'grad_norm': '1.703', 'learning_rate': '0.0001072', 'epoch': '1.007'}
+{'loss': '4.051', 'grad_norm': '1.104', 'learning_rate': '0.0001038', 'epoch': '1.027'}
+{'loss': '4.042', 'grad_norm': '1.058', 'learning_rate': '0.0001003', 'epoch': '1.048'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 22.14it/s]
+{'loss': '4.038', 'grad_norm': '1.095', 'learning_rate': '9.683e-05', 'epoch': '1.069'}
+{'loss': '4.032', 'grad_norm': '1.074', 'learning_rate': '9.337e-05', 'epoch': '1.09'}
+{'loss': '4.027', 'grad_norm': '1.18', 'learning_rate': '8.991e-05', 'epoch': '1.111'}
+{'loss': '4.02', 'grad_norm': '1.193', 'learning_rate': '8.647e-05', 'epoch': '1.132'}
+{'loss': '4.015', 'grad_norm': '1.291', 'learning_rate': '8.304e-05', 'epoch': '1.153'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.08it/s]
+{'loss': '4.012', 'grad_norm': '1.045', 'learning_rate': '7.964e-05', 'epoch': '1.174'}
+{'loss': '4.008', 'grad_norm': '1.35', 'learning_rate': '7.625e-05', 'epoch': '1.195'}
+{'loss': '4.002', 'grad_norm': '1.086', 'learning_rate': '7.29e-05', 'epoch': '1.216'}
+{'loss': '3.999', 'grad_norm': '0.8626', 'learning_rate': '6.958e-05', 'epoch': '1.237'}
+{'loss': '3.992', 'grad_norm': '1.381', 'learning_rate': '6.63e-05', 'epoch': '1.258'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.27it/s]
+{'loss': '3.99', 'grad_norm': '1.229', 'learning_rate': '6.305e-05', 'epoch': '1.279'}
+{'loss': '3.987', 'grad_norm': '0.8244', 'learning_rate': '5.985e-05', 'epoch': '1.3'}
+{'loss': '3.982', 'grad_norm': '0.9264', 'learning_rate': '5.67e-05', 'epoch': '1.321'}
+{'loss': '3.982', 'grad_norm': '1.037', 'learning_rate': '5.36e-05', 'epoch': '1.342'}
+{'loss': '3.976', 'grad_norm': '0.9665', 'learning_rate': '5.056e-05', 'epoch': '1.363'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 60.27it/s]
+{'loss': '3.975', 'grad_norm': '0.8869', 'learning_rate': '4.758e-05', 'epoch': '1.384'}
+{'loss': '3.971', 'grad_norm': '0.7576', 'learning_rate': '4.466e-05', 'epoch': '1.405'}
+{'loss': '3.968', 'grad_norm': '0.8313', 'learning_rate': '4.18e-05', 'epoch': '1.426'}
+{'loss': '3.965', 'grad_norm': '0.7926', 'learning_rate': '3.902e-05', 'epoch': '1.447'}
+{'loss': '3.963', 'grad_norm': '0.9134', 'learning_rate': '3.631e-05', 'epoch': '1.468'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 58.90it/s]
+{'loss': '3.963', 'grad_norm': '0.7194', 'learning_rate': '3.367e-05', 'epoch': '1.489'}
+{'loss': '3.96', 'grad_norm': '0.6361', 'learning_rate': '3.112e-05', 'epoch': '1.51'}
+{'loss': '3.957', 'grad_norm': '0.927', 'learning_rate': '2.865e-05', 'epoch': '1.531'}
+{'loss': '3.957', 'grad_norm': '0.6016', 'learning_rate': '2.626e-05', 'epoch': '1.552'}
+{'loss': '3.954', 'grad_norm': '0.6197', 'learning_rate': '2.397e-05', 'epoch': '1.573'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.64it/s]
+{'loss': '3.952', 'grad_norm': '0.577', 'learning_rate': '2.176e-05', 'epoch': '1.594'}
+{'loss': '3.948', 'grad_norm': '0.5791', 'learning_rate': '1.965e-05', 'epoch': '1.615'}
+{'loss': '3.951', 'grad_norm': '0.5636', 'learning_rate': '1.763e-05', 'epoch': '1.636'}
+{'loss': '3.949', 'grad_norm': '0.5653', 'learning_rate': '1.572e-05', 'epoch': '1.657'}
+{'loss': '3.948', 'grad_norm': '0.5782', 'learning_rate': '1.39e-05', 'epoch': '1.678'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.44it/s]
+{'loss': '3.946', 'grad_norm': '0.4793', 'learning_rate': '1.219e-05', 'epoch': '1.699'}
+{'loss': '3.946', 'grad_norm': '0.4931', 'learning_rate': '1.058e-05', 'epoch': '1.72'}
+{'loss': '3.945', 'grad_norm': '0.5097', 'learning_rate': '9.086e-06', 'epoch': '1.741'}
+{'loss': '3.945', 'grad_norm': '0.5356', 'learning_rate': '7.697e-06', 'epoch': '1.761'}
+{'loss': '3.945', 'grad_norm': '0.5223', 'learning_rate': '6.419e-06', 'epoch': '1.782'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.33it/s]
+{'loss': '3.944', 'grad_norm': '0.4148', 'learning_rate': '5.253e-06', 'epoch': '1.803'}
+{'loss': '3.943', 'grad_norm': '0.4304', 'learning_rate': '4.201e-06', 'epoch': '1.824'}
+{'loss': '3.943', 'grad_norm': '0.4192', 'learning_rate': '3.265e-06', 'epoch': '1.845'}
+{'loss': '3.942', 'grad_norm': '0.4074', 'learning_rate': '2.444e-06', 'epoch': '1.866'}
+{'loss': '3.941', 'grad_norm': '0.44', 'learning_rate': '1.741e-06', 'epoch': '1.887'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 20.57it/s]
+{'loss': '3.942', 'grad_norm': '0.4061', 'learning_rate': '1.156e-06', 'epoch': '1.908'}
+{'loss': '3.941', 'grad_norm': '0.3939', 'learning_rate': '6.899e-07', 'epoch': '1.929'}
+{'loss': '3.942', 'grad_norm': '0.3792', 'learning_rate': '3.431e-07', 'epoch': '1.95'}
+{'loss': '3.944', 'grad_norm': '0.3599', 'learning_rate': '1.161e-07', 'epoch': '1.971'}
+{'loss': '3.942', 'grad_norm': '0.3671', 'learning_rate': '9.142e-09', 'epoch': '1.992'}
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.80it/s]
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 59.16it/s]
+{'train_runtime': '3.949e+04', 'train_samples_per_second': '247.3', 'train_steps_per_second': '0.242', 'train_loss': '4.414', 'epoch': '2'}
+100%|████████████████████████████████████| 9538/9538 [10:58:05<00:00,  4.14s/it]
+Writing model shards: 100%|███████████████████████| 1/1 [00:00<00:00, 57.38it/s]
+[*] Training finished.

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed1f09ec8e4fd507c505c722f8f8332bfd3288995ba900f334dd814118a9818a
+size 5265