Update README.md

README.md

@@ -10,476 +10,853 @@ tags:
 - slm
 - text-generation
 model-index:
-  - name: Tenete-8M
-    results:
-      - task:
-          type: multiple-choice
-          name: ANLI R1 (0-Shot)
-        dataset:
-          type: anli_r1
-          name: ANLI R1
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3690
-
-      - task:
-          type: multiple-choice
-          name: ANLI R2 (0-Shot)
-        dataset:
-          type: anli_r2
-          name: ANLI R2
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3310
-
-      - task:
-          type: multiple-choice
-          name: ANLI R3 (0-Shot)
-        dataset:
-          type: anli_r3
-          name: ANLI R3
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3233
-
-      - task:
-          type: multiple-choice
-          name: ARC Challenge (0-Shot)
-        dataset:
-          type: arc_challenge
-          name: ARC Challenge
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.1809
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.2210
-
-      - task:
-          type: multiple-choice
-          name: ARC Easy (0-Shot)
-        dataset:
-          type: arc_easy
-          name: ARC Easy
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3283
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.3194
-
-      - task:
-          type: multiple-choice
-          name: HellaSwag (0-Shot)
-        dataset:
-          type: hellaswag
-          name: HellaSwag
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2649
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.2677
-
-      - task:
-          type: multiple-choice
-          name: MMLU (0-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2300
-
-      - task:
-          type: multiple-choice
-          name: MMLU Humanities (0-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU Humanities
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2429
-
-      - task:
-          type: multiple-choice
-          name: MMLU Other (0-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU Other
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2350
-
-      - task:
-          type: multiple-choice
-          name: MMLU Social Sciences (0-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU Social Sciences
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2168
-
-      - task:
-          type: multiple-choice
-          name: MMLU STEM (0-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU STEM
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2185
-
-      - task:
-          type: multiple-choice
-          name: PiQA (0-Shot)
-        dataset:
-          type: piqa
-          name: PiQA
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.5544
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.5571
-
-      - task:
-          type: multiple-choice
-          name: SWAG (0-Shot)
-        dataset:
-          type: swag
-          name: SWAG
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3024
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.3297
-
-      - task:
-          type: multiple-choice
-          name: TruthfulQA MC1 (0-Shot)
-        dataset:
-          type: truthfulqa_mc1
-          name: TruthfulQA MC1
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2705
-
-      - task:
-          type: multiple-choice
-          name: TruthfulQA MC2 (0-Shot)
-        dataset:
-          type: truthfulqa_mc2
-          name: TruthfulQA MC2
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.4591
-      - task:
-          type: text-generation
-          name: GSM8K (0-Shot)
-        dataset:
-          type: gsm8k
-          name: GSM8K
-        metrics:
-          - name: exact_match (flexible-extract)
-            type: exact_match
-            value: 0.0114
-          - name: exact_match (strict-match)
-            type: exact_match
-            value: 0.0015
-
-      - task:
-          type: text-generation
-          name: TruthfulQA Gen (0-Shot)
-        dataset:
-          type: truthfulqa_gen
-          name: TruthfulQA Gen
-        metrics:
-          - name: bleu_acc
-            type: bleu_acc
-            value: 0.2399
-          - name: bleu_diff
-            type: bleu_diff
-            value: -1.2697
-          - name: bleu_max
-            type: bleu_max
-            value: 10.7605
-          - name: rouge1_acc
-            type: rouge1_acc
-            value: 0.2864
-          - name: rouge1_diff
-            type: rouge1_diff
-            value: -2.4981
-          - name: rouge1_max
-            type: rouge1_max
-            value: 22.1008
-          - name: rouge2_acc
-            type: rouge2_acc
-            value: 0.0979
-          - name: rouge2_diff
-            type: rouge2_diff
-            value: -1.7592
-          - name: rouge2_max
-            type: rouge2_max
-            value: 11.8332
-          - name: rougeL_acc
-            type: rougeL_acc
-            value: 0.2815
-          - name: rougeL_diff
-            type: rougeL_diff
-            value: -2.2800
-          - name: rougeL_max
-            type: rougeL_max
-            value: 20.7733
-      - task:
-          type: multiple-choice
-          name: ANLI R1 (5-Shot)
-        dataset:
-          type: anli_r1
-          name: ANLI R1
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3500
-
-      - task:
-          type: multiple-choice
-          name: ANLI R2 (5-Shot)
-        dataset:
-          type: anli_r2
-          name: ANLI R2
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3340
-
-      - task:
-          type: multiple-choice
-          name: ANLI R3 (5-Shot)
-        dataset:
-          type: anli_r3
-          name: ANLI R3
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3250
-
-      - task:
-          type: multiple-choice
-          name: ARC Challenge (5-Shot)
-        dataset:
-          type: arc_challenge
-          name: ARC Challenge
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.1843
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.2184
-
-      - task:
-          type: multiple-choice
-          name: ARC Easy (5-Shot)
-        dataset:
-          type: arc_easy
-          name: ARC Easy
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.3380
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.3215
-
-      - task:
-          type: multiple-choice
-          name: HellaSwag (5-Shot)
-        dataset:
-          type: hellaswag
-          name: HellaSwag
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2644
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.2657
-
-      - task:
-          type: multiple-choice
-          name: MMLU (5-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2413
-
-      - task:
-          type: multiple-choice
-          name: MMLU Humanities (5-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU Humanities
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2446
-
-      - task:
-          type: multiple-choice
-          name: MMLU Other (5-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU Other
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2288
-
-      - task:
-          type: multiple-choice
-          name: MMLU Social Sciences (5-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU Social Sciences
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2317
-
-      - task:
-          type: multiple-choice
-          name: MMLU STEM (5-Shot)
-        dataset:
-          type: mmlu
-          name: MMLU STEM
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2578
-
-      - task:
-          type: multiple-choice
-          name: PiQA (5-Shot)
-        dataset:
-          type: piqa
-          name: PiQA
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.5560
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.5533
-
-      - task:
-          type: multiple-choice
-          name: SWAG (5-Shot)
-        dataset:
-          type: swag
-          name: SWAG
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2963
-          - name: accuracy_norm
-            type: acc_norm
-            value: 0.3201
-
-      - task:
-          type: multiple-choice
-          name: TruthfulQA MC1 (5-Shot)*
-        dataset:
-          type: truthfulqa_mc1
-          name: TruthfulQA MC1
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.2705
-      - task:
-          type: multiple-choice
-          name: TruthfulQA MC2 (5-Shot)*
-        dataset:
-          type: truthfulqa_mc2
-          name: TruthfulQA MC2
-        metrics:
-          - name: accuracy
-            type: acc
-            value: 0.4591
-      - task:
-          type: text-generation
-          name: GSM8K (5-Shot)
-        dataset:
-          type: gsm8k
-          name: GSM8K
-        metrics:
-          - name: exact_match (flexible-extract)
-            type: exact_match
-            value: 0.0114
-          - name: exact_match (strict-match)
-            type: exact_match
-            value: 0.0015
-      - task:
-          type: text-generation
-          name: TruthfulQA Gen (5-Shot)*
-        dataset:
-          type: truthfulqa_gen
-          name: TruthfulQA Gen
-        metrics:
-          - name: bleu_acc
-            type: bleu_acc
-            value: 0.2399
-          - name: bleu_diff
-            type: bleu_diff
-            value: -1.2697
-          - name: bleu_max
-            type: bleu_max
-            value: 10.7605
-          - name: rouge1_acc
-            type: rouge1_acc
-            value: 0.2864
-          - name: rouge1_diff
-            type: rouge1_diff
-            value: -2.4981
-          - name: rouge1_max
-            type: rouge1_max
-            value: 22.1008
-          - name: rouge2_acc
-            type: rouge2_acc
-            value: 0.0979
-          - name: rouge2_diff
-            type: rouge2_diff
-            value: -1.7592
-          - name: rouge2_max
-            type: rouge2_max
-            value: 11.8332
-          - name: rougeL_acc
-            type: rougeL_acc
-            value: 0.2815
-          - name: rougeL_diff
-            type: rougeL_diff
-            value: -2.2800
-          - name: rougeL_max
-            type: rougeL_max
-            value: 20.7733
----
+- name: Tenete-8M
+  results:
+  - task:
+      type: multiple-choice
+      name: ANLI R1 (0-Shot)
+    dataset:
+      type: anli_r1
+      name: ANLI R1
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.369
+  - task:
+      type: multiple-choice
+      name: ANLI R2 (0-Shot)
+    dataset:
+      type: anli_r2
+      name: ANLI R2
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.331
+  - task:
+      type: multiple-choice
+      name: ANLI R3 (0-Shot)
+    dataset:
+      type: anli_r3
+      name: ANLI R3
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.3233
+  - task:
+      type: multiple-choice
+      name: ARC Challenge (0-Shot)
+    dataset:
+      type: arc_challenge
+      name: ARC Challenge
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.1809
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.221
+  - task:
+      type: multiple-choice
+      name: ARC Easy (0-Shot)
+    dataset:
+      type: arc_easy
+      name: ARC Easy
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.3283
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.3194
+  - task:
+      type: multiple-choice
+      name: HellaSwag (0-Shot)
+    dataset:
+      type: hellaswag
+      name: HellaSwag
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2649
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.2677
+  - task:
+      type: multiple-choice
+      name: MMLU (0-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.23
+  - task:
+      type: multiple-choice
+      name: MMLU Humanities (0-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU Humanities
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2429
+  - task:
+      type: multiple-choice
+      name: MMLU Other (0-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU Other
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.235
+  - task:
+      type: multiple-choice
+      name: MMLU Social Sciences (0-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU Social Sciences
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2168
+  - task:
+      type: multiple-choice
+      name: MMLU STEM (0-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU STEM
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2185
+  - task:
+      type: multiple-choice
+      name: PiQA (0-Shot)
+    dataset:
+      type: piqa
+      name: PiQA
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.5544
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.5571
+  - task:
+      type: multiple-choice
+      name: SWAG (0-Shot)
+    dataset:
+      type: swag
+      name: SWAG
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.3024
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.3297
+  - task:
+      type: multiple-choice
+      name: TruthfulQA MC1 (0-Shot)
+    dataset:
+      type: truthfulqa_mc1
+      name: TruthfulQA MC1
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2705
+  - task:
+      type: multiple-choice
+      name: TruthfulQA MC2 (0-Shot)
+    dataset:
+      type: truthfulqa_mc2
+      name: TruthfulQA MC2
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.4591
+  - task:
+      type: text-generation
+      name: GSM8K (0-Shot)
+    dataset:
+      type: gsm8k
+      name: GSM8K
+    metrics:
+    - name: exact_match (flexible-extract)
+      type: exact_match
+      value: 0.0114
+    - name: exact_match (strict-match)
+      type: exact_match
+      value: 0.0015
+  - task:
+      type: text-generation
+      name: TruthfulQA Gen (0-Shot)
+    dataset:
+      type: truthfulqa_gen
+      name: TruthfulQA Gen
+    metrics:
+    - name: bleu_acc
+      type: bleu_acc
+      value: 0.2399
+    - name: bleu_diff
+      type: bleu_diff
+      value: -1.2697
+    - name: bleu_max
+      type: bleu_max
+      value: 10.7605
+    - name: rouge1_acc
+      type: rouge1_acc
+      value: 0.2864
+    - name: rouge1_diff
+      type: rouge1_diff
+      value: -2.4981
+    - name: rouge1_max
+      type: rouge1_max
+      value: 22.1008
+    - name: rouge2_acc
+      type: rouge2_acc
+      value: 0.0979
+    - name: rouge2_diff
+      type: rouge2_diff
+      value: -1.7592
+    - name: rouge2_max
+      type: rouge2_max
+      value: 11.8332
+    - name: rougeL_acc
+      type: rougeL_acc
+      value: 0.2815
+    - name: rougeL_diff
+      type: rougeL_diff
+      value: -2.28
+    - name: rougeL_max
+      type: rougeL_max
+      value: 20.7733
+  - task:
+      type: multiple-choice
+      name: ANLI R1 (5-Shot)
+    dataset:
+      type: anli_r1
+      name: ANLI R1
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.35
+  - task:
+      type: multiple-choice
+      name: ANLI R2 (5-Shot)
+    dataset:
+      type: anli_r2
+      name: ANLI R2
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.334
+  - task:
+      type: multiple-choice
+      name: ANLI R3 (5-Shot)
+    dataset:
+      type: anli_r3
+      name: ANLI R3
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.325
+  - task:
+      type: multiple-choice
+      name: ARC Challenge (5-Shot)
+    dataset:
+      type: arc_challenge
+      name: ARC Challenge
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.1843
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.2184
+  - task:
+      type: multiple-choice
+      name: ARC Easy (5-Shot)
+    dataset:
+      type: arc_easy
+      name: ARC Easy
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.338
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.3215
+  - task:
+      type: multiple-choice
+      name: HellaSwag (5-Shot)
+    dataset:
+      type: hellaswag
+      name: HellaSwag
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2644
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.2657
+  - task:
+      type: multiple-choice
+      name: MMLU (5-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2413
+  - task:
+      type: multiple-choice
+      name: MMLU Humanities (5-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU Humanities
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2446
+  - task:
+      type: multiple-choice
+      name: MMLU Other (5-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU Other
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2288
+  - task:
+      type: multiple-choice
+      name: MMLU Social Sciences (5-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU Social Sciences
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2317
+  - task:
+      type: multiple-choice
+      name: MMLU STEM (5-Shot)
+    dataset:
+      type: mmlu
+      name: MMLU STEM
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2578
+  - task:
+      type: multiple-choice
+      name: PiQA (5-Shot)
+    dataset:
+      type: piqa
+      name: PiQA
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.556
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.5533
+  - task:
+      type: multiple-choice
+      name: SWAG (5-Shot)
+    dataset:
+      type: swag
+      name: SWAG
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2963
+    - name: accuracy_norm
+      type: acc_norm
+      value: 0.3201
+  - task:
+      type: multiple-choice
+      name: TruthfulQA MC1 (5-Shot)*
+    dataset:
+      type: truthfulqa_mc1
+      name: TruthfulQA MC1
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.2705
+  - task:
+      type: multiple-choice
+      name: TruthfulQA MC2 (5-Shot)*
+    dataset:
+      type: truthfulqa_mc2
+      name: TruthfulQA MC2
+    metrics:
+    - name: accuracy
+      type: acc
+      value: 0.4591
+  - task:
+      type: text-generation
+      name: GSM8K (5-Shot)
+    dataset:
+      type: gsm8k
+      name: GSM8K
+    metrics:
+    - name: exact_match (flexible-extract)
+      type: exact_match
+      value: 0.0114
+    - name: exact_match (strict-match)
+      type: exact_match
+      value: 0.0015
+  - task:
+      type: text-generation
+      name: TruthfulQA Gen (5-Shot)*
+    dataset:
+      type: truthfulqa_gen
+      name: TruthfulQA Gen
+    metrics:
+    - name: bleu_acc
+      type: bleu_acc
+      value: 0.2399
+    - name: bleu_diff
+      type: bleu_diff
+      value: -1.2697
+    - name: bleu_max
+      type: bleu_max
+      value: 10.7605
+    - name: rouge1_acc
+      type: rouge1_acc
+      value: 0.2864
+    - name: rouge1_diff
+      type: rouge1_diff
+      value: -2.4981
+    - name: rouge1_max
+      type: rouge1_max
+      value: 22.1008
+    - name: rouge2_acc
+      type: rouge2_acc
+      value: 0.0979
+    - name: rouge2_diff
+      type: rouge2_diff
+      value: -1.7592
+    - name: rouge2_max
+      type: rouge2_max
+      value: 11.8332
+    - name: rougeL_acc
+      type: rougeL_acc
+      value: 0.2815
+    - name: rougeL_diff
+      type: rougeL_diff
+      value: -2.28
+    - name: rougeL_max
+      type: rougeL_max
+      value: 20.7733
+datasets:
+- kmfoda/booksum
+- nampdn-ai/tiny-textbooks
+- fabiochiu/medium-articles
+---
+
+# Tenete-8M
+
+**Tenete-8M** is an **eight-million parameter model** trained on **five hundred and seventy-seven million tokens**.
+While it can't answer "2 + 2" or write a coherent, logically sound essay, it will *surprise* you, and the credit goes to nampdnai's [**tiny-textbooks**](https://huggingface.co/datasets/nampdn-ai/tiny-textbooks).
+
+## Why "Tenete"?
+
+Tenete means "**Small Canoe**" in **Taushiro**, an endangered language with only **one fluent speaker**. This seemed the most fitting name. Tenete, being the closest word to "small" in Taushiro that had an English translation, and the fact that the language itself has only one fluent speaker, reflects the **tiny and limited size** that Tenete-8M represents.
+
+## Architecture
+
+Tenete-8M uses the **Qwen3 architecture**.
+
+| Parameter                | value                |
+|--------------------------|----------------------|
+| NUM_HIDDEN_LAYERS        | 4                    |
+| MAX_WINDOW_LAYERS        | 3                    |
+| HIDDEN_SIZE              | 256                  |
+| NUM_ATTENTION_HEADS      | 4                    |
+| NUM_KEY_VALUE_HEADS      | 4                    |
+| VOCAB_SIZE               | 16000                |
+| INTERMEDIATE_SIZE        | 1024                 |
+| ROPE_THETA               | 30000.0              |
+| MAX_POSITION_EMBEDDINGS  | 1024                 |
+| sliding_window           | 384                  |
+| TIE_WORD_EMBEDDINGS      | True                 |
+
+| Embedding parameters | Non-embedding parameters | Total parameters | % of kv heads out of total heads | % of swa layers out of total layers |
+|----------------------|--------------------------|------------------|----------------------------------|--------------------------------------|
+| 4,096,000            | 4,197,888                | 8,293,888        | 100%                             | 75%                                  |
+
+## Training
+
+Tenete-8M was trained on an **RTX 2060 6GB** for one epoch with a batch size of 4 and a gradient accumulation of 18, resulting in an **effective batch size of 72**.
+
+### Dataset
+
+The dataset encompasses **577M tokens**, and includes **3 sources**:
+
+1. **Textbooks**: Web data is too noisy, so we decided to use Tiny-Textbooks, a synthetic dataset generated by
+2. **Medium Articles**: While web data, especially medium articles, is noisy, we still need human-written examples
+3. **Books**: Albeit small, books are still needed to instill creativity into the model
+4. **Q&A**: just sprinkled in, just to add more knowledge and question-answering.
+
+We chose to not include code, raw webdata (e.g., fineweb, c4, etc.), and more narrow domains (e.g., arxiv, clinical trials, lesswrong, etc.).
+
+#### Stats
+
+| Metric               | Value                    |
+|----------------------|--------------------------|
+| tokens               | 577M                     |
+| Words                | 384M                     |
+| Characters           | 2.428B                   |
+| Bits/byte            | 1.7054                   |
+| Nats/byte            | 1.1821                   |
+| Nats/token           | 5.0926                   |
+| Characters/Token     | 4.3081509289548          |
+
+### Training Results
+
+| Epoch | Train Loss | Eval Loss | Train PPL | Eval PPL | Train BPB | Eval BPB | Train BPW | Eval BPW |
+|-------|------------|-----------|-----------|----------|-----------|----------|-----------|----------|
+| 0.07234 | 6.548 | 4.870 | 698.0 | 130.4 | 2.193 | 1.631 | 14.195 | 10.558 |
+| 0.14470 | 4.297 | 3.816 | 73.5 | 45.4 | 1.439 | 1.278 | 9.313 | 8.273 |
+| 0.21700 | 3.584 | 3.436 | 36.0 | 31.1 | 1.200 | 1.151 | 7.769 | 7.446 |
+| 0.28930 | 3.337 | 3.279 | 28.1 | 26.5 | 1.117 | 1.098 | 7.234 | 7.107 |
+| 0.36170 | 3.217 | 3.184 | 25.0 | 24.1 | 1.077 | 1.066 | 6.974 | 6.903 |
+| 0.43400 | 3.151 | 3.119 | 23.4 | 22.6 | 1.055 | 1.044 | 6.831 | 6.761 |
+| 0.50640 | 3.091 | 3.075 | 22.0 | 21.7 | 1.035 | 1.030 | 6.700 | 6.665 |
+| 0.57870 | 3.045 | 3.036 | 21.0 | 20.8 | 1.020 | 1.017 | 6.599 | 6.580 |
+| 0.65100 | 3.015 | 3.003 | 20.4 | 20.2 | 1.010 | 1.006 | 6.535 | 6.509 |
+| 0.72340 | 2.986 | 2.978 | 19.8 | 19.6 | 1.000 | 0.997 | 6.471 | 6.455 |
+| 0.79570 | 2.963 | 2.958 | 19.4 | 19.3 | 0.992 | 0.991 | 6.422 | 6.411 |
+| 0.86800 | 2.938 | 2.940 | 18.9 | 18.9 | 0.984 | 0.985 | 6.368 | 6.372 |
+| 0.94040 | **2.927** | **2.927** | **18.7** | **18.7** | **0.980** | **0.980** | **6.343** | **6.343** |
+
+Note: BPB stands for Bits Per Byte, and BPW stands for Bits Per Word.
+BPB is simply the amount of yes-no questions the model needs to predict the next byte accurately (1.0 BPB = 1 yes-no question), and BPW is the same thing but at the word level.
+
+---
+
+We decided to evaluate the model on each source it trained on to see the difference in perplexity.
+
+[omitted temporarily]
+
+---
+
+## Benchmarks
+
+| Task | Dataset | Metric | 0-shot | 5-shot |
+|------|---------|--------|--------|--------|
+| ANLI R1 | anli_r1 | acc | 0.369 | 0.35 |
+| ANLI R2 | anli_r2 | acc | 0.331 | 0.334 |
+| ANLI R3 | anli_r3 | acc | 0.3233 | 0.325 |
+| ARC Challenge | arc_challenge | acc_norm | 0.221 | 0.2184 |
+| ARC Easy | arc_easy | acc_norm | 0.3194 | 0.3215 |
+| HellaSwag | hellaswag | acc_norm | 0.2677 | 0.2657 |
+| MMLU | mmlu | acc | 0.23 | 0.2413 |
+| MMLU Humanities | mmlu | acc | 0.2429 | 0.2446 |
+| MMLU Other | mmlu | acc | 0.235 | 0.2288 |
+| MMLU Social Sciences | mmlu | acc | 0.2168 | 0.2317 |
+| MMLU STEM | mmlu | acc | 0.2185 | 0.2578 |
+| PiQA | piqa | acc_norm | **0.5571** | 0.5533 |
+| SWAG | swag | acc_norm | 0.3297 | 0.3201 |
+| TruthfulQA MC1 | truthfulqa_mc1 | acc | 0.2705 | 0.2705 |
+| TruthfulQA MC2 | truthfulqa_mc2 | acc | 0.4591 | 0.4591 |
+| GSM8K | gsm8k | exact_match (flexible) | 0.0114 | 0.0114 |
+| TruthfulQA Gen | truthfulqa_gen | rouge1_acc | 0.2864 | 0.2864 |
+
+The model achieves random or near-random on most tasks, which is expected. An 8M parameter model cannot store world-level knowledge or thoroughly reason.
+
+### Coherency Benchmark
+
+To evaluate the coherency, factuality, and fluency of our (and other) models, we use Qwen3-32B to grade 300 different generations generated from an unconditional prompt.
+
+Example configuration:
+```
+# --- Inference settings (local or huggingface) ---
+NUM_GENERATIONS = 300
+MAX_NEW_TOKENS  = 256
+MIN_NEW_TOKENS  = 30
+TEMPERATURE     = 0.7
+TOP_K           = 30
+TOP_P           = 0.9
+REP_PENALTY     = 1.2
+DO_SAMPLE       = True
+INFERENCE_BATCH = 5
+
+# --- Judge settings ---
+JUDGE_MODEL    = "qwen/qwen3-32b"
+JUDGE_MAX_TOKENS     = 80
+JUDGE_TEMPERATURE    = 0.0
+```
+
+| Model                          | Avg Score | Incoherent | Mostly Coherent | Partially Coherent | Coherent |
+|--------------------------------|-----------|------------|-----------------|--------------------|----------|
+| pythia-31m-deduped             | 0.3196    | 207        | 53              | 35                 | 5        |
+| Tenete-8M                      | 0.4862    | 133        | 86              | 53                 | 28       |
+| gpt2                           | 0.5052    | 108        | 96              | 75                 | 21       |
+
+Our model outperforms Pythia-31M-deduped while being **3.8 times smaller**, and looses to GPT2 by only **~0.02 points**.
+Note: In our other model cards, the score may be lower or higher for the same model, because `MAX_NEW_TOKENS` was set to **80, not 256**. Furthermore, we changed the `judge_prompt` to be more accurate. This doesn't mean the scores from other model cards are invalid; they're just a little bit outdated.
+
+## Generations
+
+Prompt: `The`
+
+Output: 
+```
+higher with a great taste of the damp and a more vibrant beauty than others.  It also has its own unique style.  They have been looking for different shades, and they have a wide range of shades to suit their tastes.  Some people find themselves in love with a rich variety of styles that are popular among those who feel it's beautiful. For example, many brands like Nielson are known for their bright colors and bold color.  Here is a simple recipe:
+
+- What is your favorite type of hair?
+- How do you use your skin to make a smooth look?
+- Can I use my hair instead of cotton or wool?
+- Can I use my hair instead of cotton?
+
+Section 3: Practical Examples and Real-World Scenarios
+Let's say you want to create an outfit that looks very well on your face. You can choose from a traditional one such as Jyubande, Scikit-learn, Shoeson, Ring and Towl, or Aramis, or Ladies. Your hair should be slightly darker than other options. On the other hand, if you're looking for clothes with a black dress, you may need to paint a white palette (like red or blue) around your skin.
+
+Conclusion:
+In this lesson, we learned how to set up a dress for a fashion design by adding color and texture, pairing and tanks, and sewing techniques. By following these guidelines, you will become a savvy designer and be able to create a personal touch of clothing while enjoying a strong look at all things.
+```
+
+Prompt: `Question: What is AI?\nAnswer:`
+
+Output:
+```
+ Machine learning is a method of computer programs that allows us to learn from each other and perform tasks more efficiently. This makes it easier for students to find information about the world around them, whether they are interested in learning or not.
+## III. Why is AI important?
+Answer: AI is essential because it helps us make informed decisions based on data and decision-making. It enables us to identify trends, patterns, and relationships with others, which can help us stay focused and motivated during difficult times. Additionally, AI technology has been used by companies like Amazon, Apple, and Google, helping businesses improve their operations, productivity, and profitability.
+```
+
+Prompt: `'GPT2 is`
+
+Output:
+```
+ a type of gympos that uses the OA to produce an image. This technique has many applications in various fields, including:
+
+1. **Interpreting:** An example of this method is a process by which you create an image using the ERGE model to produce images and other images.
+
+2. **Conditioning:** A method can be used to manipulate the image using the ERGE model. For instance, if you have 3 cmd4(3) and 4 cmd5(3).
+
+**Glossary:**
+- Image processing: The process of converting images into images.
+- ERPPO: Visual Modeling for images with high quality pixels.
+- Image processing: The process of converting images from image to image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image on image image image image image image image image image image image image image image image image image image? Image image image images image image image image image image image image.
+
+### Section 1: Introduction to Image Image Images
+Artography is a powerful tool that allows us to visualize and visualize images visually. It provides a wide range of tools and techniques that allow us to observe and understand images accurately. In this lesson, we will focus on image image image images image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image images image image image image image image image image image images image image image image image image image image image image image image image image image image image image image image image image image.
+
+Example 2: Image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image or image image image image image images image image image image image image image image image image image image image image.
+```
+
+Prompt: `The color of the sky is`
+
+Output:
+```
+1.
+2. It takes four-dimensional space in a triangle, where each inch of its shape has two sides and one or more angles to represent them. This creates a sense of depth and meaning that the sky is beautifully different from other shapes like circles and squares.
+3. It can be used as a symbol for navigation and appreciation. For example: "The world is just beginning on the earth but still changing itself."
+4. Color Scheme (rather colors)
+```
+Here's an example of how I learned about color theory:
+```
+import matplotlib as pdipis as y = ['Size', 'Size'].last_id(x))
+```
+Explanation:
+To understand the concept of color theory, let's say you have three sides and one side, with the following equation:
+
+Number of sides of length - 1 * 3 = 10^0/9
+Total length - 2 + 5 = 12
+Step 1: Find the top of the left hand
+If there are no sides of length or number, we would be assigned to the bottom of the right hand. For example: "The world is not only darker than traditional, but also lightning."
+```
+Explanation:
+Now, let's look at some examples of color theory:
+
+1. What is the difference between color theory and composition?
+Answer: Color theory is the study of colors and their properties.
+2. How do you know which one part of the left hand?
+Answer: To see if it means looking upwards and finding the right pair of shades, they may use geometric patterns or textures.
+3. How does a color scheme benefit us?
+Answer: A color scheme benefits both sides by providing a visual representation of the same value. Some examples include simplicity, creativity, and minimalism.
+## Section 4: Practical Examples
+Let's consider an example of how color theory can be applied in everyday life. Suppose you want to add a pair of shoes to your home. You could start by taking a pair of shoes with a black dress, then sketch out the size of the hips. Then, you might use a variety of colors to create a balanced and visually appealing shoe. Once you have chosen the type of shirt, you might choose a pair of shoes that complements the specific design elements. Finally, you might apply the appropriate pair of shoes to create a cohesive experience.
+```
+
+Prompt: `dfuhudfhudishf88383838838EXMAPLEisdu938u39uAIs`
+
+Output:
+```
+Want. We also discussed how to use these techniques and best practices, such as using different strategies for delivering personalized experiences and tracking messages with our loved ones. By understanding the principles of self-reflection and learning from these tips, we can apply these skills to other areas of life.
+```
+
+---
+
+We showcase the bad generations, such as the third generation, and good generations, such as the second generation.
+As you can see, the model is surprisingly coherent for its size, but it isn't as consistent as larger models. 
+Some generations are coherent, while others can be well... broken. A larger model would be the same-ish quality consistently.
+
+# Limitations
+
+1. Cannot reason, chat, code, or follow prompts accurately all the time
+2. May loop or generate repetitive sequences
+3. Almost all generations are unfactual, incoherent, or just wrong
+
+## Use Cases
+
+1. Educational use and testing
+2. Deployment on edge devices
+3. Fine-tuned for downstream use
+4. Or more simply, for fun.
+
+
+# Inference
+
+```
+# =============================================================================
+# Inference
+# =============================================================================
+
+
+MODEL_DIR       = "Harley-ml/Tenete-8M"   # path
+TOKENIZER_PATH = MODEL_DIR
+
+# --- Generation settings ---
+PROMPT             = "The"   # prompt
+MAX_NEW_TOKENS     = 256
+TEMPERATURE        = 0.7
+TOP_P              = 0.95
+TOP_K              = 30
+REPETITION_PENALTY = 1.2
+DO_SAMPLE          = True
+
+# =============================================================================
+
+import torch
+from pathlib import Path
+from transformers import (
+    AutoModelForCausalLM,
+    PreTrainedTokenizerFast,
+    AddedToken,
+)
+
+# ---------------------------------------------------------------------------
+# Device
+# ---------------------------------------------------------------------------
+
+device = (
+    "cuda" if torch.cuda.is_available() else
+    "mps"  if torch.backends.mps.is_available() else
+    "cpu"
+)
+print(f"Device : {device}")
+
+# ---------------------------------------------------------------------------
+# Tokenizer  (mirrors training setup)
+# ---------------------------------------------------------------------------
+
+def load_tokenizer(path: str):
+    p = Path(path).resolve()
+    if not p.exists():
+        raise FileNotFoundError(f"Tokenizer not found: {p}")
+    tok = PreTrainedTokenizerFast(tokenizer_file=str(p))
+    specials = {}
+    if tok.bos_token is None: specials["bos_token"] = AddedToken("<|bos|>", special=True)
+    if tok.eos_token is None: specials["eos_token"] = AddedToken("<|eos|>", special=True)
+    if tok.unk_token is None: specials["unk_token"] = AddedToken("<|unk|>", special=True)
+    if tok.pad_token is None:
+        if tok.eos_token is not None:
+            tok.pad_token = tok.eos_token
+        else:
+            specials["pad_token"] = AddedToken("<|pad|>", special=True)
+    if specials:
+        tok.add_special_tokens(specials)
+    tok.padding_side = "left" 
+    return tok
+
+print("Loading tokenizer...")
+tokenizer = load_tokenizer(TOKENIZER_PATH)
+print(f"  Vocab size : {tokenizer.vocab_size}")
+print(f"  BOS        : {tokenizer.bos_token!r}")
+print(f"  EOS        : {tokenizer.eos_token!r}")
+print(f"  PAD        : {tokenizer.pad_token!r}  (id={tokenizer.pad_token_id})")
+
+# ---------------------------------------------------------------------------
+# Model
+# ---------------------------------------------------------------------------
+
+print(f"\nLoading model from {MODEL_DIR} ...")
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_DIR,
+    dtype=torch.float16 if device == "cuda" else torch.float32,
+    low_cpu_mem_usage=True,
+)
+
+model.eval()
+model.to(device)
+
+total_params = sum(p.numel() for p in model.parameters())
+print(f"  Parameters : {total_params:,}")
+
+# ---------------------------------------------------------------------------
+# Generation helper
+# ---------------------------------------------------------------------------
+
+def generate(
+    prompt: str             = PROMPT,
+    max_new_tokens: int     = MAX_NEW_TOKENS,
+    temperature: float      = TEMPERATURE,
+    top_p: float            = TOP_P,
+    top_k: int              = TOP_K,
+    repetition_penalty: float = REPETITION_PENALTY,
+    do_sample: bool         = DO_SAMPLE,
+) -> str:
+    
+    bos         = tokenizer.bos_token or ""
+    full_prompt = bos + prompt
+
+    inputs = tokenizer(
+        full_prompt,
+        return_tensors="pt",
+        add_special_tokens=False,
+    ).to(device)
+    inputs.pop("token_type_ids", None)  # Qwen3 doesn't use this
+
+    gen_kwargs = dict(
+        max_new_tokens     = max_new_tokens,
+        do_sample          = do_sample,
+        repetition_penalty = repetition_penalty,
+        eos_token_id       = tokenizer.eos_token_id,
+        pad_token_id       = tokenizer.pad_token_id,
+    )
+    if do_sample:
+        gen_kwargs["temperature"] = temperature
+        gen_kwargs["top_p"]       = top_p
+        gen_kwargs["top_k"]       = top_k
+
+    with torch.inference_mode():
+        output_ids = model.generate(**inputs, **gen_kwargs)
+
+    # Strip the prompt tokens so we only return what was generated
+    prompt_len = inputs["input_ids"].shape[-1]
+    new_ids    = output_ids[0][prompt_len:]
+    return tokenizer.decode(new_ids, skip_special_tokens=True)
+
+
+# ---------------------------------------------------------------------------
+# Run
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    print(f"\nPrompt : {PROMPT!r}")
+    print("-" * 60)
+
+    output = generate(PROMPT)
+
+    print("Generated:")
+    print(output)
+```