File size: 30,991 Bytes

---
license: mit
language:
- en
tags:
- harley-ml
- tenete
- small
- sota
- slm
- text-generation
model-index:
- name: Tenete-8M
  results:
  - task:
      type: multiple-choice
      name: ANLI R1 (0-Shot)
    dataset:
      type: anli_r1
      name: ANLI R1
    metrics:
    - name: accuracy
      type: acc
      value: 0.369
  - task:
      type: multiple-choice
      name: ANLI R2 (0-Shot)
    dataset:
      type: anli_r2
      name: ANLI R2
    metrics:
    - name: accuracy
      type: acc
      value: 0.331
  - task:
      type: multiple-choice
      name: ANLI R3 (0-Shot)
    dataset:
      type: anli_r3
      name: ANLI R3
    metrics:
    - name: accuracy
      type: acc
      value: 0.3233
  - task:
      type: multiple-choice
      name: ARC Challenge (0-Shot)
    dataset:
      type: arc_challenge
      name: ARC Challenge
    metrics:
    - name: accuracy
      type: acc
      value: 0.1809
    - name: accuracy_norm
      type: acc_norm
      value: 0.221
  - task:
      type: multiple-choice
      name: ARC Easy (0-Shot)
    dataset:
      type: arc_easy
      name: ARC Easy
    metrics:
    - name: accuracy
      type: acc
      value: 0.3283
    - name: accuracy_norm
      type: acc_norm
      value: 0.3194
  - task:
      type: multiple-choice
      name: HellaSwag (0-Shot)
    dataset:
      type: hellaswag
      name: HellaSwag
    metrics:
    - name: accuracy
      type: acc
      value: 0.2649
    - name: accuracy_norm
      type: acc_norm
      value: 0.2677
  - task:
      type: multiple-choice
      name: MMLU (0-Shot)
    dataset:
      type: mmlu
      name: MMLU
    metrics:
    - name: accuracy
      type: acc
      value: 0.23
  - task:
      type: multiple-choice
      name: MMLU Humanities (0-Shot)
    dataset:
      type: mmlu
      name: MMLU Humanities
    metrics:
    - name: accuracy
      type: acc
      value: 0.2429
  - task:
      type: multiple-choice
      name: MMLU Other (0-Shot)
    dataset:
      type: mmlu
      name: MMLU Other
    metrics:
    - name: accuracy
      type: acc
      value: 0.235
  - task:
      type: multiple-choice
      name: MMLU Social Sciences (0-Shot)
    dataset:
      type: mmlu
      name: MMLU Social Sciences
    metrics:
    - name: accuracy
      type: acc
      value: 0.2168
  - task:
      type: multiple-choice
      name: MMLU STEM (0-Shot)
    dataset:
      type: mmlu
      name: MMLU STEM
    metrics:
    - name: accuracy
      type: acc
      value: 0.2185
  - task:
      type: multiple-choice
      name: PiQA (0-Shot)
    dataset:
      type: piqa
      name: PiQA
    metrics:
    - name: accuracy
      type: acc
      value: 0.5544
    - name: accuracy_norm
      type: acc_norm
      value: 0.5571
  - task:
      type: multiple-choice
      name: SWAG (0-Shot)
    dataset:
      type: swag
      name: SWAG
    metrics:
    - name: accuracy
      type: acc
      value: 0.3024
    - name: accuracy_norm
      type: acc_norm
      value: 0.3297
  - task:
      type: multiple-choice
      name: TruthfulQA MC1 (0-Shot)
    dataset:
      type: truthfulqa_mc1
      name: TruthfulQA MC1
    metrics:
    - name: accuracy
      type: acc
      value: 0.2705
  - task:
      type: multiple-choice
      name: TruthfulQA MC2 (0-Shot)
    dataset:
      type: truthfulqa_mc2
      name: TruthfulQA MC2
    metrics:
    - name: accuracy
      type: acc
      value: 0.4591
  - task:
      type: text-generation
      name: GSM8K (0-Shot)
    dataset:
      type: gsm8k
      name: GSM8K
    metrics:
    - name: exact_match (flexible-extract)
      type: exact_match
      value: 0.0114
    - name: exact_match (strict-match)
      type: exact_match
      value: 0.0015
  - task:
      type: text-generation
      name: TruthfulQA Gen (0-Shot)
    dataset:
      type: truthfulqa_gen
      name: TruthfulQA Gen
    metrics:
    - name: bleu_acc
      type: bleu_acc
      value: 0.2399
    - name: bleu_diff
      type: bleu_diff
      value: -1.2697
    - name: bleu_max
      type: bleu_max
      value: 10.7605
    - name: rouge1_acc
      type: rouge1_acc
      value: 0.2864
    - name: rouge1_diff
      type: rouge1_diff
      value: -2.4981
    - name: rouge1_max
      type: rouge1_max
      value: 22.1008
    - name: rouge2_acc
      type: rouge2_acc
      value: 0.0979
    - name: rouge2_diff
      type: rouge2_diff
      value: -1.7592
    - name: rouge2_max
      type: rouge2_max
      value: 11.8332
    - name: rougeL_acc
      type: rougeL_acc
      value: 0.2815
    - name: rougeL_diff
      type: rougeL_diff
      value: -2.28
    - name: rougeL_max
      type: rougeL_max
      value: 20.7733
  - task:
      type: multiple-choice
      name: ANLI R1 (5-Shot)
    dataset:
      type: anli_r1
      name: ANLI R1
    metrics:
    - name: accuracy
      type: acc
      value: 0.35
  - task:
      type: multiple-choice
      name: ANLI R2 (5-Shot)
    dataset:
      type: anli_r2
      name: ANLI R2
    metrics:
    - name: accuracy
      type: acc
      value: 0.334
  - task:
      type: multiple-choice
      name: ANLI R3 (5-Shot)
    dataset:
      type: anli_r3
      name: ANLI R3
    metrics:
    - name: accuracy
      type: acc
      value: 0.325
  - task:
      type: multiple-choice
      name: ARC Challenge (5-Shot)
    dataset:
      type: arc_challenge
      name: ARC Challenge
    metrics:
    - name: accuracy
      type: acc
      value: 0.1843
    - name: accuracy_norm
      type: acc_norm
      value: 0.2184
  - task:
      type: multiple-choice
      name: ARC Easy (5-Shot)
    dataset:
      type: arc_easy
      name: ARC Easy
    metrics:
    - name: accuracy
      type: acc
      value: 0.338
    - name: accuracy_norm
      type: acc_norm
      value: 0.3215
  - task:
      type: multiple-choice
      name: HellaSwag (5-Shot)
    dataset:
      type: hellaswag
      name: HellaSwag
    metrics:
    - name: accuracy
      type: acc
      value: 0.2644
    - name: accuracy_norm
      type: acc_norm
      value: 0.2657
  - task:
      type: multiple-choice
      name: MMLU (5-Shot)
    dataset:
      type: mmlu
      name: MMLU
    metrics:
    - name: accuracy
      type: acc
      value: 0.2413
  - task:
      type: multiple-choice
      name: MMLU Humanities (5-Shot)
    dataset:
      type: mmlu
      name: MMLU Humanities
    metrics:
    - name: accuracy
      type: acc
      value: 0.2446
  - task:
      type: multiple-choice
      name: MMLU Other (5-Shot)
    dataset:
      type: mmlu
      name: MMLU Other
    metrics:
    - name: accuracy
      type: acc
      value: 0.2288
  - task:
      type: multiple-choice
      name: MMLU Social Sciences (5-Shot)
    dataset:
      type: mmlu
      name: MMLU Social Sciences
    metrics:
    - name: accuracy
      type: acc
      value: 0.2317
  - task:
      type: multiple-choice
      name: MMLU STEM (5-Shot)
    dataset:
      type: mmlu
      name: MMLU STEM
    metrics:
    - name: accuracy
      type: acc
      value: 0.2578
  - task:
      type: multiple-choice
      name: PiQA (5-Shot)
    dataset:
      type: piqa
      name: PiQA
    metrics:
    - name: accuracy
      type: acc
      value: 0.556
    - name: accuracy_norm
      type: acc_norm
      value: 0.5533
  - task:
      type: multiple-choice
      name: SWAG (5-Shot)
    dataset:
      type: swag
      name: SWAG
    metrics:
    - name: accuracy
      type: acc
      value: 0.2963
    - name: accuracy_norm
      type: acc_norm
      value: 0.3201
  - task:
      type: multiple-choice
      name: TruthfulQA MC1 (5-Shot)*
    dataset:
      type: truthfulqa_mc1
      name: TruthfulQA MC1
    metrics:
    - name: accuracy
      type: acc
      value: 0.2705
  - task:
      type: multiple-choice
      name: TruthfulQA MC2 (5-Shot)*
    dataset:
      type: truthfulqa_mc2
      name: TruthfulQA MC2
    metrics:
    - name: accuracy
      type: acc
      value: 0.4591
  - task:
      type: text-generation
      name: GSM8K (5-Shot)
    dataset:
      type: gsm8k
      name: GSM8K
    metrics:
    - name: exact_match (flexible-extract)
      type: exact_match
      value: 0.0114
    - name: exact_match (strict-match)
      type: exact_match
      value: 0.0015
  - task:
      type: text-generation
      name: TruthfulQA Gen (5-Shot)*
    dataset:
      type: truthfulqa_gen
      name: TruthfulQA Gen
    metrics:
    - name: bleu_acc
      type: bleu_acc
      value: 0.2399
    - name: bleu_diff
      type: bleu_diff
      value: -1.2697
    - name: bleu_max
      type: bleu_max
      value: 10.7605
    - name: rouge1_acc
      type: rouge1_acc
      value: 0.2864
    - name: rouge1_diff
      type: rouge1_diff
      value: -2.4981
    - name: rouge1_max
      type: rouge1_max
      value: 22.1008
    - name: rouge2_acc
      type: rouge2_acc
      value: 0.0979
    - name: rouge2_diff
      type: rouge2_diff
      value: -1.7592
    - name: rouge2_max
      type: rouge2_max
      value: 11.8332
    - name: rougeL_acc
      type: rougeL_acc
      value: 0.2815
    - name: rougeL_diff
      type: rougeL_diff
      value: -2.28
    - name: rougeL_max
      type: rougeL_max
      value: 20.7733
datasets:
- kmfoda/booksum
- nampdn-ai/tiny-textbooks
- fabiochiu/medium-articles
---

**Note**: This was mostly a proof-of-concept. A new, improved second version will be coming out soon with a much larger training budget and better architectural decisions. Stay tuned! :)

# Tenete-8M

**Tenete-8M** is an **eight-million parameter model** trained on **five hundred and seventy-seven million tokens**.
While it can't answer "2 + 2" or write a coherent, logically sound essay, it will *surprise* you, and the credit goes to nampdnai's [**tiny-textbooks**](https://huggingface.co/datasets/nampdn-ai/tiny-textbooks).

## Why "Tenete"?

Tenete means "**Small Canoe**" in **Taushiro**, an endangered language with only **one fluent speaker**. This seemed like the most fitting name. Tenete, being the closest word to "small" in Taushiro that had an English translation, and the fact that the language itself has only one fluent speaker, reflects the **tiny and limited size** that Tenete-8M represents.

## Architecture

Tenete-8M uses the **Qwen3 architecture**.

| Parameter                | value                |
|--------------------------|----------------------|
| NUM_HIDDEN_LAYERS        | 4                    |
| MAX_WINDOW_LAYERS        | 3                    |
| HIDDEN_SIZE              | 256                  |
| NUM_ATTENTION_HEADS      | 4                    |
| NUM_KEY_VALUE_HEADS      | 4                    |
| VOCAB_SIZE               | 16000                |
| INTERMEDIATE_SIZE        | 1024                 |
| ROPE_THETA               | 30000.0              |
| MAX_POSITION_EMBEDDINGS  | 1024                 |
| sliding_window           | 384                  |
| TIE_WORD_EMBEDDINGS      | True                 |

| Embedding parameters | Non-embedding parameters | Total parameters | % of kv heads out of total heads | % of swa layers out of total layers |
|----------------------|--------------------------|------------------|----------------------------------|--------------------------------------|
| 4,096,000            | 4,197,888                | 8,293,888        | 100%                             | 75%                                  |

## Training

Tenete-8M was trained on an **RTX 2060 6GB** for one epoch with a batch size of 4 and a gradient accumulation of 18 (**effective batch size=72**) for two hours and twenty minutes.

### Dataset

The dataset encompasses **577M tokens**, and includes **4 sources**:

1. [Textbooks](https://huggingface.co/datasets/nampdn-ai/tiny-textbooks) (1.2GB): Web data is too noisy, so we decided to use Tiny-Textbooks, a synthetic dataset generated by [Nous-Hermes-Llama2-13b](https://huggingface.co/NousResearch/Nous-Hermes-Llama2-13b)
2. [**Medium Articles**](https://huggingface.co/datasets/fabiochiu/medium-articles) (960MB): While web data, especially medium articles, is noisy, we still need human-written examples
3. [**Books**](https://huggingface.co/datasets/kmfoda/booksum) (284MB): Albeit small, books are still needed to instill creativity into the model
4. **Q&A** (14MB): Sprinkled in to add more knowledgeable examples and question-answering.

We chose to not include code, raw webdata (e.g., fineweb, c4, etc.), and more narrow domains (e.g., arxiv, clinical trials, lesswrong, etc.).

#### Stats

| Metric               | Value                    |
|----------------------|--------------------------|
| tokens               | 577M                     |
| Words                | 384M                     |
| Characters           | 2.428B                   |
| Bits/byte            | 1.7054                   |
| Nats/byte            | 1.1821                   |
| Nats/token           | 5.0926                   |
| Characters/Token     | 4.3081509289548          |

### Training Results

| Epoch | Train Loss | Eval Loss | Train PPL | Eval PPL | Train BPB | Eval BPB | Train BPW | Eval BPW |
|-------|------------|-----------|-----------|----------|-----------|----------|-----------|----------|
| 0.07234 | 6.548 | 4.870 | 698.0 | 130.4 | 2.193 | 1.631 | 14.195 | 10.558 |
| 0.14470 | 4.297 | 3.816 | 73.5 | 45.4 | 1.439 | 1.278 | 9.313 | 8.273 |
| 0.21700 | 3.584 | 3.436 | 36.0 | 31.1 | 1.200 | 1.151 | 7.769 | 7.446 |
| 0.28930 | 3.337 | 3.279 | 28.1 | 26.5 | 1.117 | 1.098 | 7.234 | 7.107 |
| 0.36170 | 3.217 | 3.184 | 25.0 | 24.1 | 1.077 | 1.066 | 6.974 | 6.903 |
| 0.43400 | 3.151 | 3.119 | 23.4 | 22.6 | 1.055 | 1.044 | 6.831 | 6.761 |
| 0.50640 | 3.091 | 3.075 | 22.0 | 21.7 | 1.035 | 1.030 | 6.700 | 6.665 |
| 0.57870 | 3.045 | 3.036 | 21.0 | 20.8 | 1.020 | 1.017 | 6.599 | 6.580 |
| 0.65100 | 3.015 | 3.003 | 20.4 | 20.2 | 1.010 | 1.006 | 6.535 | 6.509 |
| 0.72340 | 2.986 | 2.978 | 19.8 | 19.6 | 1.000 | 0.997 | 6.471 | 6.455 |
| 0.79570 | 2.963 | 2.958 | 19.4 | 19.3 | 0.992 | 0.991 | 6.422 | 6.411 |
| 0.86800 | 2.938 | 2.940 | 18.9 | 18.9 | 0.984 | 0.985 | 6.368 | 6.372 |
| 0.94040 | **2.927** | **2.927** | **18.7** | **18.7** | **0.980** | **0.980** | **6.343** | **6.343** |

![Loss and Perplexity Graph](images/training_graph.png
)
Note: BPB stands for Bits Per Byte, and BPW stands for Bits Per Word.
BPB is simply the amount of yes-no questions the model needs to predict the next byte accurately (1.0 BPB = 1 yes-no question), and BPW is the same thing but at the word level.

---

We decided to evaluate the model on each source to see the difference in perplexity.


| Source | Loss | Perplexity |
|--------|------|------------|
| Textbooks | **2.02** | **7.57** |
| Q&A | 3.20 | 24.65 |
| Books | 3.73 | 41.88 |
| Medium articles | 3.79 | 44.40 |

The textbooks' perplexity is six times lower than that of the Medium articles. This is expected. Tiny-Textbooks uses a templated structure (e.g., Section 1, conclusion, etc.) and an LLM generates the rest, resulting in a lower entropy than standard English. Medium articles are structurally, tonally, and stylistically more diverse and unpredictable. The same could be said for the books.

---

## Benchmarks

| Task | Dataset | Metric | 0-shot | 5-shot |
|------|---------|--------|--------|--------|
| ANLI R1 | anli_r1 | acc | 0.369 | 0.35 |
| ANLI R2 | anli_r2 | acc | 0.331 | 0.334 |
| ANLI R3 | anli_r3 | acc | 0.3233 | 0.325 |
| ARC Challenge | arc_challenge | acc_norm | 0.221 | 0.2184 |
| ARC Easy | arc_easy | acc_norm | 0.3194 | 0.3215 |
| HellaSwag | hellaswag | acc_norm | 0.2677 | 0.2657 |
| MMLU | mmlu | acc | 0.23 | 0.2413 |
| MMLU Humanities | mmlu | acc | 0.2429 | 0.2446 |
| MMLU Other | mmlu | acc | 0.235 | 0.2288 |
| MMLU Social Sciences | mmlu | acc | 0.2168 | 0.2317 |
| MMLU STEM | mmlu | acc | 0.2185 | 0.2578 |
| PiQA | piqa | acc_norm | **0.5571** | 0.5533 |
| SWAG | swag | acc_norm | 0.3297 | 0.3201 |
| TruthfulQA MC1 | truthfulqa_mc1 | acc | 0.2705 | 0.2705 |
| TruthfulQA MC2 | truthfulqa_mc2 | acc | 0.4591 | 0.4591 |
| GSM8K | gsm8k | exact_match (flexible) | 0.0114 | 0.0114 |
| TruthfulQA Gen | truthfulqa_gen | rouge1_acc | 0.2864 | 0.2864 |

The model achieves random or near-random on most tasks, which is expected. An 8M parameter model cannot store world-level knowledge or thoroughly reason.

Note: The full breakdown (LM Harness Output) is right [here](https://huggingface.co/Harley-ml/Tenete-8M/blob/main/raw_lmharness_eval_output.txt)

### Coherency Benchmark

To evaluate the **coherency, factuality, and fluency** of our (and other) models, we use **Qwen3-32B** to grade **300 different generations** generated from an **unconditional prompt**.

Example configuration:
```
# --- Inference settings (local or huggingface) ---
NUM_GENERATIONS = 300
MAX_NEW_TOKENS  = 256
MIN_NEW_TOKENS  = 30
TEMPERATURE     = 0.7
TOP_K           = 30
TOP_P           = 0.9
REP_PENALTY     = 1.2
DO_SAMPLE       = True
INFERENCE_BATCH = 5

# --- Judge settings ---
JUDGE_MODEL    = "qwen/qwen3-32b"
JUDGE_MAX_TOKENS     = 80
JUDGE_TEMPERATURE    = 0.0
```

| Model                          | Avg Score | Incoherent | Mostly Coherent | Partially Coherent | Coherent |
|--------------------------------|-----------|------------|-----------------|--------------------|----------|
| pythia-31m-deduped             | 0.3196    | 207        | 53              | 35                 | 5        |
| Tenete-8M                      | 0.4862    | 133        | 86              | 53                 | 28       |
| gpt2                           | 0.5052    | 108        | 96              | 75                 | 21       |

Our model outperforms Pythia-31M-deduped while being **3.8 times smaller**, and looses to GPT2 by only **~0.02 points**.
Note: In our other model cards, the score may be lower or higher for the same model, because `MAX_NEW_TOKENS` was set to **80, not 256**. Furthermore, we changed the `judge_prompt` to be more accurate. This doesn't mean the scores from other model cards are invalid; they're just a little bit outdated.

## Generations

Prompt: `The`

Output: 
```
higher with a great taste of the damp and a more vibrant beauty than others.  It also has its own unique style.  They have been looking for different shades, and they have a wide range of shades to suit their tastes.  Some people find themselves in love with a rich variety of styles that are popular among those who feel it's beautiful. For example, many brands like Nielson are known for their bright colors and bold color.  Here is a simple recipe:

- What is your favorite type of hair?
- How do you use your skin to make a smooth look?
- Can I use my hair instead of cotton or wool?
- Can I use my hair instead of cotton?

Section 3: Practical Examples and Real-World Scenarios
Let's say you want to create an outfit that looks very well on your face. You can choose from a traditional one such as Jyubande, Scikit-learn, Shoeson, Ring and Towl, or Aramis, or Ladies. Your hair should be slightly darker than other options. On the other hand, if you're looking for clothes with a black dress, you may need to paint a white palette (like red or blue) around your skin.

Conclusion:
In this lesson, we learned how to set up a dress for a fashion design by adding color and texture, pairing and tanks, and sewing techniques. By following these guidelines, you will become a savvy designer and be able to create a personal touch of clothing while enjoying a strong look at all things.
```

Prompt: `Question: What is AI?\nAnswer:`

Output:
```
 Machine learning is a method of computer programs that allows us to learn from each other and perform tasks more efficiently. This makes it easier for students to find information about the world around them, whether they are interested in learning or not.
## III. Why is AI important?
Answer: AI is essential because it helps us make informed decisions based on data and decision-making. It enables us to identify trends, patterns, and relationships with others, which can help us stay focused and motivated during difficult times. Additionally, AI technology has been used by companies like Amazon, Apple, and Google, helping businesses improve their operations, productivity, and profitability.
```

Prompt: `'GPT2 is`

Output:
```
 a type of gympos that uses the OA to produce an image. This technique has many applications in various fields, including:

1. **Interpreting:** An example of this method is a process by which you create an image using the ERGE model to produce images and other images.

2. **Conditioning:** A method can be used to manipulate the image using the ERGE model. For instance, if you have 3 cmd4(3) and 4 cmd5(3).

**Glossary:**
- Image processing: The process of converting images into images.
- ERPPO: Visual Modeling for images with high quality pixels.
- Image processing: The process of converting images from image to image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image on image image image image image image image image image image image image image image image image image image? Image image image images image image image image image image image image.

### Section 1: Introduction to Image Image Images
Artography is a powerful tool that allows us to visualize and visualize images visually. It provides a wide range of tools and techniques that allow us to observe and understand images accurately. In this lesson, we will focus on image image image images image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image images image image image image image image image image image images image image image image image image image image image image image image image image image image image image image image image image image.

Example 2: Image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image image or image image image image image images image image image image image image image image image image image image image image.
```

Prompt: `The color of the sky is`

Output:
```
1.
2. It takes four-dimensional space in a triangle, where each inch of its shape has two sides and one or more angles to represent them. This creates a sense of depth and meaning that the sky is beautifully different from other shapes like circles and squares.
3. It can be used as a symbol for navigation and appreciation. For example: "The world is just beginning on the earth but still changing itself."
4. Color Scheme (rather colors)

Here's an example of how I learned about color theory:

import matplotlib as pdipis as y = ['Size', 'Size'].last_id(x))

Explanation:
To understand the concept of color theory, let's say you have three sides and one side, with the following equation:

Number of sides of length - 1 * 3 = 10^0/9
Total length - 2 + 5 = 12
Step 1: Find the top of the left hand
If there are no sides of length or number, we would be assigned to the bottom of the right hand. For example: "The world is not only darker than traditional, but also lightning."

Explanation:
Now, let's look at some examples of color theory:

1. What is the difference between color theory and composition?
Answer: Color theory is the study of colors and their properties.
2. How do you know which one part of the left hand?
Answer: To see if it means looking upwards and finding the right pair of shades, they may use geometric patterns or textures.
3. How does a color scheme benefit us?
Answer: A color scheme benefits both sides by providing a visual representation of the same value. Some examples include simplicity, creativity, and minimalism.
## Section 4: Practical Examples
Let's consider an example of how color theory can be applied in everyday life. Suppose you want to add a pair of shoes to your home. You could start by taking a pair of shoes with a black dress, then sketch out the size of the hips. Then, you might use a variety of colors to create a balanced and visually appealing shoe. Once you have chosen the type of shirt, you might choose a pair of shoes that complements the specific design elements. Finally, you might apply the appropriate pair of shoes to create a cohesive experience.
```

Prompt: `dfuhudfhudishf88383838838EXMAPLEisdu938u39uAIs`

Output:
```
Want. We also discussed how to use these techniques and best practices, such as using different strategies for delivering personalized experiences and tracking messages with our loved ones. By understanding the principles of self-reflection and learning from these tips, we can apply these skills to other areas of life.
```

---

We showcase the bad generations, such as the third generation, and good generations, such as the second generation.
As you can see, the model is surprisingly coherent for its size, but it isn't as consistent as larger models. 
Some generations are coherent, while others can be well... broken. A larger model would be the same-ish quality consistently.

# Limitations

1. Cannot reason, chat, code, or follow prompts accurately all the time
2. May loop or generate repetitive sequences
3. Almost all generations are unfactual, incoherent, or just wrong

## Use Cases

1. Educational use and testing
2. Deployment on edge devices
3. Fine-tuned for downstream use
4. Or more simply, for fun.


# Inference

```
# =============================================================================
# Inference
# =============================================================================


MODEL_DIR       = "Harley-ml/Tenete-8M"   # path
TOKENIZER_PATH = MODEL_DIR

# --- Generation settings ---
PROMPT             = "The"   # prompt
MAX_NEW_TOKENS     = 256
TEMPERATURE        = 0.7
TOP_P              = 0.95
TOP_K              = 30
REPETITION_PENALTY = 1.2
DO_SAMPLE          = True

# =============================================================================

import torch
from pathlib import Path
from transformers import (
    AutoModelForCausalLM,
    PreTrainedTokenizerFast,
    AddedToken,
)

# ---------------------------------------------------------------------------
# Device
# ---------------------------------------------------------------------------

device = (
    "cuda" if torch.cuda.is_available() else
    "mps"  if torch.backends.mps.is_available() else
    "cpu"
)
print(f"Device : {device}")

# ---------------------------------------------------------------------------
# Tokenizer  (mirrors training setup)
# ---------------------------------------------------------------------------

def load_tokenizer(path: str):
    p = Path(path).resolve()
    if not p.exists():
        raise FileNotFoundError(f"Tokenizer not found: {p}")
    tok = PreTrainedTokenizerFast(tokenizer_file=str(p))
    specials = {}
    if tok.bos_token is None: specials["bos_token"] = AddedToken("<|bos|>", special=True)
    if tok.eos_token is None: specials["eos_token"] = AddedToken("<|eos|>", special=True)
    if tok.unk_token is None: specials["unk_token"] = AddedToken("<|unk|>", special=True)
    if tok.pad_token is None:
        if tok.eos_token is not None:
            tok.pad_token = tok.eos_token
        else:
            specials["pad_token"] = AddedToken("<|pad|>", special=True)
    if specials:
        tok.add_special_tokens(specials)
    tok.padding_side = "left" 
    return tok

print("Loading tokenizer...")
tokenizer = load_tokenizer(TOKENIZER_PATH)
print(f"  Vocab size : {tokenizer.vocab_size}")
print(f"  BOS        : {tokenizer.bos_token!r}")
print(f"  EOS        : {tokenizer.eos_token!r}")
print(f"  PAD        : {tokenizer.pad_token!r}  (id={tokenizer.pad_token_id})")

# ---------------------------------------------------------------------------
# Model
# ---------------------------------------------------------------------------

print(f"\nLoading model from {MODEL_DIR} ...")
model = AutoModelForCausalLM.from_pretrained(
    MODEL_DIR,
    dtype=torch.float16 if device == "cuda" else torch.float32,
    low_cpu_mem_usage=True,
)

model.eval()
model.to(device)

total_params = sum(p.numel() for p in model.parameters())
print(f"  Parameters : {total_params:,}")

# ---------------------------------------------------------------------------
# Generation helper
# ---------------------------------------------------------------------------

def generate(
    prompt: str             = PROMPT,
    max_new_tokens: int     = MAX_NEW_TOKENS,
    temperature: float      = TEMPERATURE,
    top_p: float            = TOP_P,
    top_k: int              = TOP_K,
    repetition_penalty: float = REPETITION_PENALTY,
    do_sample: bool         = DO_SAMPLE,
) -> str:
    
    bos         = tokenizer.bos_token or ""
    full_prompt = bos + prompt

    inputs = tokenizer(
        full_prompt,
        return_tensors="pt",
        add_special_tokens=False,
    ).to(device)
    inputs.pop("token_type_ids", None)  # Qwen3 doesn't use this

    gen_kwargs = dict(
        max_new_tokens     = max_new_tokens,
        do_sample          = do_sample,
        repetition_penalty = repetition_penalty,
        eos_token_id       = tokenizer.eos_token_id,
        pad_token_id       = tokenizer.pad_token_id,
    )
    if do_sample:
        gen_kwargs["temperature"] = temperature
        gen_kwargs["top_p"]       = top_p
        gen_kwargs["top_k"]       = top_k

    with torch.inference_mode():
        output_ids = model.generate(**inputs, **gen_kwargs)

    # Strip the prompt tokens so we only return what was generated
    prompt_len = inputs["input_ids"].shape[-1]
    new_ids    = output_ids[0][prompt_len:]
    return tokenizer.decode(new_ids, skip_special_tokens=True)


# ---------------------------------------------------------------------------
# Run
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    print(f"\nPrompt : {PROMPT!r}")
    print("-" * 60)

    output = generate(PROMPT)

    print("Generated:")
    print(output)
```

## Citation

```bibtex
@misc{tenete-8m,
  title     = {Tenete-8M: An 8M Parameter Language Model that Beats Pythia-31M in Coherence},
  author    = {Harley-ml},
  year      = {2026},
  url       = {https://huggingface.co/Harley-ml/Tenete-8M}
}
```