Upload 11 files

README.md

@@ -1,3 +1,178 @@
 ---
 license: apache-2.0
+datasets:
+- roneneldan/TinyStories
+language:
+- en
+pipeline_tag: text-generation
+library_name: transformers
+tags:
+- small
+- tiny
+- story
+- tinystories
+- roneneldan
+- cpu
+- free
+- open-source
 ---
+
+# 📖 StorySupra 10M
+
+## Config
+- Parameters: 12,587,264 (~10M)
+- Hidden Size: 256
+- Intermediate Size: 1024
+- Hidden Layers: 8
+- Attention Heads: 8
+- Max Position Embeddings: 256
+- Vocab Size: 8192
+
+## Samples
+Once upon a time , a small bird was flying in the sky . It saw a big tree and wanted to rest under it . But the tree was too high for the bird to reach . The bird tried to fly up , but it could not . Then , a wise old owl flew by and saw the bird struggling . The owl said , " Don ' t worry little bird , I can help you ." The owl used its strong beak to climb the tree and get the bird down . The bird was
+<br><br>
+Once upon a time , there was a little boy named Timmy . He loved to play with his toys and run around outside . One day , he found a shiny penny on the ground . It was so pretty that he picked it up and showed it to his mom . " Look , Mommy ! I found a penny !" he said . His mom smiled and said , " That ' s great , Timmy . But be careful , it ' s very special ." Timmy didn ' t understand what " valuable " meant , but he knew it meant something important . So
+<br><br>
+Once upon a time , there was a lovely princess . She had long , blonde hair and a sparkly crown . One day , she wanted to go for a walk in the forest . She put on her dress and started walking . As she walked , she saw something strange . It was a big , scary bear ! The princess was scared , but she didn ' t want to get away . So she just kept walking until she reached the forest . When she got there , she saw a little rabbit . He was wearing a bright red bow and he looked very friendly .
+
+## Training
+- GPU: single RTX 5060 Ti 16GB
+- Time: ~20 minutes
+- Epochs: 3
+- Samples of the dataset: 200k
+
+## Dataset
+200k samples of roneneldan/TinyStories
+
+## Code
+You can find the full code in this repo as `train.py` and inference.py. Have fun :-)
+
+## Usage
+Use this to run the model:
+```python3
+"""
+StorySupra-10M — Interactive Story Generator
+Loads model weights directly from HuggingFace: SupraLabs/StorySupra-10M
+"""
+
+import torch
+from transformers import LlamaForCausalLM, PreTrainedTokenizerFast
+
+# ──────────────────────────────────────────────
+# Configuration
+# ──────────────────────────────────────────────
+MODEL_ID = "SupraLabs/StorySupra-10M"
+
+GENERATION_DEFAULTS = {
+    "max_new_tokens": 100,
+    "temperature": 0.55,
+    "top_k": 25,
+    "top_p": 0.85,
+    "repetition_penalty": 1.1,
+    "do_sample": True,
+}
+
+EXIT_COMMANDS = {"exit", "quit", "leave"}
+
+# ──────────────────────────────────────────────
+# Model loading
+# ──────────────────────────────────────────────
+
+def load_model(model_id: str):
+    """Download and return the tokenizer and model from HuggingFace Hub."""
+    print(f"Downloading model from HuggingFace: {model_id}")
+    print("(This may take a moment on first run — weights will be cached locally.)\n")
+
+    tokenizer = PreTrainedTokenizerFast.from_pretrained(model_id)
+    model = LlamaForCausalLM.from_pretrained(model_id)
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Using device: {device}\n")
+
+    model.to(device)
+    model.eval()
+
+    return tokenizer, model, device
+
+
+# ──────────────────────────────────────────────
+# Text generation
+# ──────────────────────────────────────────────
+
+def generate_text(
+    prompt: str,
+    tokenizer,
+    model,
+    device: str,
+    max_new_tokens: int = GENERATION_DEFAULTS["max_new_tokens"],
+    temperature: float = GENERATION_DEFAULTS["temperature"],
+    top_k: int = GENERATION_DEFAULTS["top_k"],
+    top_p: float = GENERATION_DEFAULTS["top_p"],
+    repetition_penalty: float = GENERATION_DEFAULTS["repetition_penalty"],
+) -> str:
+    """Generate a story continuation from the given prompt."""
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
+
+    with torch.no_grad():
+        output_tokens = model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            do_sample=True,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            repetition_penalty=repetition_penalty,
+            pad_token_id=tokenizer.pad_token_id,
+            eos_token_id=tokenizer.eos_token_id,
+        )
+
+    return tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+
+
+# ──────────────────────────────────────────────
+# Interactive loop
+# ──────────────────────────────────────────────
+
+def run():
+    print("=" * 50)
+    print("  StorySupra-10M — Interactive Story Generator")
+    print("=" * 50)
+
+    tokenizer, model, device = load_model(MODEL_ID)
+
+    print("-" * 50)
+    print("Model ready! Type a prompt to generate a story.")
+    print(f"Type {' / '.join(EXIT_COMMANDS)} to quit.")
+    print("-" * 50)
+
+    while True:
+        try:
+            user_prompt = input("\nYour prompt: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\nExiting. Goodbye!")
+            break
+
+        if not user_prompt:
+            print("Please enter a prompt.")
+            continue
+
+        if user_prompt.lower() in EXIT_COMMANDS:
+            print("Goodbye!")
+            break
+
+        print("\nGenerating...\n")
+        story = generate_text(user_prompt, tokenizer, model, device)
+
+        print("Generated story:")
+        print("-" * 20)
+        print(story)
+        print("-" * 20)
+
+
+# ──────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────
+
+if __name__ == "__main__":
+    run()
+```

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 256,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 8,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.8.1",
+  "use_cache": false,
+  "vocab_size": 8192
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 1,
+  "transformers_version": "5.8.1",
+  "use_cache": true
+}

gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

inference.py

@@ -0,0 +1,50 @@
+print("Loading...")
+
+import torch
+from transformers import LlamaForCausalLM, PreTrainedTokenizerFast
+
+def run_inference():
+    model_path = "./StorySupra-10M"
+    
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Using device: {device}")
+
+    tokenizer = PreTrainedTokenizerFast.from_pretrained(model_path)
+    
+    model = LlamaForCausalLM.from_pretrained(model_path)
+    model.to(device)
+    model.eval()
+
+    def generate_text(prompt, max_new_tokens=100, temperature=0.55, top_k=25, top_p=0.85, repetition_penalty=1.1):
+        inputs = tokenizer(prompt, return_tensors="pt").to(device)
+        
+        with torch.no_grad():
+            output_tokens = model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                do_sample=True,
+                temperature=temperature,
+                top_k=top_k,
+                top_p=top_p,
+                repetition_penalty=repetition_penalty,
+                pad_token_id=tokenizer.pad_token_id,
+                eos_token_id=tokenizer.eos_token_id
+            )
+        
+        return tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+
+    print("-" * 30)
+    print("StorySupra Story Generator loaded!")
+    print("Enter a prompt (or type 'exit' to quit):")
+    
+    while True:
+        user_prompt = input("\nYour prompt: ")
+        if user_prompt.lower() in ["exit", "quit", "leave"]:
+            break
+            
+        story = generate_text(user_prompt)
+        print(f"\nGenerated story:\n{story}")
+        print("-" * 20)
+
+if __name__ == "__main__":
+    run_inference()

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9c874a48b24de2df0d12ec4a8a7e3e9c310d41aeaddff0e79d03803383dbf42
+size 50357208

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

train.py

@@ -0,0 +1,88 @@
+import torch
+from datasets import load_dataset
+from tokenizers import Tokenizer, models, trainers, pre_tokenizers
+from transformers import LlamaConfig, LlamaForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
+
+dataset = load_dataset("roneneldan/TinyStories", split="train[:200000]")
+
+def train_tokenizer(dataset):
+    tokenizer = Tokenizer(models.BPE(unk_token="<unk>"))
+    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
+    
+    trainer = trainers.BpeTrainer(
+        vocab_size=8192,
+        special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]
+    )
+    
+    def batch_iterator():
+        for i in range(0, len(dataset), 1000):
+            yield dataset[i : i + 1000]["text"]
+
+    tokenizer.train_from_iterator(batch_iterator(), trainer=trainer)
+    
+    from transformers import PreTrainedTokenizerFast
+    return PreTrainedTokenizerFast(
+        tokenizer_object=tokenizer,
+        bos_token="<s>",
+        eos_token="</s>",
+        unk_token="<unk>",
+        pad_token="<pad>"
+    )
+
+tokenizer = train_tokenizer(dataset)
+
+def tokenize_function(examples):
+    return tokenizer(examples["text"], truncation=True, max_length=256)
+
+tokenized_dataset = dataset.map(tokenize_function, batched=True, remove_columns=["text"])
+
+config = LlamaConfig(
+    vocab_size=8192,
+    hidden_size=256,
+    intermediate_size=1024,
+    num_hidden_layers=8,
+    num_attention_heads=8,
+    max_position_embeddings=256,
+    pad_token_id=tokenizer.pad_token_id,
+    bos_token_id=tokenizer.bos_token_id,
+    eos_token_id=tokenizer.eos_token_id,
+)
+
+model = LlamaForCausalLM(config)
+print(f"Model parameters: {model.num_parameters():,}")
+
+training_args = TrainingArguments(
+    output_dir="./StorySupra-10M",
+    per_device_train_batch_size=32,
+    num_train_epochs=3,
+    save_steps=500,
+    logging_steps=100,
+    learning_rate=5e-4,
+    weight_decay=0.01,
+    fp16=True,
+    push_to_hub=False,
+    report_to="none",
+    lr_scheduler_type="cosine"
+)
+
+data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_dataset,
+    data_collator=data_collator,
+)
+
+trainer.train()
+
+def generate_story(prompt):
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+    model.to("cuda")
+    outputs = model.generate(**inputs, max_length=100, do_sample=True, temperature=0.55, top_k=25, top_p=0.85, repetition_penalty=1.1)
+    print(tokenizer.decode(outputs[0], skip_special_tokens=True))
+
+generate_story("Once upon a time, a small bird")
+
+trainer.save_model("./StorySupra-10M")
+tokenizer.save_pretrained("./StorySupra-10M")

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b0cca96b3100c2a57b8e16275daef5d68c6a103f14efbbb8dd80db4ca8f2738
+size 5265

use-from-hf.py

@@ -0,0 +1,125 @@
+"""
+StorySupra-10M — Interactive Story Generator
+Loads model weights directly from HuggingFace: SupraLabs/StorySupra-10M
+"""
+
+import torch
+from transformers import LlamaForCausalLM, PreTrainedTokenizerFast
+
+# ──────────────────────────────────────────────
+# Configuration
+# ──────────────────────────────────────────────
+MODEL_ID = "SupraLabs/StorySupra-10M"
+
+GENERATION_DEFAULTS = {
+    "max_new_tokens": 100,
+    "temperature": 0.55,
+    "top_k": 25,
+    "top_p": 0.85,
+    "repetition_penalty": 1.1,
+    "do_sample": True,
+}
+
+EXIT_COMMANDS = {"exit", "quit", "leave"}
+
+# ──────────────────────────────────────────────
+# Model loading
+# ──────────────────────────────────────────────
+
+def load_model(model_id: str):
+    """Download and return the tokenizer and model from HuggingFace Hub."""
+    print(f"Downloading model from HuggingFace: {model_id}")
+    print("(This may take a moment on first run — weights will be cached locally.)\n")
+
+    tokenizer = PreTrainedTokenizerFast.from_pretrained(model_id)
+    model = LlamaForCausalLM.from_pretrained(model_id)
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    print(f"Using device: {device}\n")
+
+    model.to(device)
+    model.eval()
+
+    return tokenizer, model, device
+
+
+# ──────────────────────────────────────────────
+# Text generation
+# ──────────────────────────────────────────────
+
+def generate_text(
+    prompt: str,
+    tokenizer,
+    model,
+    device: str,
+    max_new_tokens: int = GENERATION_DEFAULTS["max_new_tokens"],
+    temperature: float = GENERATION_DEFAULTS["temperature"],
+    top_k: int = GENERATION_DEFAULTS["top_k"],
+    top_p: float = GENERATION_DEFAULTS["top_p"],
+    repetition_penalty: float = GENERATION_DEFAULTS["repetition_penalty"],
+) -> str:
+    """Generate a story continuation from the given prompt."""
+    inputs = tokenizer(prompt, return_tensors="pt").to(device)
+
+    with torch.no_grad():
+        output_tokens = model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            do_sample=True,
+            temperature=temperature,
+            top_k=top_k,
+            top_p=top_p,
+            repetition_penalty=repetition_penalty,
+            pad_token_id=tokenizer.pad_token_id,
+            eos_token_id=tokenizer.eos_token_id,
+        )
+
+    return tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+
+
+# ──────────────────────────────────────────────
+# Interactive loop
+# ──────────────────────────────────────────────
+
+def run():
+    print("=" * 50)
+    print("  StorySupra-10M — Interactive Story Generator")
+    print("=" * 50)
+
+    tokenizer, model, device = load_model(MODEL_ID)
+
+    print("-" * 50)
+    print("Model ready! Type a prompt to generate a story.")
+    print(f"Type {' / '.join(EXIT_COMMANDS)} to quit.")
+    print("-" * 50)
+
+    while True:
+        try:
+            user_prompt = input("\nYour prompt: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            print("\nExiting. Goodbye!")
+            break
+
+        if not user_prompt:
+            print("Please enter a prompt.")
+            continue
+
+        if user_prompt.lower() in EXIT_COMMANDS:
+            print("Goodbye!")
+            break
+
+        print("\nGenerating...\n")
+        story = generate_text(user_prompt, tokenizer, model, device)
+
+        print("Generated story:")
+        print("-" * 20)
+        print(story)
+        print("-" * 20)
+
+
+# ──────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────
+
+if __name__ == "__main__":
+    run()