stegsoph/llama3.1-8b-openwebtext-llama3-stochastok0.5-lora-kp3jeb

LoRA Adapter

This repository contains a LoRA adapter trained on top of meta-llama/Llama-3.1-8B.

How to Use

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

adapter_repo = "stegsoph/llama3.1-8b-openwebtext-llama3-stochastok0.5-lora-kp3jeb"
base_model = "meta-llama/Llama-3.1-8B"
dtype = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float16

tokenizer = AutoTokenizer.from_pretrained(base_model)
model = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=dtype, device_map="auto")
model = PeftModel.from_pretrained(model, adapter_repo)
model.eval()

prompt = "Once upon a time,"
inputs = tokenizer(prompt, return_tensors="pt")
if torch.cuda.is_available():
    inputs = {k: v.to("cuda") for k, v in inputs.items()}

out = model.generate(
    **inputs,
    max_new_tokens=64,
    temperature=0.7,
    top_k=50,
    top_p=0.9,
    do_sample=True,
    pad_token_id=tokenizer.eos_token_id,
    eos_token_id=tokenizer.eos_token_id,
)
print(tokenizer.decode(out[0], skip_special_tokens=True))