Upload Filipino_Llama3_1_Inference_Only.ipynb

Filipino_Llama3_1_Inference_Only.ipynb

@@ -0,0 +1,110 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "!pip install \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"\n",
+        "!pip install peft transformers accelerate bitsandbytes\n"
+      ],
+      "metadata": {
+        "id": "RTea66-cz-9p"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title This uses Unsloth and the fine-tuned lora for faster inference\n",
+        "from unsloth import FastLanguageModel\n",
+        "from peft import PeftModel\n",
+        "import torch\n",
+        "\n",
+        "max_seq_length = 2048\n",
+        "\n",
+        "# Load the BASE model (must match the base used to train the LoRA)\n",
+        "model, tokenizer = FastLanguageModel.from_pretrained(\n",
+        "    model_name = \"unsloth/Llama-3.1-8B\",\n",
+        "    max_seq_length = max_seq_length,\n",
+        "    dtype = None,\n",
+        "    load_in_4bit = True,\n",
+        ")\n",
+        "\n",
+        "# Load the LoRA adapter from HuggingFace\n",
+        "model = PeftModel.from_pretrained(\n",
+        "    model,\n",
+        "    \"welyjesch/filipino_llama_3.1_FT_lora\"\n",
+        ")\n",
+        "\n",
+        "# Enable Unsloth optimized inference\n",
+        "FastLanguageModel.for_inference(model)\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "xXc4bcMG4_0e"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "#@title Run this after entering your instructions\n",
+        "\n",
+        "alpaca_prompt = \"\"\"Ang nasa ibaba ay isang instruksyon na naglalarawan ng isang gawain. Sumulat ng isang tugon na angkop na kumukumpleto sa kahilingan.\n",
+        "\n",
+        "### Instruction:\n",
+        "{}\n",
+        "\n",
+        "### Response:\n",
+        "{}\n",
+        "\"\"\"\n",
+        "\n",
+        "instructions = \"Gumawa ng isang kantang Tagalog\" # @param {type:\"string\"}\n",
+        "inputs = tokenizer(\n",
+        "[\n",
+        "    alpaca_prompt.format(\n",
+        "        instructions,\n",
+        "        \"\",\n",
+        "    )\n",
+        "], return_tensors=\"pt\").to(\"cuda\")\n",
+        "\n",
+        "outputs = model.generate(\n",
+        "    **inputs,\n",
+        "    max_new_tokens=256,\n",
+        "    use_cache=True\n",
+        ")\n",
+        "\n",
+        "import textwrap\n",
+        "raw_output = tokenizer.batch_decode(outputs)[0]\n",
+        "\n",
+        "# Split the output into lines, wrap each line, and then join them back\n",
+        "wrapped_lines = [textwrap.fill(line, width=80) for line in raw_output.splitlines()]\n",
+        "wrapped_output = '\\n'.join(wrapped_lines)\n",
+        "\n",
+        "print(wrapped_output)"
+      ],
+      "metadata": {
+        "id": "PO0DWOjw568p"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}