| Step | \n", "Training Loss | \n", "
|---|---|
| 50 | \n", "0.883000 | \n", "
| 100 | \n", "0.620700 | \n", "
| 150 | \n", "0.568800 | \n", "
| 200 | \n", "0.604400 | \n", "
| 250 | \n", "0.594200 | \n", "
| 300 | \n", "0.569300 | \n", "
| 350 | \n", "0.543300 | \n", "
| 400 | \n", "0.526700 | \n", "
| 450 | \n", "0.523200 | \n", "
| 500 | \n", "0.568000 | \n", "
| 550 | \n", "0.517000 | \n", "
| 600 | \n", "0.485400 | \n", "
| 650 | \n", "0.451500 | \n", "
| 700 | \n", "0.508000 | \n", "
" ] }, "metadata": {} }, { "output_type": "stream", "name": "stderr", "text": [ "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "TrainOutput(global_step=713, training_loss=0.5695890691470966, metrics={'train_runtime': 347.8085, 'train_samples_per_second': 8.194, 'train_steps_per_second': 2.05, 'total_flos': 464916248985600.0, 'train_loss': 0.5695890691470966, 'entropy': 0.5866664808530074, 'num_tokens': 729600.0, 'mean_token_accuracy': 0.8567119194911077, 'epoch': 1.0})" ] }, "metadata": {}, "execution_count": 12 } ] }, { "cell_type": "code", "source": [ "trainer.save_model(\"smol-medical-meadow-FT\")\n", "tokenizer.save_pretrained(\"smol-medical-meadow-FT\")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "qLhReX-lwPOG", "outputId": "a3edce2d-bcaf-4a4a-d840-e4db5e396ffa" }, "execution_count": 13, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "('smol-medical-meadow-FT/tokenizer_config.json',\n", " 'smol-medical-meadow-FT/special_tokens_map.json',\n", " 'smol-medical-meadow-FT/vocab.json',\n", " 'smol-medical-meadow-FT/merges.txt',\n", " 'smol-medical-meadow-FT/added_tokens.json',\n", " 'smol-medical-meadow-FT/tokenizer.json')" ] }, "metadata": {}, "execution_count": 13 } ] }, { "cell_type": "code", "source": [ "from transformers import AutoModelForCausalLM, AutoTokenizer\n", "import torch\n", "\n", "model_name = \"HuggingFaceTB/SmolLM2-135M\"\n", "\n", "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "tokenizer.pad_token = tokenizer.eos_token\n", "\n", "model = AutoModelForCausalLM.from_pretrained(\n", " model_name,\n", " device_map=\"auto\",\n", " torch_dtype=torch.float32\n", ")\n", "\n", "model.config.pad_token_id = tokenizer.eos_token_id\n", "model.config.use_cache = False\n" ], "metadata": { "id": "QtH0BkiGUxLZ" }, "execution_count": 14, "outputs": [] }, { "cell_type": "code", "source": [ "import torch\n", "\n", "def chat(question, max_new_tokens=80):\n", " prompt = f\"User: {question}\\nAssistant:\"\n", " inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n", "\n", " with torch.no_grad():\n", " outputs = model.generate(\n", " **inputs,\n", " max_new_tokens=max_new_tokens,\n", " do_sample=False,\n", " eos_token_id=tokenizer.eos_token_id,\n", " )\n", "\n", " generated = outputs[0][inputs[\"input_ids\"].shape[1]:]\n", " text = tokenizer.decode(generated, skip_special_tokens=True).strip()\n", "\n", " cut_markers = [\n", " \"\\nWhat is\",\n", " \"\\nQ:\",\n", " \"\\nUser:\",\n", " ]\n", " for marker in cut_markers:\n", " idx = text.find(marker)\n", " if idx != -1:\n", " text = text[:idx].strip()\n", " break\n", "\n", " print(\"Q:\", question)\n", " print(\"A:\", text)\n", " print(\"-\" * 60)\n" ], "metadata": { "id": "w07jE5JNUkkv" }, "execution_count": 15, "outputs": [] }, { "cell_type": "code", "source": [ "chat(\"What is diabetic ketoacidosis?\")\n", "chat(\"What is the difference between type 1 and type 2 diabetes?\")\n", "chat(\"What is hypertension?\")\n", "chat(\"What are the symptoms of lupus\")\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "z6DdXtvqqieq", "outputId": "ceb01662-a305-4ec7-a10c-a270de5457e7" }, "execution_count": 16, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n", "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Q: What is diabetic ketoacidosis?\n", "A: It is a condition that occurs when the body cannot process sugar properly.\n", "------------------------------------------------------------\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Q: What is the difference between type 1 and type 2 diabetes?\n", "A: Type 1 diabetes is a condition in which the body does not produce insulin. Type 2 diabetes is a condition in which the body does not produce enough insulin.\n", "------------------------------------------------------------\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "Setting `pad_token_id` to `eos_token_id`:0 for open-end generation.\n" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Q: What is hypertension?\n", "A: Hypertension is a condition in which the blood pressure is too high.\n", "------------------------------------------------------------\n", "Q: What are the symptoms of lupus\n", "A: What are the symptoms of lupus?\n", "\n", "The symptoms of lupus are similar to those of other autoimmune diseases. The most common symptoms are:\n", "\n", "- Joint pain\n", "- Skin rashes\n", "- Fatigue\n", "- Fever\n", "- Swollen glands\n", "- Mouth sores\n", "- Joint pain\n", "- Fatigue\n", "- Fever\n", "- Swollen glands\n", "- Mouth sores\n", "- Joint pain\n", "- Fatigue\n", "-\n", "------------------------------------------------------------\n" ] } ] }, { "cell_type": "code", "source": [ "!pip install -q huggingface_hub\n", "\n", "from huggingface_hub import login\n", "login()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 17, "referenced_widgets": [ "bcc8f74515564e189a6697393a54726d", "33d6a69cd12f4ba78bfbe4c7bc869d70", "1cc4ba2ddfcb42d9a41f84f8f226c060", "3c22fe36b8e44b9db0a931a46528b446", "b997d869c93a49919ac7d233f92feefe", "2a882de6e69b44069d61c1d81986d310", "1641641ea6bc4d91a29154632ad2346b", "00135935b304469b90633666ff836139", "2016d1a8330244eda3964127e983e338", "a0586f3c36a84dbd9d403a88bb524358", "0a963985e9894b10936df0ca7d6cf2a4", "9534cd73c3e5472b9f398c1f2dccf958", "c9fd6aac73704db29d6deed135563862", "98766e1dd5674171a0f43e6904b47af5", "73c990e40dc646ec8348d3009b046a9b", "e0a7c3e4cd054b4ebba3d962ee89e58a", "93f11b69d3854c7c98088255024e6516", "1b119e885f454288b36c417ee61c3753", "43f3adf6d7e8415a8c632cad0b322e71", "5e7e09d18e054b9cb1e17f2c6f58bbe7" ] }, "id": "NhM6L4ma2Yl8", "outputId": "29483409-3a8b-4d7e-9291-f5f718ac040e" }, "execution_count": 17, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "VBox(children=(HTML(value='