{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "f74714d6-2f2c-4578-820c-165b24b384e4", "metadata": {}, "outputs": [], "source": [ "# !pip install transformers datasets torch scikit-learn" ] }, { "cell_type": "code", "execution_count": 2, "id": "4d408bd1-630e-44bd-a29f-272b70897479", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/saisab/py10/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DatasetDict({\n", " train: Dataset({\n", " features: ['text', 'label'],\n", " num_rows: 16000\n", " })\n", " validation: Dataset({\n", " features: ['text', 'label'],\n", " num_rows: 2000\n", " })\n", " test: Dataset({\n", " features: ['text', 'label'],\n", " num_rows: 2000\n", " })\n", "})\n" ] } ], "source": [ "from datasets import load_dataset\n", "data_files = {\n", " \"train\": \"train-00000-of-00001.parquet\",\n", " \"validation\": \"validation-00000-of-00001.parquet\",\n", " \"test\": \"test-00000-of-00001.parquet\"\n", "}\n", "emotion_dataset = load_dataset(\"parquet\", data_files=data_files)\n", "print(emotion_dataset)" ] }, { "cell_type": "markdown", "id": "866924e3-ae0a-4e54-b52d-b55e39a61993", "metadata": {}, "source": [ "Task 1: Baseline Fine-tuning (FP32/FP16)\n", "In this step, we will fine-tune a pre-trained bert-base-uncased model on our emotion dataset. This will serve as our baseline for performance and model size, against which we'll compare our quantized models later.\n", "1.1. Preprocessing the Data\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "8dee48a4-7c22-417e-a19b-02b422a0909a", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Map: 100%|█| 2000/2000 [00:00<00:00, 3358.52 examp" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DatasetDict({\n", " train: Dataset({\n", " features: ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n", " num_rows: 16000\n", " })\n", " validation: Dataset({\n", " features: ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n", " num_rows: 2000\n", " })\n", " test: Dataset({\n", " features: ['text', 'label', 'input_ids', 'token_type_ids', 'attention_mask'],\n", " num_rows: 2000\n", " })\n", "})\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "from transformers import AutoTokenizer\n", "model_checkpoint = \"bert-base-uncased\"\n", "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n", "def tokenize_function(examples):\n", " return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n", "tokenized_datasets = emotion_dataset.map(tokenize_function, batched=True)\n", "print(tokenized_datasets)" ] }, { "cell_type": "markdown", "id": "071fd634-7077-45da-b6a9-de7abe935082", "metadata": {}, "source": [ "1.2. Loading the Model\n", "Now, we'll load the bert-base-uncased model. We need to configure it for sequence classification with 6 labels, corresponding to the 6 emotions in our dataset. We'll also create mappings between the label IDs (0, 1, 2...) and their names (\"sadness\", \"joy\", etc.) for better readability." ] }, { "cell_type": "code", "execution_count": 4, "id": "eb817928-c546-46b4-b231-c571a5d31f0b", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/saisab/py10/lib/python3.10/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/home/saisab/py10/lib/python3.10/site-packages/torchvision/image.so: undefined symbol: _ZN3c1017RegisterOperatorsD1Ev'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n", " warn(\n", "2025-11-19 01:27:11.237598: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", "2025-11-19 01:27:11.273783: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2025-11-19 01:27:11.273822: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2025-11-19 01:27:11.275099: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2025-11-19 01:27:11.281657: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "2025-11-19 01:27:12.147896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "from transformers import AutoModelForSequenceClassification\n", "num_labels = 6\n", "labels = emotion_dataset[\"train\"].features[\"label\"].names\n", "id2label = {i: label for i, label in enumerate(labels)}\n", "label2id = {label: i for i, label in enumerate(labels)}\n", "model = AutoModelForSequenceClassification.from_pretrained(\n", " model_checkpoint,\n", " num_labels=num_labels,\n", " id2label=id2label,\n", " label2id=label2id\n", ")" ] }, { "cell_type": "markdown", "id": "092d2f5f-f161-474d-9a7d-0f76029be4bc", "metadata": {}, "source": [ "1.3. Defining Metrics\n", "The assignment requires us to report the macro F1-score and accuracy. We'll create a function that computes these metrics during the evaluation phase of our training." ] }, { "cell_type": "code", "execution_count": 5, "id": "f8309e7e-ff28-450d-b7ff-94ddc7c86838", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from sklearn.metrics import accuracy_score, f1_score\n", "\n", "def compute_metrics(eval_pred):\n", " \"\"\"Computes accuracy and F1 score for a given set of predictions.\"\"\"\n", " logits, labels = eval_pred\n", " predictions = np.argmax(logits, axis=-1)\n", " \n", " # Calculate metrics\n", " accuracy = accuracy_score(labels, predictions)\n", " macro_f1 = f1_score(labels, predictions, average=\"macro\")\n", " \n", " return {\n", " \"accuracy\": accuracy,\n", " \"macro_f1\": macro_f1,\n", " }" ] }, { "cell_type": "markdown", "id": "674cb718-82f4-4500-8733-2078137264fe", "metadata": {}, "source": [ "1.4. Fine-Tuning the Model\n", "This is where we set up and run the training process. We'll use the Trainer API from the Hugging Face transformers library, which simplifies the training loop.\n", "The assignment suggests using FP16 (mixed-precision training) to speed things up, which we'll enable. We'll train for 3 epochs, which is usually sufficient for fine-tuning tasks like this." ] }, { "cell_type": "code", "execution_count": 6, "id": "9445d846-ae7a-43ac-92b8-d470baf34346", "metadata": {}, "outputs": [], "source": [ "# !pip install --upgrade transformers" ] }, { "cell_type": "code", "execution_count": null, "id": "dc905f68-0e26-413b-9281-62246dc80af6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/ipykernel_1754079/4283579479.py:34: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n", " trainer = Trainer(\n" ] }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [ 263/1500 02:32 < 12:04, 1.71 it/s, Epoch 0.52/3]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation Loss

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from transformers import TrainingArguments, Trainer\n", "import numpy as np\n", "from sklearn.metrics import accuracy_score, f1_score\n", "\n", "# Re-defining the compute_metrics function to be safe\n", "def compute_metrics(eval_pred):\n", " logits, labels = eval_pred\n", " predictions = np.argmax(logits, axis=-1)\n", " accuracy = accuracy_score(labels, predictions)\n", " macro_f1 = f1_score(labels, predictions, average=\"macro\")\n", " return {\"accuracy\": accuracy, \"macro_f1\": macro_f1}\n", "\n", "model_name = \"finetuned-bert-emotion\"\n", "\n", "# Updated TrainingArguments with modern parameters\n", "training_args = TrainingArguments(\n", " output_dir=model_name,\n", " num_train_epochs=3,\n", " per_device_train_batch_size=32,\n", " per_device_eval_batch_size=64,\n", " \n", " # --- Modern arguments (replacements) ---\n", " eval_strategy=\"epoch\", # Replaces evaluate_during_training\n", " save_strategy=\"epoch\", # Save at end of each epoch\n", " logging_strategy=\"epoch\", # Log at end of each epoch\n", " # --------------------------------------\n", " \n", " load_best_model_at_end=True,\n", " metric_for_best_model=\"macro_f1\",\n", " fp16=True,\n", " push_to_hub=False,\n", ")\n", "\n", "trainer = Trainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=tokenized_datasets[\"train\"],\n", " eval_dataset=tokenized_datasets[\"validation\"],\n", " tokenizer=tokenizer,\n", " compute_metrics=compute_metrics,\n", ")\n", "\n", "# Train the model\n", "trainer.train()" ] }, { "cell_type": "code", "execution_count": null, "id": "54aeb214-3a1f-4108-b58a-5bcea3f91e18", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python (py10)", "language": "python", "name": "py10" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.18" } }, "nbformat": 4, "nbformat_minor": 5 }