more

app.py

@@ -1,62 +1,198 @@
+# gradio app.py --watch-dirs app.py
+
 import gradio as gr
 import numpy as np
 import matplotlib.pyplot as plt
 import matplotlib.animation as animation
 import tempfile
+import torch
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+import torchaudio
+import torchaudio.transforms as T
+from matplotlib.patches import Circle
+from stable_baselines3 import SAC
+from warehouse_env import WarehouseEnv
+from types import SimpleNamespace
+
+
+# ---------------------------- #
+# global variables
+# ---------------------------- #
+# models
+# a model for the automatic-speech-recognition task
+# device = "cuda:0" if torch.cuda.is_available() else "cpu"
+# torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+# model_id = "./models_for_proj/librispeech_asr_dummy"
+# model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True)
+# model.to(device)
+# processor = AutoProcessor.from_pretrained(model_id)
+# asr_pipe = pipeline(
+#     "automatic-speech-recognition",
+#     model=model,
+#     tokenizer=processor.tokenizer,
+#     feature_extractor=processor.feature_extractor,
+#     max_new_tokens=128,
+#     torch_dtype=torch_dtype,
+#     device=device,
+# )
+asr_pipe_default = pipeline("automatic-speech-recognition")
+
+
+# env variables
+rl_model_name = 'agent_policies/sac_warehouse_r_10_working_v1.zip'
+# agent_pos = {'x': 50.0, 'y': 50.0}
+agent_pos = SimpleNamespace(**{'x': 50.0, 'y': 50.0})
+goal_dict = {
+    '1': (20, 20),
+    '2': (80, 20),
+    '3': (80, 80),
+    '4': (20, 80),
+}
+targets_x, targets_y = [], []
+for k, v in goal_dict.items():
+    targets_x.append(v[0])
+    targets_y.append(v[1])
+r_coverage = 10
+
 
 
-def create_animation():
+
+# ---------------------------- #
+# functions
+# ---------------------------- #
+def create_standing_animation():
+    path = [(agent_pos.x, agent_pos.y)]
+    return create_animation(path, targets_x, targets_y, r_coverage)
+
+
+def create_animation(path, targets_x, targets_y, r_coverage):
+    # path = [(i,i) for i in range(90)]
+    # targets_x = [20, 80, 80, 20]
+    # targets_y = [20, 20, 80, 80]
+    # RADIUS_COVERAGE = 10
     fig, ax = plt.subplots(figsize=(7, 7))
-    xdata, ydata = [], []
-    ln, = plt.plot([], [], 'b-', animated=True)
+
+    # agent
+    ln1, = plt.plot([path[0][0]], [path[0][1]], marker='o', color='b', alpha=0.5, linewidth=5, markersize=15)
+
+    # targets
+    ln2, = plt.plot(targets_x, targets_y, marker='X', color='orange', alpha=0.5, linestyle='none', markersize=15)
+    for t_x, t_y in zip(targets_x, targets_y):
+        circle = Circle((t_x, t_y), r_coverage, color='orange', fill=True, alpha=0.3)
+        ax.add_patch(circle)
+    # plt.tight_layout()
 
     def init():
-        ax.set_xlim(0, 2*np.pi)
-        ax.set_ylim(-1.1, 1.1)
-        return ln,
+        ax.set_xlim([0, 100])
+        ax.set_ylim([0, 100])
+        ax.set_title(f'Warehouse Env', fontweight="bold", size=10)
+        return ln1,
 
     def update(frame):
-        xdata.append(frame)
-        ydata.append(np.sin(frame))
-        ln.set_data(xdata, ydata)
-        return ln,
+        # for each frame, update the data stored on each artist.
+        x = [path[frame][0]]
+        y = [path[frame][1]]
+
+        ln1.set_data(x, y)
+        return ln1,
 
-    ani = animation.FuncAnimation(
-        fig, update, frames=np.linspace(0, 2*np.pi, 100),
-        init_func=init, blit=True, repeat=False
-    )
+    ani = animation.FuncAnimation(fig, update, frames=len(path),
+                                  init_func=init, blit=True, repeat=False)
+    # plt.show()
 
     # Save to MP4
     temp_video = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
-    ani.save(temp_video.name, writer='ffmpeg', fps=20)
+    ani.save(temp_video.name, writer='ffmpeg', fps=30)
     plt.close(fig)
-
     return temp_video.name
 
 
+def move_agent(target_input: int):
+    if target_input not in goal_dict:
+        return create_standing_animation(), 'Did not find a target.'
+    # get goal locations:
+    goal_x, goal_y = goal_dict[target_input]
+    # build the path
+    env: WarehouseEnv = WarehouseEnv(render_mode='')
+    model = SAC.load(rl_model_name)
+    obs, info = env.reset(agent_x=agent_pos.x, agent_y=agent_pos.y, goal_x=goal_x, goal_y=goal_y)
+    path = []
+    while True:
+        action, _ = model.predict(obs)
+        obs, rewards, done, trunc, info = env.step(action)
+        path.append((env.agent_x, env.agent_y))
+        if done:
+            break
+        if trunc:
+            obs, info = env.reset(agent_x=agent_pos.x, agent_y=agent_pos.y, goal_x=goal_x, goal_y=goal_y)
+            path = []
+
+    agent_pos.x = path[-1][0]
+    agent_pos.y = path[-1][1]
+    # create animation
+    video_output = create_animation(path, targets_x, targets_y, r_coverage)
+
+    # update status
+    status = f'Went to target {target_input}.'
+
+    return video_output, status
+
 
 def load_image_on_start():
     return np.random.rand(700, 700)
-    # return None
-
-with gr.Blocks() as demo:
-    gr.Markdown("## Agent Control with Language")
-    gr.Markdown('## Say the agent where to go and what to do')
-
-    with gr.Row():
-        with gr.Column():
-            request_audio = gr.Audio()
-            send_btn = gr.Button(value='Send Request')
-            request_text = gr.Textbox(label="Request:", lines=2, interactive=False)
-            request_target = gr.Textbox(label='Target:', lines=2)
-            request_plan = gr.Textbox(label='Plan status:', lines=2)
-        with gr.Column():
-            output_env = gr.Video(label="Env:", autoplay=True)
-
-    # EVENTS:
-    # gr.on(triggers=["load"], fn=load_image_on_start, outputs=output_env_image)
-    # demo.load(fn=load_image_on_start, outputs=output_env_image)
-    demo.load(fn=create_animation, outputs=output_env)
-
-    demo.launch()
+
+def get_text_request(audio_input):
+    audio_input_sr, audio_input_np = audio_input
+    audio_input_t = torch.tensor(audio_input_np, dtype=torch.float32)
+    target_sr = 16000
+    resampler = T.Resample(audio_input_sr, target_sr, dtype=audio_input_t.dtype)
+    resampled_audio_input_t: torch.Tensor = resampler(audio_input_t)
+    resampled_audio_input_np = resampled_audio_input_t.numpy()
+    # result = asr_pipe(resampled_audio_input_np)
+    result = asr_pipe_default(resampled_audio_input_np)
+    return result["text"]
+
+def get_target_from_request(request_text):
+    if 'ONE' in request_text:
+        return 1
+    if 'TWO' in request_text:
+        return 2
+    if 'THREE' in request_text:
+        return 3
+    if 'FOUR' in request_text:
+        return 4
+    return 'NO TARGET FOUND'
+
+
+def create_demo():
+    # main blocks
+    with gr.Blocks() as demo:
+        gr.Markdown("## Agent Control with Language")
+        gr.Markdown('## Say the agent where to go and what to do')
+
+        with gr.Row():
+            with gr.Column():
+                request_audio = gr.Microphone(editable=False)
+                # send_btn = gr.Button(value='Send Request')
+                request_text = gr.Textbox(label="Request:", lines=2, interactive=False)
+                request_target = gr.Textbox(label='Target:', lines=2)
+                status = gr.Textbox(label='Plan status:', lines=2)
+            with gr.Column():
+                output_env = gr.Video(label="Env:", autoplay=True)
+
+        # EVENTS:
+        # gr.on(triggers=["load"], fn=load_image_on_start, outputs=output_env_image)
+        # demo.load(fn=load_image_on_start, outputs=output_env_image)
+        demo.load(fn=create_standing_animation, outputs=output_env)
+        # request_audio.stream(fn=get_text_request, inputs=request_audio, outputs=request_text)
+        request_audio.stop_recording(fn=get_text_request, inputs=request_audio, outputs=request_text)
+        request_text.change(fn=get_target_from_request, inputs=request_text, outputs=request_target)
+        request_target.change(fn=move_agent, inputs=request_target, outputs=[output_env, status])
+        request_audio.stop_recording(lambda: None, outputs=request_audio)
+    return demo
+
+# ---------------------------- #
+# main
+# ---------------------------- #
+demo = create_demo()
 demo.launch()

draft_1.ipynb

@@ -0,0 +1,647 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "id": "bf22c176a849df32",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T06:10:01.065321Z",
+     "start_time": "2025-04-21T06:10:01.060267Z"
+    }
+   },
+   "source": [
+    "from transformers import pipeline\n",
+    "from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, AutoTokenizer, pipeline, AutoFeatureExtractor\n",
+    "import torchaudio\n",
+    "import torchaudio.transforms as T"
+   ],
+   "outputs": [],
+   "execution_count": 32
+  },
+  {
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2025-04-21T05:03:48.582040Z",
+     "start_time": "2025-04-21T04:51:46.343821Z"
+    }
+   },
+   "cell_type": "code",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "model.safetensors:   0%|          | 0.00/151M [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "51ffb4afb57446278c28d690aa1b22e4"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/3.75k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "86143c7cd15341e39db1e81231d4fd7e"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "tokenizer_config.json:   0%|          | 0.00/283k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "01d06664af1c4c169175cd38b00fa78e"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "vocab.json:   0%|          | 0.00/836k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "8833df76fdf24e92bf51c748aa71bc48"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/2.48M [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "3f5dfd342c574c2698f42c51a567a77e"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "merges.txt:   0%|          | 0.00/494k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "e9363f15071d48878ef8230bd6c39177"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "normalizer.json:   0%|          | 0.00/52.7k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "d0e777a74b9a47f3ad6a18254825122b"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "added_tokens.json:   0%|          | 0.00/34.6k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "fd44e41876984d81a593d55e07e71be6"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "special_tokens_map.json:   0%|          | 0.00/2.19k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "c3baafd24e1c4c4d9dcaf5a4715e846e"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "preprocessor_config.json:   0%|          | 0.00/185k [00:00<?, ?B/s]"
+      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "version_major": 2,
+       "version_minor": 0,
+       "model_id": "3a671889c0504b50bcff2aec93497d78"
+      }
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use mps:0\n"
+     ]
+    }
+   ],
+   "execution_count": 4,
+   "source": [
+    "\n",
+    "pipe = pipeline(model=\"openai/whisper-tiny\", task=\"automatic-speech-recognition\")\n"
+   ],
+   "id": "initial_id"
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": [
+    "# Load audio file\n",
+    "waveform_1, sample_rate = torchaudio.load(\"sample.wav\")\n",
+    "# Target sampling rate (e.g., 16000 Hz for Whisper)\n",
+    "target_sr = 16000\n",
+    "\n",
+    "resampler = T.Resample(orig_freq=sample_rate, new_freq=target_sr, dtype=waveform.dtype)\n",
+    "waveform = resampler(waveform_1)\n",
+    "waveform_np = waveform.squeeze().numpy()\n",
+    "\n",
+    "print(waveform.shape)      # (channels, samples) — usually (1, N)\n",
+    "print(sample_rate)\n",
+    "print(waveform_np)"
+   ],
+   "id": "dc202f529230fa87"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T05:08:38.144954Z",
+     "start_time": "2025-04-21T05:08:38.087644Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "save_dir = \"./models_for_proj/whisper-tiny\"\n",
+    "device = 'cpu'\n",
+    "pipe.generation_config.save_pretrained(save_dir)\n",
+    "pipe.tokenizer.save_pretrained(save_dir)\n",
+    "pipe.feature_extractor.save_pretrained(save_dir)\n"
+   ],
+   "id": "ed09605af0b78939",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['./models_for_proj/whisper-tiny/preprocessor_config.json']"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 6
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T05:35:59.540770Z",
+     "start_time": "2025-04-21T05:35:59.476164Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "\n",
+    "# model = AutoModelForSpeechSeq2Seq.from_pretrained(save_dir, device=device)\n",
+    "# model.config.forced_decoder_ids = None\n",
+    "# processor = AutoProcessor.from_pretrained(save_dir, device=device)\n",
+    "# tokenizer = AutoTokenizer.from_pretrained(save_dir, device=device)\n",
+    "# feature_extractor = AutoFeatureExtractor.from_pretrained(save_dir, device=device)\n",
+    "# pipe = pipeline(\"automatic-speech-recognition\", model=model, tokenizer=tokenizer, feature_extractor=feature_extractor)\n",
+    "# result = pipe(\"sample.wav\")\n",
+    "# result[\"text\"]"
+   ],
+   "id": "1dcd38e5ca08781b",
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "WhisperForConditionalGeneration.__init__() got an unexpected keyword argument 'device'",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mTypeError\u001B[39m                                 Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[30]\u001B[39m\u001B[32m, line 3\u001B[39m\n\u001B[32m      1\u001B[39m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34;01mtransformers\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m AutoModelForSpeechSeq2Seq, AutoProcessor, AutoTokenizer, pipeline, AutoFeatureExtractor\n\u001B[32m      2\u001B[39m device = \u001B[33m'\u001B[39m\u001B[33mcpu\u001B[39m\u001B[33m'\u001B[39m\n\u001B[32m----> \u001B[39m\u001B[32m3\u001B[39m model = \u001B[43mAutoModelForSpeechSeq2Seq\u001B[49m\u001B[43m.\u001B[49m\u001B[43mfrom_pretrained\u001B[49m\u001B[43m(\u001B[49m\u001B[43msave_dir\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdevice\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdevice\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m      4\u001B[39m model.config.forced_decoder_ids = \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[32m      5\u001B[39m processor = AutoProcessor.from_pretrained(save_dir, device=device)\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:573\u001B[39m, in \u001B[36m_BaseAutoModelClass.from_pretrained\u001B[39m\u001B[34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001B[39m\n\u001B[32m    571\u001B[39m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28mtype\u001B[39m(config) \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mcls\u001B[39m._model_mapping.keys():\n\u001B[32m    572\u001B[39m     model_class = _get_model_class(config, \u001B[38;5;28mcls\u001B[39m._model_mapping)\n\u001B[32m--> \u001B[39m\u001B[32m573\u001B[39m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mmodel_class\u001B[49m\u001B[43m.\u001B[49m\u001B[43mfrom_pretrained\u001B[49m\u001B[43m(\u001B[49m\n\u001B[32m    574\u001B[39m \u001B[43m        \u001B[49m\u001B[43mpretrained_model_name_or_path\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43mmodel_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[43m=\u001B[49m\u001B[43mconfig\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mhub_kwargs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\n\u001B[32m    575\u001B[39m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    576\u001B[39m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[32m    577\u001B[39m     \u001B[33mf\u001B[39m\u001B[33m\"\u001B[39m\u001B[33mUnrecognized configuration class \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mconfig.\u001B[34m__class__\u001B[39m\u001B[38;5;132;01m}\u001B[39;00m\u001B[33m for this kind of AutoModel: \u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[38;5;28mcls\u001B[39m.\u001B[34m__name__\u001B[39m\u001B[38;5;132;01m}\u001B[39;00m\u001B[33m.\u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[33m\"\u001B[39m\n\u001B[32m    578\u001B[39m     \u001B[33mf\u001B[39m\u001B[33m\"\u001B[39m\u001B[33mModel type should be one of \u001B[39m\u001B[38;5;132;01m{\u001B[39;00m\u001B[33m'\u001B[39m\u001B[33m, \u001B[39m\u001B[33m'\u001B[39m.join(c.\u001B[34m__name__\u001B[39m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mfor\u001B[39;00m\u001B[38;5;250m \u001B[39mc\u001B[38;5;250m \u001B[39m\u001B[38;5;129;01min\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28mcls\u001B[39m._model_mapping.keys())\u001B[38;5;132;01m}\u001B[39;00m\u001B[33m.\u001B[39m\u001B[33m\"\u001B[39m\n\u001B[32m    579\u001B[39m )\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py:272\u001B[39m, in \u001B[36mrestore_default_torch_dtype.<locals>._wrapper\u001B[39m\u001B[34m(*args, **kwargs)\u001B[39m\n\u001B[32m    270\u001B[39m old_dtype = torch.get_default_dtype()\n\u001B[32m    271\u001B[39m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[32m--> \u001B[39m\u001B[32m272\u001B[39m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    273\u001B[39m \u001B[38;5;28;01mfinally\u001B[39;00m:\n\u001B[32m    274\u001B[39m     torch.set_default_dtype(old_dtype)\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py:4401\u001B[39m, in \u001B[36mPreTrainedModel.from_pretrained\u001B[39m\u001B[34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)\u001B[39m\n\u001B[32m   4395\u001B[39m     config = \u001B[38;5;28mcls\u001B[39m._autoset_attn_implementation(\n\u001B[32m   4396\u001B[39m         config, use_flash_attention_2=use_flash_attention_2, torch_dtype=torch_dtype, device_map=device_map\n\u001B[32m   4397\u001B[39m     )\n\u001B[32m   4399\u001B[39m \u001B[38;5;28;01mwith\u001B[39;00m ContextManagers(model_init_context):\n\u001B[32m   4400\u001B[39m     \u001B[38;5;66;03m# Let's make sure we don't run the init function of buffer modules\u001B[39;00m\n\u001B[32m-> \u001B[39m\u001B[32m4401\u001B[39m     model = \u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mconfig\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43mmodel_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mmodel_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m   4403\u001B[39m \u001B[38;5;66;03m# Make sure to tie the weights correctly\u001B[39;00m\n\u001B[32m   4404\u001B[39m model.tie_weights()\n",
+      "\u001B[31mTypeError\u001B[39m: WhisperForConditionalGeneration.__init__() got an unexpected keyword argument 'device'"
+     ]
+    }
+   ],
+   "execution_count": 30
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T06:13:00.420733Z",
+     "start_time": "2025-04-21T06:13:00.033330Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from transformers import WhisperProcessor, WhisperForConditionalGeneration\n",
+    "# load dummy dataset and read audio files\n",
+    "\n",
+    "# input\n",
+    "waveform, sample_rate = torchaudio.load(\"sample.wav\")\n",
+    "target_sr = 16000\n",
+    "resampler = T.Resample(orig_freq=sample_rate, new_freq=target_sr, dtype=waveform.dtype)\n",
+    "waveform = resampler(waveform)\n",
+    "waveform_np = waveform.squeeze().numpy()\n",
+    "\n",
+    "\n",
+    "processor = WhisperProcessor.from_pretrained(save_dir)\n",
+    "model = WhisperForConditionalGeneration.from_pretrained(save_dir)\n",
+    "model.config.forced_decoder_ids = None\n",
+    "\n",
+    "input_features = processor(waveform_np, sampling_rate=target_sr, return_tensors=\"pt\", device=device).input_features\n",
+    "\n",
+    "# generate token ids\n",
+    "predicted_ids = model.generate(input_features)\n",
+    "# decode token ids to text\n",
+    "transcription = processor.batch_decode(predicted_ids, skip_special_tokens=False)\n",
+    "# ['<|startoftranscript|><|en|><|transcribe|><|notimestamps|> Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.<|endoftext|>']\n",
+    "print(transcription)\n",
+    "transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)\n",
+    "# [' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.']\n",
+    "print(transcription)"
+   ],
+   "id": "b0865456fed26d31",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "You have explicitly specified `forced_decoder_ids`. Please remove the `forced_decoder_ids` argument in favour of `input_ids` or `decoder_input_ids` respectively.",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mValueError\u001B[39m                                Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[34]\u001B[39m\u001B[32m, line 19\u001B[39m\n\u001B[32m     16\u001B[39m input_features = processor(waveform_np, sampling_rate=target_sr, return_tensors=\u001B[33m\"\u001B[39m\u001B[33mpt\u001B[39m\u001B[33m\"\u001B[39m, device=device).input_features\n\u001B[32m     18\u001B[39m \u001B[38;5;66;03m# generate token ids\u001B[39;00m\n\u001B[32m---> \u001B[39m\u001B[32m19\u001B[39m predicted_ids = \u001B[43mmodel\u001B[49m\u001B[43m.\u001B[49m\u001B[43mgenerate\u001B[49m\u001B[43m(\u001B[49m\u001B[43minput_features\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m     20\u001B[39m \u001B[38;5;66;03m# decode token ids to text\u001B[39;00m\n\u001B[32m     21\u001B[39m transcription = processor.batch_decode(predicted_ids, skip_special_tokens=\u001B[38;5;28;01mFalse\u001B[39;00m)\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/models/whisper/generation_whisper.py:774\u001B[39m, in \u001B[36mWhisperGenerationMixin.generate\u001B[39m\u001B[34m(self, input_features, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, return_timestamps, task, language, is_multilingual, prompt_ids, prompt_condition_type, condition_on_prev_tokens, temperature, compression_ratio_threshold, logprob_threshold, no_speech_threshold, num_segment_frames, attention_mask, time_precision, time_precision_features, return_token_timestamps, return_segments, return_dict_in_generate, force_unique_generate_call, **kwargs)\u001B[39m\n\u001B[32m    765\u001B[39m             proc.set_begin_index(decoder_input_ids.shape[-\u001B[32m1\u001B[39m])\n\u001B[32m    767\u001B[39m \u001B[38;5;66;03m# 6.6 Run generate with fallback\u001B[39;00m\n\u001B[32m    768\u001B[39m (\n\u001B[32m    769\u001B[39m     seek_sequences,\n\u001B[32m    770\u001B[39m     seek_outputs,\n\u001B[32m    771\u001B[39m     should_skip,\n\u001B[32m    772\u001B[39m     do_condition_on_prev_tokens,\n\u001B[32m    773\u001B[39m     model_output_type,\n\u001B[32m--> \u001B[39m\u001B[32m774\u001B[39m ) = \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mgenerate_with_fallback\u001B[49m\u001B[43m(\u001B[49m\n\u001B[32m    775\u001B[39m \u001B[43m    \u001B[49m\u001B[43msegment_input\u001B[49m\u001B[43m=\u001B[49m\u001B[43msegment_input\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    776\u001B[39m \u001B[43m    \u001B[49m\u001B[43mdecoder_input_ids\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdecoder_input_ids\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    777\u001B[39m \u001B[43m    \u001B[49m\u001B[43mcur_bsz\u001B[49m\u001B[43m=\u001B[49m\u001B[43mcur_bsz\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    778\u001B[39m \u001B[43m    \u001B[49m\u001B[43mbatch_idx_map\u001B[49m\u001B[43m=\u001B[49m\u001B[43mbatch_idx_map\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    779\u001B[39m \u001B[43m    \u001B[49m\u001B[43mseek\u001B[49m\u001B[43m=\u001B[49m\u001B[43mseek\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    780\u001B[39m \u001B[43m    \u001B[49m\u001B[43mnum_segment_frames\u001B[49m\u001B[43m=\u001B[49m\u001B[43mnum_segment_frames\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    781\u001B[39m \u001B[43m    \u001B[49m\u001B[43mmax_frames\u001B[49m\u001B[43m=\u001B[49m\u001B[43mmax_frames\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    782\u001B[39m \u001B[43m    \u001B[49m\u001B[43mtemperatures\u001B[49m\u001B[43m=\u001B[49m\u001B[43mtemperatures\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    783\u001B[39m \u001B[43m    \u001B[49m\u001B[43mgeneration_config\u001B[49m\u001B[43m=\u001B[49m\u001B[43mgeneration_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    784\u001B[39m \u001B[43m    \u001B[49m\u001B[43mlogits_processor\u001B[49m\u001B[43m=\u001B[49m\u001B[43mlogits_processor\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    785\u001B[39m \u001B[43m    \u001B[49m\u001B[43mstopping_criteria\u001B[49m\u001B[43m=\u001B[49m\u001B[43mstopping_criteria\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    786\u001B[39m \u001B[43m    \u001B[49m\u001B[43mprefix_allowed_tokens_fn\u001B[49m\u001B[43m=\u001B[49m\u001B[43mprefix_allowed_tokens_fn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    787\u001B[39m \u001B[43m    \u001B[49m\u001B[43msynced_gpus\u001B[49m\u001B[43m=\u001B[49m\u001B[43msynced_gpus\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    788\u001B[39m \u001B[43m    \u001B[49m\u001B[43mreturn_token_timestamps\u001B[49m\u001B[43m=\u001B[49m\u001B[43mreturn_token_timestamps\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    789\u001B[39m \u001B[43m    \u001B[49m\u001B[43mdo_condition_on_prev_tokens\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdo_condition_on_prev_tokens\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    790\u001B[39m \u001B[43m    \u001B[49m\u001B[43mis_shortform\u001B[49m\u001B[43m=\u001B[49m\u001B[43mis_shortform\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    791\u001B[39m \u001B[43m    \u001B[49m\u001B[43mbatch_size\u001B[49m\u001B[43m=\u001B[49m\u001B[43mbatch_size\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    792\u001B[39m \u001B[43m    \u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[43m=\u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    793\u001B[39m \u001B[43m    \u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m=\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    794\u001B[39m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    796\u001B[39m \u001B[38;5;66;03m# 6.7 In every generated sequence, split by timestamp tokens and extract segments\u001B[39;00m\n\u001B[32m    797\u001B[39m \u001B[38;5;28;01mfor\u001B[39;00m i, seek_sequence \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28menumerate\u001B[39m(seek_sequences):\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/models/whisper/generation_whisper.py:950\u001B[39m, in \u001B[36mWhisperGenerationMixin.generate_with_fallback\u001B[39m\u001B[34m(self, segment_input, decoder_input_ids, cur_bsz, batch_idx_map, seek, num_segment_frames, max_frames, temperatures, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, return_token_timestamps, do_condition_on_prev_tokens, is_shortform, batch_size, attention_mask, kwargs)\u001B[39m\n\u001B[32m    945\u001B[39m     \u001B[38;5;28;01mif\u001B[39;00m generate_kwargs.get(\u001B[33m\"\u001B[39m\u001B[33mencoder_outputs\u001B[39m\u001B[33m\"\u001B[39m) \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[32m    946\u001B[39m         generate_kwargs[\u001B[33m\"\u001B[39m\u001B[33mencoder_outputs\u001B[39m\u001B[33m\"\u001B[39m] = F.pad(\n\u001B[32m    947\u001B[39m             generate_kwargs[\u001B[33m\"\u001B[39m\u001B[33mencoder_outputs\u001B[39m\u001B[33m\"\u001B[39m], (\u001B[32m0\u001B[39m, \u001B[32m0\u001B[39m, \u001B[32m0\u001B[39m, \u001B[32m0\u001B[39m, \u001B[32m0\u001B[39m, batch_size - cur_bsz), value=\u001B[32m0\u001B[39m\n\u001B[32m    948\u001B[39m         )\n\u001B[32m--> \u001B[39m\u001B[32m950\u001B[39m seek_outputs = \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m.\u001B[49m\u001B[43mgenerate\u001B[49m\u001B[43m(\u001B[49m\n\u001B[32m    951\u001B[39m \u001B[43m    \u001B[49m\u001B[43msegment_input\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    952\u001B[39m \u001B[43m    \u001B[49m\u001B[43mgeneration_config\u001B[49m\u001B[43m=\u001B[49m\u001B[43mgeneration_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    953\u001B[39m \u001B[43m    \u001B[49m\u001B[43mlogits_processor\u001B[49m\u001B[43m=\u001B[49m\u001B[43mlogits_processor\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    954\u001B[39m \u001B[43m    \u001B[49m\u001B[43mstopping_criteria\u001B[49m\u001B[43m=\u001B[49m\u001B[43mstopping_criteria\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    955\u001B[39m \u001B[43m    \u001B[49m\u001B[43mprefix_allowed_tokens_fn\u001B[49m\u001B[43m=\u001B[49m\u001B[43mprefix_allowed_tokens_fn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    956\u001B[39m \u001B[43m    \u001B[49m\u001B[43msynced_gpus\u001B[49m\u001B[43m=\u001B[49m\u001B[43msynced_gpus\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    957\u001B[39m \u001B[43m    \u001B[49m\u001B[43mdecoder_input_ids\u001B[49m\u001B[43m=\u001B[49m\u001B[43mdecoder_input_ids\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    958\u001B[39m \u001B[43m    \u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[43m=\u001B[49m\u001B[43mattention_mask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    959\u001B[39m \u001B[43m    \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mgenerate_kwargs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m    960\u001B[39m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    962\u001B[39m model_output_type = \u001B[38;5;28mtype\u001B[39m(seek_outputs)\n\u001B[32m    964\u001B[39m \u001B[38;5;66;03m# post-process sequence tokens and outputs to be in list form\u001B[39;00m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/torch/utils/_contextlib.py:116\u001B[39m, in \u001B[36mcontext_decorator.<locals>.decorate_context\u001B[39m\u001B[34m(*args, **kwargs)\u001B[39m\n\u001B[32m    113\u001B[39m \u001B[38;5;129m@functools\u001B[39m.wraps(func)\n\u001B[32m    114\u001B[39m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[34mdecorate_context\u001B[39m(*args, **kwargs):\n\u001B[32m    115\u001B[39m     \u001B[38;5;28;01mwith\u001B[39;00m ctx_factory():\n\u001B[32m--> \u001B[39m\u001B[32m116\u001B[39m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/generation/utils.py:2219\u001B[39m, in \u001B[36mGenerationMixin.generate\u001B[39m\u001B[34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, use_model_defaults, **kwargs)\u001B[39m\n\u001B[32m   2208\u001B[39m     warnings.warn(\n\u001B[32m   2209\u001B[39m         \u001B[33m\"\u001B[39m\u001B[33mYou are calling .generate() with the `input_ids` being on a device type different\u001B[39m\u001B[33m\"\u001B[39m\n\u001B[32m   2210\u001B[39m         \u001B[33mf\u001B[39m\u001B[33m\"\u001B[39m\u001B[33m than your model\u001B[39m\u001B[33m'\u001B[39m\u001B[33ms device. `input_ids` is on \u001B[39m\u001B[38;5;132;01m{\u001B[39;00minput_ids.device.type\u001B[38;5;132;01m}\u001B[39;00m\u001B[33m, whereas the model\u001B[39m\u001B[33m\"\u001B[39m\n\u001B[32m   (...)\u001B[39m\u001B[32m   2215\u001B[39m         \u001B[38;5;167;01mUserWarning\u001B[39;00m,\n\u001B[32m   2216\u001B[39m     )\n\u001B[32m   2218\u001B[39m \u001B[38;5;66;03m# 9. prepare logits processors and stopping criteria\u001B[39;00m\n\u001B[32m-> \u001B[39m\u001B[32m2219\u001B[39m prepared_logits_processor = \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43m_get_logits_processor\u001B[49m\u001B[43m(\u001B[49m\n\u001B[32m   2220\u001B[39m \u001B[43m    \u001B[49m\u001B[43mgeneration_config\u001B[49m\u001B[43m=\u001B[49m\u001B[43mgeneration_config\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2221\u001B[39m \u001B[43m    \u001B[49m\u001B[43minput_ids_seq_length\u001B[49m\u001B[43m=\u001B[49m\u001B[43minput_ids_length\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2222\u001B[39m \u001B[43m    \u001B[49m\u001B[43mencoder_input_ids\u001B[49m\u001B[43m=\u001B[49m\u001B[43minputs_tensor\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2223\u001B[39m \u001B[43m    \u001B[49m\u001B[43mprefix_allowed_tokens_fn\u001B[49m\u001B[43m=\u001B[49m\u001B[43mprefix_allowed_tokens_fn\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2224\u001B[39m \u001B[43m    \u001B[49m\u001B[43mlogits_processor\u001B[49m\u001B[43m=\u001B[49m\u001B[43mlogits_processor\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2225\u001B[39m \u001B[43m    \u001B[49m\u001B[43mdevice\u001B[49m\u001B[43m=\u001B[49m\u001B[43minputs_tensor\u001B[49m\u001B[43m.\u001B[49m\u001B[43mdevice\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2226\u001B[39m \u001B[43m    \u001B[49m\u001B[43mmodel_kwargs\u001B[49m\u001B[43m=\u001B[49m\u001B[43mmodel_kwargs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2227\u001B[39m \u001B[43m    \u001B[49m\u001B[43mnegative_prompt_ids\u001B[49m\u001B[43m=\u001B[49m\u001B[43mnegative_prompt_ids\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2228\u001B[39m \u001B[43m    \u001B[49m\u001B[43mnegative_prompt_attention_mask\u001B[49m\u001B[43m=\u001B[49m\u001B[43mnegative_prompt_attention_mask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[32m   2229\u001B[39m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m   2230\u001B[39m prepared_stopping_criteria = \u001B[38;5;28mself\u001B[39m._get_stopping_criteria(\n\u001B[32m   2231\u001B[39m     generation_config=generation_config, stopping_criteria=stopping_criteria, tokenizer=tokenizer, **kwargs\n\u001B[32m   2232\u001B[39m )\n\u001B[32m   2234\u001B[39m \u001B[38;5;66;03m# Set model_kwargs `use_cache` so we can use it later in forward runs\u001B[39;00m\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/generation/utils.py:1083\u001B[39m, in \u001B[36mGenerationMixin._get_logits_processor\u001B[39m\u001B[34m(self, generation_config, input_ids_seq_length, encoder_input_ids, prefix_allowed_tokens_fn, logits_processor, device, model_kwargs, negative_prompt_ids, negative_prompt_attention_mask)\u001B[39m\n\u001B[32m   1074\u001B[39m     processors.append(\n\u001B[32m   1075\u001B[39m         SuppressTokensAtBeginLogitsProcessor(\n\u001B[32m   1076\u001B[39m             generation_config.begin_suppress_tokens,\n\u001B[32m   (...)\u001B[39m\u001B[32m   1079\u001B[39m         )\n\u001B[32m   1080\u001B[39m     )\n\u001B[32m   1081\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m generation_config.forced_decoder_ids \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[32m   1082\u001B[39m     \u001B[38;5;66;03m# TODO (sanchit): move this exception to GenerationConfig.validate() when TF & FLAX are aligned with PT\u001B[39;00m\n\u001B[32m-> \u001B[39m\u001B[32m1083\u001B[39m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[32m   1084\u001B[39m         \u001B[33m\"\u001B[39m\u001B[33mYou have explicitly specified `forced_decoder_ids`. Please remove the `forced_decoder_ids` argument \u001B[39m\u001B[33m\"\u001B[39m\n\u001B[32m   1085\u001B[39m         \u001B[33m\"\u001B[39m\u001B[33min favour of `input_ids` or `decoder_input_ids` respectively.\u001B[39m\u001B[33m\"\u001B[39m,\n\u001B[32m   1086\u001B[39m     )\n\u001B[32m   1088\u001B[39m \u001B[38;5;66;03m# TODO (joao): find a strategy to specify the order of the processors\u001B[39;00m\n\u001B[32m   1089\u001B[39m processors = \u001B[38;5;28mself\u001B[39m._merge_criteria_processor_list(processors, logits_processor)\n",
+      "\u001B[31mValueError\u001B[39m: You have explicitly specified `forced_decoder_ids`. Please remove the `forced_decoder_ids` argument in favour of `input_ids` or `decoder_input_ids` respectively."
+     ]
+    }
+   ],
+   "execution_count": 34
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T06:15:41.079099Z",
+     "start_time": "2025-04-21T06:15:37.277194Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from transformers import WhisperProcessor, WhisperForConditionalGeneration\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "# load model and processor\n",
+    "processor = WhisperProcessor.from_pretrained(\"openai/whisper-tiny\")\n",
+    "model = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-tiny\")\n",
+    "model.config.forced_decoder_ids = None\n",
+    "\n",
+    "# load dummy dataset and read audio files\n",
+    "ds = load_dataset(\"hf-internal-testing/librispeech_asr_dummy\", \"clean\", split=\"validation\")\n",
+    "sample = ds[0][\"audio\"]\n",
+    "input_features = processor(sample[\"array\"], sampling_rate=sample[\"sampling_rate\"], return_tensors=\"pt\").input_features\n",
+    "\n",
+    "# generate token ids\n",
+    "predicted_ids = model.generate(input_features)\n",
+    "# decode token ids to text\n",
+    "transcription = processor.batch_decode(predicted_ids, skip_special_tokens=False)\n",
+    "processor(transcription)\n",
+    "transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)\n",
+    "processor(transcription)\n"
+   ],
+   "id": "b4137e08d1a516e5",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "It is strongly recommended to pass the `sampling_rate` argument to `WhisperFeatureExtractor()`. Failing to do so can result in silent errors that might be hard to debug.\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "could not convert string to float: ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[31m---------------------------------------------------------------------------\u001B[39m",
+      "\u001B[31mValueError\u001B[39m                                Traceback (most recent call last)",
+      "\u001B[36mCell\u001B[39m\u001B[36m \u001B[39m\u001B[32mIn[37]\u001B[39m\u001B[32m, line 18\u001B[39m\n\u001B[32m     16\u001B[39m \u001B[38;5;66;03m# decode token ids to text\u001B[39;00m\n\u001B[32m     17\u001B[39m transcription = processor.batch_decode(predicted_ids, skip_special_tokens=\u001B[38;5;28;01mFalse\u001B[39;00m)\n\u001B[32m---> \u001B[39m\u001B[32m18\u001B[39m \u001B[43mprocessor\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtranscription\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m     19\u001B[39m transcription = processor.batch_decode(predicted_ids, skip_special_tokens=\u001B[38;5;28;01mTrue\u001B[39;00m)\n\u001B[32m     20\u001B[39m processor(transcription)\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/models/whisper/processing_whisper.py:69\u001B[39m, in \u001B[36mWhisperProcessor.__call__\u001B[39m\u001B[34m(self, *args, **kwargs)\u001B[39m\n\u001B[32m     66\u001B[39m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[33m\"\u001B[39m\u001B[33mYou need to specify either an `audio` or `text` input to process.\u001B[39m\u001B[33m\"\u001B[39m)\n\u001B[32m     68\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m audio \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[32m---> \u001B[39m\u001B[32m69\u001B[39m     inputs = \u001B[38;5;28;43mself\u001B[39;49m\u001B[43m.\u001B[49m\u001B[43mfeature_extractor\u001B[49m\u001B[43m(\u001B[49m\u001B[43maudio\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msampling_rate\u001B[49m\u001B[43m=\u001B[49m\u001B[43msampling_rate\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m*\u001B[49m\u001B[43m*\u001B[49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m     70\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m text \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[32m     71\u001B[39m     encodings = \u001B[38;5;28mself\u001B[39m.tokenizer(text, **kwargs)\n",
+      "\u001B[36mFile \u001B[39m\u001B[32m~/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/models/whisper/feature_extraction_whisper.py:281\u001B[39m, in \u001B[36mWhisperFeatureExtractor.__call__\u001B[39m\u001B[34m(self, raw_speech, truncation, pad_to_multiple_of, return_tensors, return_attention_mask, padding, max_length, sampling_rate, do_normalize, device, return_token_timestamps, **kwargs)\u001B[39m\n\u001B[32m    279\u001B[39m     raw_speech = [np.asarray([speech], dtype=np.float32).T \u001B[38;5;28;01mfor\u001B[39;00m speech \u001B[38;5;129;01min\u001B[39;00m raw_speech]\n\u001B[32m    280\u001B[39m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m is_batched \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(raw_speech, np.ndarray):\n\u001B[32m--> \u001B[39m\u001B[32m281\u001B[39m     raw_speech = \u001B[43mnp\u001B[49m\u001B[43m.\u001B[49m\u001B[43masarray\u001B[49m\u001B[43m(\u001B[49m\u001B[43mraw_speech\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdtype\u001B[49m\u001B[43m=\u001B[49m\u001B[43mnp\u001B[49m\u001B[43m.\u001B[49m\u001B[43mfloat32\u001B[49m\u001B[43m)\u001B[49m\n\u001B[32m    282\u001B[39m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(raw_speech, np.ndarray) \u001B[38;5;129;01mand\u001B[39;00m raw_speech.dtype \u001B[38;5;129;01mis\u001B[39;00m np.dtype(np.float64):\n\u001B[32m    283\u001B[39m     raw_speech = raw_speech.astype(np.float32)\n",
+      "\u001B[31mValueError\u001B[39m: could not convert string to float: ' Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.'"
+     ]
+    }
+   ],
+   "execution_count": 37
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T05:31:26.352787Z",
+     "start_time": "2025-04-21T05:31:26.343398Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "",
+   "id": "37fa63b1c22f4a69",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([1, 24192])\n",
+      "24000\n",
+      "[ 0.0000000e+00  0.0000000e+00  0.0000000e+00 ... -1.3932839e-05\n",
+      " -3.6663318e-05 -1.3932839e-05]\n"
+     ]
+    }
+   ],
+   "execution_count": 25
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T06:28:40.294060Z",
+     "start_time": "2025-04-21T06:28:35.493462Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import torch\n",
+    "from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "\n",
+    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
+    "torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32\n",
+    "\n",
+    "# model_id = \"distil-whisper/distil-small.en\"\n",
+    "model_id = \"./models_for_proj/librispeech_asr_dummy\"\n",
+    "\n",
+    "model = AutoModelForSpeechSeq2Seq.from_pretrained(\n",
+    "    model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True\n",
+    ")\n",
+    "model.to(device)\n",
+    "\n",
+    "processor = AutoProcessor.from_pretrained(model_id)\n",
+    "\n",
+    "pipe = pipeline(\n",
+    "    \"automatic-speech-recognition\",\n",
+    "    model=model,\n",
+    "    tokenizer=processor.tokenizer,\n",
+    "    feature_extractor=processor.feature_extractor,\n",
+    "    max_new_tokens=128,\n",
+    "    torch_dtype=torch_dtype,\n",
+    "    device=device,\n",
+    ")\n",
+    "\n",
+    "# dataset = load_dataset(\"hf-internal-testing/librispeech_asr_dummy\", \"clean\", split=\"validation\")\n",
+    "# sample = dataset[0][\"audio\"]\n",
+    "# result = pipe(sample)\n",
+    "\n",
+    "# input\n",
+    "waveform, sample_rate = torchaudio.load(\"sample.wav\")\n",
+    "target_sr = 16000\n",
+    "resampler = T.Resample(orig_freq=sample_rate, new_freq=target_sr, dtype=waveform.dtype)\n",
+    "waveform = resampler(waveform)\n",
+    "waveform_np = waveform.squeeze().numpy()\n",
+    "# sample = dataset[2][\"audio\"]\n",
+    "\n",
+    "# result = pipe(sample)\n",
+    "result = pipe(waveform_np)\n",
+    "print(result[\"text\"])\n"
+   ],
+   "id": "e7f0a5bccb4e204f",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use cpu\n",
+      "/Users/perchik/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/pipelines/automatic_speech_recognition.py:312: FutureWarning: `max_new_tokens` is deprecated and will be removed in version 4.49 of Transformers. To remove this warning, pass `max_new_tokens` as a key inside `generate_kwargs` instead.\n",
+      "  warnings.warn(\n",
+      "/Users/perchik/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/models/whisper/generation_whisper.py:573: FutureWarning: The input name `inputs` is deprecated. Please make sure to use `input_features` instead.\n",
+      "  warnings.warn(\n",
+      "`generation_config` default values have been modified to match model-specific defaults: {'suppress_tokens': [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50357, 50358, 50359, 50360, 50361], 'begin_suppress_tokens': [220, 50256]}. If this is not desired, please set these values explicitly.\n",
+      "A custom logits processor of type <class 'transformers.generation.logits_process.SuppressTokensLogitsProcessor'> has been passed to `.generate()`, but it was also created in `.generate()`, given its parameterization. The custom <class 'transformers.generation.logits_process.SuppressTokensLogitsProcessor'> will take precedence. Please check the docstring of <class 'transformers.generation.logits_process.SuppressTokensLogitsProcessor'> to see related `.generate()` flags.\n",
+      "A custom logits processor of type <class 'transformers.generation.logits_process.SuppressTokensAtBeginLogitsProcessor'> has been passed to `.generate()`, but it was also created in `.generate()`, given its parameterization. The custom <class 'transformers.generation.logits_process.SuppressTokensAtBeginLogitsProcessor'> will take precedence. Please check the docstring of <class 'transformers.generation.logits_process.SuppressTokensAtBeginLogitsProcessor'> to see related `.generate()` flags.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " Mr. Quilter is the Apostle of the Middle Classes, and we are glad to welcome his Gospel.\n"
+     ]
+    }
+   ],
+   "execution_count": 46
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T06:27:16.239153Z",
+     "start_time": "2025-04-21T06:27:15.587609Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "save_dir = \"./models_for_proj/librispeech_asr_dummy\"\n",
+    "pipe.model.save_pretrained(save_dir)\n",
+    "pipe.tokenizer.save_pretrained(save_dir)\n",
+    "pipe.feature_extractor.save_pretrained(save_dir)"
+   ],
+   "id": "81b57090829a7294",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/perchik/PycharmProjects/Learning_LLMs/.venv/lib/python3.12/site-packages/transformers/modeling_utils.py:3353: UserWarning: Moving the following attributes in the config to the generation config: {'max_length': 448, 'suppress_tokens': [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50357, 50358, 50359, 50360, 50361]}. You are seeing this warning because you've set generation parameters in the model config, as opposed to in the generation config.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "['./models_for_proj/librispeech_asr_dummy/preprocessor_config.json']"
+      ]
+     },
+     "execution_count": 45,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 45
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T05:31:45.237137Z",
+     "start_time": "2025-04-21T05:31:45.234474Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "target_sr",
+   "id": "61b31c4b81fd098f",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "16000"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 26
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T11:20:26.931270Z",
+     "start_time": "2025-04-21T11:20:24.762498Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "# input\n",
+    "waveform, sample_rate = torchaudio.load(\"sample.wav\")\n",
+    "target_sr = 16000\n",
+    "resampler = T.Resample(orig_freq=sample_rate, new_freq=target_sr, dtype=waveform.dtype)\n",
+    "waveform = resampler(waveform)\n",
+    "waveform_np = waveform.squeeze().numpy()\n",
+    "# sample = dataset[2][\"audio\"]\n",
+    "\n",
+    "# result = pipe(sample)\n",
+    "result = pipe(waveform_np)\n",
+    "print(result[\"text\"])"
+   ],
+   "id": "5c9f9ff839e346f8",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " This is a simple text.\n"
+     ]
+    }
+   ],
+   "execution_count": 48
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-04-21T11:54:49.800197Z",
+     "start_time": "2025-04-21T11:54:47.143900Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC\n",
+    "processor = Wav2Vec2Processor.from_pretrained(\"facebook/wav2vec2-base-960h\")\n",
+    "model = Wav2Vec2ForCTC.from_pretrained(\"facebook/wav2vec2-base-960h\")"
+   ],
+   "id": "a7084d040f38e0f5",
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-base-960h and are newly initialized: ['wav2vec2.masked_spec_embed']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "execution_count": 49
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "f886807e783c9532"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

draft_2.py

@@ -1,27 +0,0 @@
-import gymnasium as gym
-
-from stable_baselines3 import PPO
-from stable_baselines3.common.env_util import make_vec_env
-import torch
-
-# Parallel environments
-vec_env = make_vec_env("CartPole-v1", n_envs=4)
-
-policy_kwargs = dict(activation_fn=torch.nn.ReLU,
-                     net_arch=dict(pi=[32, 32], vf=[32, 32]))
-model = PPO("MlpPolicy", vec_env,
-            verbose=1,
-            policy_kwargs=policy_kwargs,
-            tensorboard_log="./ppo_tensorboard/")
-model.learn(total_timesteps=100000, tb_log_name="CartPole")
-model.save("ppo_cartpole")
-
-del model # remove to demonstrate saving and loading
-
-model = PPO.load("ppo_cartpole")
-
-obs = vec_env.reset()
-while True:
-    action, _states = model.predict(obs)
-    obs, rewards, dones, info = vec_env.step(action)
-    vec_env.render("human")

draft_animation.py

@@ -0,0 +1,83 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import matplotlib.animation as animation
+import tempfile
+from matplotlib.patches import Circle
+
+
+def create_dummy_animation():
+    fig, ax = plt.subplots(figsize=(7, 7))
+    xdata, ydata = [], []
+    ln, = plt.plot([], [], 'b-', animated=True)
+
+    def init():
+        ax.set_xlim(0, 2*np.pi)
+        ax.set_ylim(-1.1, 1.1)
+        return ln,
+
+    def update(frame):
+        xdata.append(frame)
+        ydata.append(np.sin(frame))
+        ln.set_data(xdata, ydata)
+        return ln,
+
+    ani = animation.FuncAnimation(
+        fig, update, frames=np.linspace(0, 2*np.pi, 100),
+        init_func=init, blit=True, repeat=False
+    )
+
+    # Save to MP4
+    temp_video = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
+    ani.save(temp_video.name, writer='ffmpeg', fps=20)
+    plt.close(fig)
+
+    return temp_video.name
+
+
+def create_animation():
+    path = [(i,i) for i in range(50)]
+    targets_x = [20, 80, 80, 20]
+    targets_y = [20, 20, 80, 80]
+    RADIUS_COVERAGE = 10
+    fig, ax = plt.subplots(figsize=(7, 7))
+
+    # agent
+    ln1, = plt.plot([path[0][0]], [path[0][1]], marker='o', color='b', alpha=0.5, linewidth=5, markersize=15)
+
+    # targets
+    ln2, = plt.plot(targets_x, targets_y, marker='X', color='orange', alpha=0.5, linestyle='none', markersize=15)
+    for t_x, t_y in zip(targets_x, targets_y):
+        circle = Circle((t_x, t_y), RADIUS_COVERAGE, color='orange', fill=True, alpha=0.3)
+        ax.add_patch(circle)
+
+    def init():
+        ax.set_xlim([0, 100])
+        ax.set_ylim([0, 100])
+        ax.set_title(f'Warehouse Env', fontweight="bold", size=10)
+        return ln1,
+
+    def update(frame):
+        # for each frame, update the data stored on each artist.
+        x = [path[frame][0]]
+        y = [path[frame][1]]
+
+        ln1.set_data(x, y)
+        return ln1,
+
+    ani = animation.FuncAnimation(fig, update, frames=40,
+                                  init_func=init, blit=True, repeat=False)
+    # plt.show()
+
+    # Save to MP4
+    temp_video = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4")
+    ani.save(temp_video.name, writer='ffmpeg', fps=20)
+    plt.close(fig)
+    return temp_video.name
+
+
+def main():
+    create_animation()
+
+
+if __name__ == '__main__':
+    main()

draft_gradio_update_example.py

@@ -0,0 +1,34 @@
+import gradio as gr
+
+# Function to validate the input and enable/disable the button
+def validate_input(text):
+    error_msg = ""
+    button_state = False
+
+    if len(text.strip()) < 3:
+        error_msg = "_Input_ must be at least 3 characters."
+        button_state = False
+    else:
+        button_state = True
+
+    return gr.update(value=error_msg), gr.update(interactive=button_state)
+
+# Function that runs when the button is clicked
+def on_submit(text):
+    return f"Processed: {text.strip()}"
+
+with gr.Blocks() as demo:
+    gr.Markdown("### Input Validation Example")
+
+    inp = gr.Textbox(label="Enter something")
+    validation = gr.Label(value="", visible=True)
+    btn = gr.Button("Submit", interactive=False)
+    out = gr.Textbox(label="Output", interactive=False)
+
+    # When the input changes, validate it and enable/disable the button
+    inp.change(validate_input, inputs=inp, outputs=[validation, btn])
+
+    # When the button is clicked, process the input
+    btn.click(on_submit, inputs=inp, outputs=out)
+
+demo.launch()

requirements.txt

@@ -1 +1,11 @@
 gradio
+
+numpy
+matplotlib
+torch
+torchaudio
+transformers
+stable_baselines3
+gymnasium
+vmas
+datasets

train_agent.py

@@ -37,7 +37,7 @@ def train_func(alg_name='PPO'):
 
 
 
-def exec_func(alg_name='PPO', model_name=None):
+def exec_func(alg_name='SAC', model_name=None):
     env = WarehouseEnv(render_mode='human')
     if alg_name == 'PPO':
         model_name = "ppo_warehouse" if model_name is None else model_name
@@ -50,7 +50,7 @@ def exec_func(alg_name='PPO', model_name=None):
     # vec_env = model.get_env()
     obs, info = env.reset()
     while True:
-        action, _states = model.predict(obs)
+        action, _ = model.predict(obs)
         obs, rewards, done, trunc, info = env.step(action)
         env.render()
         if done or trunc:
@@ -60,7 +60,7 @@ def exec_func(alg_name='PPO', model_name=None):
 def main():
     # alg_name = 'PPO'
     alg_name = 'SAC'
-    model_name = 'sac_warehouse_working_v1'
+    model_name = 'agent_policies/sac_warehouse_r_10_working_v1.zip'
     # train_func(alg_name)
     exec_func(alg_name=alg_name, model_name=model_name)
 

warehouse_env.py

@@ -55,13 +55,21 @@ class WarehouseEnv(gym.Env):
     def rel_y(self) -> int:
         return self.agent_y - self.goal_y
 
-    def reset(self, seed=None, options=None):
-        self.agent_x = np.random.uniform(0, self.SIDE)
-        self.agent_y = np.random.uniform(0, self.SIDE)
-        # self.agent_x = 50.0
-        # self.agent_y = 50.0
-        self.goal_x = np.random.uniform(0, self.SIDE)
-        self.goal_y = np.random.uniform(0, self.SIDE)
+    def reset(self, seed=None, options=None, agent_x=None, agent_y=None, goal_x=None, goal_y=None):
+        if agent_x is None:
+            self.agent_x = np.random.uniform(0, self.SIDE)
+            self.agent_y = np.random.uniform(0, self.SIDE)
+            # self.agent_x = 50.0
+            # self.agent_y = 50.0
+        else:
+            self.agent_x = agent_x
+            self.agent_y = agent_y
+        if goal_x is None:
+            self.goal_x = np.random.uniform(0, self.SIDE)
+            self.goal_y = np.random.uniform(0, self.SIDE)
+        else:
+            self.goal_x = goal_x
+            self.goal_y = goal_y
         self.step_counter = 0
         self.terminated = False
         self.truncated = False