reduce buffer duration and adjust transcription parameters for lower latency

app.py

@@ -43,7 +43,8 @@ def get_device_and_dtype():
 # Globals / constants
 # -------------------------
 SAMPLE_RATE = 16000
-BUFFER_DURATION = 6  # seconds
+# Keep a shorter rolling buffer to reduce model input length and latency.
+BUFFER_DURATION = 4  # seconds
 
 # VAD (webrtcvad)
 vad = webrtcvad.Vad(2)  # aggressiveness 0-3 (tune as needed)
@@ -214,8 +215,8 @@ def stream_transcribe(audio, state):
             buffer = buffer[-max_len:]
         state["buffer"] = buffer
 
-        # If buffer too short, wait
-        if buffer.shape[0] < int(0.5 * SAMPLE_RATE):
+        # If buffer too short, wait (very short context tends to be unstable)
+        if buffer.shape[0] < int(0.3 * SAMPLE_RATE):
             return full_transcript, state
 
         # --- VAD: find voiced segments and pick the latest one ---
@@ -231,7 +232,7 @@ def stream_transcribe(audio, state):
         segment_audio = buffer[s:e]
 
         # If segment too short skip
-        if len(segment_audio) < int(0.25 * SAMPLE_RATE):
+        if len(segment_audio) < int(0.15 * SAMPLE_RATE):
             return full_transcript, state
 
         # Process ONLY the voiced segment
@@ -242,8 +243,8 @@ def stream_transcribe(audio, state):
         with torch.no_grad():
             predicted_ids = model.generate(
                 input_features,
-                max_new_tokens=128,          # shorter prevents hallucinated continuation
-                num_beams=5,                 # stronger beam search for stability
+                max_new_tokens=64,           # shorter prevents long continuations (lower latency)
+                num_beams=1,                 # single beam for faster decoding
                 no_repeat_ngram_size=4,      # block repeated loops
                 repetition_penalty=1.3,      # punish repeated phrasing
                 length_penalty=0.7,          # prefer shorter, factual outputs