Pad short series for Toto + persist forecast log to HF Dataset

- src/forecast.py: when context is shorter than one Toto patch (32), left-
pad with the first value and set target_mask False on padded steps so
the model ignores them. Fixes the 4-hour cadence dropdown error when the
station has fewer than 32 four-hour points (~5 days of data).
- src/persist.py: pull/push the forecast SQLite to a private HF Dataset
(default bitsofchris/toto-weather-forecast-log) so the scoreboard
survives Space rebuilds. Push is locked + coalesced (60s min interval),
fired async after each refresh; pull runs once at startup.
- app.py: call persist.pull_db() on launch and persist.push_db_async()
at the end of refresh().

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

app.py

@@ -17,7 +17,7 @@ from datetime import datetime, timedelta, timezone
 import gradio as gr
 import pandas as pd
 
-from src import ecowitt, forecast_log, nws
+from src import ecowitt, forecast_log, nws, persist
 from src.forecast import forecast_series
 from src.weather_ui import (
     combined_figure,
@@ -154,6 +154,9 @@ def refresh(cycle_label: str = "Hourly", horizon_label: str = "24 h"):
     strip = emoji_strip_markdown(nws_df_raw, DISPLAY_TZ, n=12)
     scoreboard = render_scoreboard(log_conn)
 
+    # Backup the SQLite log to the HF dataset (non-blocking).
+    persist.push_db_async()
+
     return hero, toto_md, nws_md, strip, fig, scoreboard
 
 
@@ -252,5 +255,6 @@ with gr.Blocks(title="Toto Weather Forecast", theme=gr.themes.Soft()) as demo:
 
 
 if __name__ == "__main__":
+    persist.pull_db()  # bootstrap the forecast log from the HF Dataset
     _start_autorefresh()
     demo.launch()

src/forecast.py

@@ -87,23 +87,32 @@ def forecast_series(
     if series.empty:
         raise ValueError("Cannot forecast an empty series")
 
+    import numpy as np  # noqa: PLC0415
+
     clean = series.astype(float).interpolate(limit_direction="both")
 
     # Toto requires the context length to be a multiple of the model's
-    # patch_size (32 for Toto-2.0-4m). Truncate the oldest points to fit.
+    # patch_size (32 for Toto-2.0-4m). If we have at least one full patch,
+    # truncate the oldest points to fit. If we have fewer, left-pad with the
+    # first value and mark the padded region False in the mask so Toto
+    # ignores it.
     model = load_model(model_id, device=device)
     patch = int(model.config.patch_size)
-    n = (len(clean) // patch) * patch
-    if n < patch:
-        raise ValueError(
-            f"Need at least {patch} points (got {len(clean)}); "
-            "fetch a longer history window."
-        )
-    clean = clean.iloc[-n:]
-    arr = clean.to_numpy(dtype=np.float32)
+    raw = clean.to_numpy(dtype=np.float32)
+    n_raw = len(raw)
+
+    if n_raw >= patch:
+        n = (n_raw // patch) * patch
+        arr = raw[-n:]
+        mask_vec = np.ones(n, dtype=bool)
+    else:
+        n = patch
+        pad = n - n_raw
+        arr = np.concatenate([np.full(pad, raw[0], dtype=np.float32), raw])
+        mask_vec = np.concatenate([np.zeros(pad, dtype=bool), np.ones(n_raw, dtype=bool)])
 
     target = torch.from_numpy(arr).unsqueeze(0).unsqueeze(0)  # (1, 1, T)
-    target_mask = torch.ones_like(target, dtype=torch.bool)
+    target_mask = torch.from_numpy(mask_vec).unsqueeze(0).unsqueeze(0)
     series_ids = torch.zeros(1, 1, dtype=torch.long)
 
     target = target.to(device)

src/persist.py

@@ -0,0 +1,103 @@
+"""Persist the forecast SQLite DB across HF Space rebuilds.
+
+HF Spaces' free tier has ephemeral storage — every `git push` rebuilds the
+container and wipes any local files. We back the forecast log with a
+private HF Dataset:
+
+  - On startup: pull the latest forecasts.db from the dataset (if any).
+  - After every refresh: push the current forecasts.db back.
+
+Environment:
+  HF_TOKEN          must have write access to the dataset
+  LOG_DATASET_REPO  override the default dataset repo id
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import threading
+import time
+import traceback
+
+DEFAULT_REPO = "bitsofchris/toto-weather-forecast-log"
+PATH_IN_REPO = "forecasts.db"
+DEFAULT_LOCAL = "data/forecasts.db"
+
+_push_lock = threading.Lock()
+_last_push_at = 0.0
+PUSH_MIN_INTERVAL = 60.0  # seconds — coalesce rapid pushes
+
+
+def _repo_id() -> str:
+    return os.environ.get("LOG_DATASET_REPO", DEFAULT_REPO)
+
+
+def _token() -> str | None:
+    return os.environ.get("HF_TOKEN")
+
+
+def pull_db(local_path: str = DEFAULT_LOCAL) -> bool:
+    """Download the latest DB from the dataset, overwriting any local copy.
+    Returns True on success."""
+    tok = _token()
+    if not tok:
+        print("[persist] HF_TOKEN not set — skipping pull")
+        return False
+    try:
+        from huggingface_hub import hf_hub_download  # noqa: PLC0415
+        downloaded = hf_hub_download(
+            repo_id=_repo_id(),
+            repo_type="dataset",
+            filename=PATH_IN_REPO,
+            token=tok,
+        )
+        os.makedirs(os.path.dirname(local_path) or ".", exist_ok=True)
+        shutil.copyfile(downloaded, local_path)
+        print(f"[persist] pulled DB from {_repo_id()} ({os.path.getsize(local_path)} bytes)")
+        return True
+    except Exception:  # noqa: BLE001
+        print(f"[persist] pull skipped (no remote DB or network error):")
+        traceback.print_exc()
+        return False
+
+
+def push_db(local_path: str = DEFAULT_LOCAL) -> bool:
+    """Upload the local DB to the dataset. Coalesced and lock-protected so
+    overlapping refreshes don't issue redundant uploads."""
+    global _last_push_at
+    tok = _token()
+    if not tok or not os.path.exists(local_path):
+        return False
+
+    # Coalesce: if we just pushed, skip.
+    if time.time() - _last_push_at < PUSH_MIN_INTERVAL:
+        return False
+    if not _push_lock.acquire(blocking=False):
+        return False
+    try:
+        from huggingface_hub import HfApi  # noqa: PLC0415
+        api = HfApi(token=tok)
+        api.upload_file(
+            path_or_fileobj=local_path,
+            path_in_repo=PATH_IN_REPO,
+            repo_id=_repo_id(),
+            repo_type="dataset",
+            commit_message="forecast log update",
+        )
+        _last_push_at = time.time()
+        print(f"[persist] pushed DB to {_repo_id()} ({os.path.getsize(local_path)} bytes)")
+        return True
+    except Exception:  # noqa: BLE001
+        print("[persist] push failed:")
+        traceback.print_exc()
+        return False
+    finally:
+        _push_lock.release()
+
+
+def push_db_async(local_path: str = DEFAULT_LOCAL) -> None:
+    """Fire-and-forget push so refresh() returns to the user immediately."""
+    threading.Thread(
+        target=push_db, args=(local_path,), daemon=True, name="persist-push"
+    ).start()