feat: terramind_synthesis now routes through droplet remote inference

Synthesis (DEM -> generative LULC via TerraMind v1 base) was the only
EO specialist still local-only. On the HF Space, terratorch's import
chain crashes with `RuntimeError: operator torchvision::nms does not
exist` because torchvision's C extension can't load against our CPU
torch wheel — so synthesis returned a clean skip rather than running.

Add a remote dispatch path:

- services/riprap-models/main.py: extend `_TERRAMIND_SPECS` with the
`synthesis` entry (10 ESRI LULC labels), add `_load_terramind_synthesis`
that pulls the v1 base generate model from terratorch's
FULL_MODEL_REGISTRY, and `_terramind_synthesis_inference` that
takes a 4-D (B, 1, H, W) DEM and emits class fractions matching the
local code's response shape. Same /v1/terramind dispatch handles
all three adapter names now (lulc / buildings / synthesis).

- TerramindIn schema: make all modality fields optional. lulc/buildings
still require s2; synthesis requires dem; the dispatch enforces.

- app/inference.py: terramind() accepts None for s2l2a so the synthesis
call site doesn't fabricate a placeholder S2 chip.

- app/context/terramind_synthesis.py: try the remote path first via
app.inference.terramind('synthesis', dem=...). Local path is kept as
a fallback for environments where terratorch DOES load.

Hot-patched on the droplet (docker cp + docker restart). Source
commit so scripts/deploy_droplet.sh picks it up next bring-up.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

app/context/terramind_synthesis.py

@@ -273,8 +273,6 @@ def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
     """
     if not ENABLE:
         return {"ok": False, "skipped": "RIPRAP_TERRAMIND_ENABLE=0"}
-    if not _DEPS_OK:
-        return {"ok": False, "skipped": f"deps unavailable: {_DEPS_MISSING}"}
     t0 = time.time()
     try:
         import numpy as np
@@ -284,6 +282,58 @@ def fetch(lat: float, lon: float, timeout_s: float = 60.0) -> dict[str, Any]:
         dem, bounds_4326 = patch
         dem_mean = float(dem.mean())
 
+        # v0.4.5+ — try the MI300X inference service first if configured.
+        # The droplet's /v1/terramind dispatch handles adapter='synthesis'
+        # via _terramind_synthesis_inference (DEM -> generative LULC). On
+        # the HF Space terratorch's torchvision binary doesn't load, so
+        # this is the only working path there.
+        try:
+            from app import inference as _inf
+            if _inf.remote_enabled():
+                # Local code uses (1, 1, H, W); send the same shape.
+                dem_remote = dem[None, None, :, :].astype("float32")
+                remote = _inf.terramind("synthesis", None, None, dem_remote,
+                                          timeout=timeout_s)
+                if remote.get("ok"):
+                    elapsed = round(time.time() - t0, 2)
+                    out = {
+                        "ok": True,
+                        "synthetic_modality": True,
+                        "tim_chain": ["DEM", "LULC_synthetic"],
+                        "diffusion_steps": remote.get("diffusion_steps",
+                                                       DEFAULT_STEPS),
+                        "diffusion_seed": DEFAULT_SEED,
+                        "dem_mean_m": round(dem_mean, 2),
+                        "class_fractions": remote.get("class_fractions") or {},
+                        "dominant_class": remote.get("dominant_class") or "unknown",
+                        "dominant_pct": remote.get("dominant_pct") or 0.0,
+                        "n_classes_observed": remote.get("n_classes_observed") or 0,
+                        "chip_shape": remote.get("shape") or [],
+                        "bounds_4326": list(bounds_4326),
+                        "polygons_geojson": None,
+                        "label_schema": remote.get("label_schema") or "",
+                        "compute": f"remote · {remote.get('device', 'gpu')}",
+                        "elapsed_s": elapsed,
+                    }
+                    return out
+                # remote returned non-ok — surface that signal directly
+                return {"ok": False,
+                        "skipped": f"remote terramind synthesis non-ok: "
+                                   f"{remote.get('error') or remote.get('detail') or 'unknown'}",
+                        "elapsed_s": round(time.time() - t0, 2)}
+        except _inf.RemoteUnreachable as e:
+            log.info("terramind_synthesis: remote unreachable (%s); local fallback", e)
+        except Exception as e:
+            log.exception("terramind_synthesis: remote call failed")
+            return {"ok": False,
+                    "skipped": f"remote terramind synthesis error: "
+                               f"{type(e).__name__}: {e}",
+                    "elapsed_s": round(time.time() - t0, 2)}
+
+        # Local fallback — original path; only available where terratorch
+        # imports without the torchvision::nms RuntimeError.
+        if not _DEPS_OK:
+            return {"ok": False, "skipped": f"deps unavailable: {_DEPS_MISSING}"}
         import torch
         random.seed(DEFAULT_SEED)
         torch.manual_seed(DEFAULT_SEED)

app/inference.py

@@ -167,15 +167,18 @@ def prithvi_pluvial(s2_chip, *, scene_id: str | None = None,
     }, timeout=timeout)
 
 
-def terramind(adapter: str, s2l2a, s1rtc=None, dem=None, *,
+def terramind(adapter: str, s2l2a=None, s1rtc=None, dem=None, *,
                timeout: float | None = None) -> dict[str, Any]:
     """Remote forward through TerraMind-NYC-Adapters (LULC or Buildings)
-    or the v1 base (synthetic).  `adapter` is one of: lulc, buildings,
-    synthesis. Each modality is a numpy array or None."""
+    or the v1 base generative path (synthesis). `adapter` is one of:
+    lulc, buildings, synthesis. Each modality is a numpy array, torch
+    tensor, or None — `synthesis` only needs DEM; the LoRA adapters
+    need at minimum S2L2A."""
     payload: dict[str, Any] = {"adapter": adapter}
-    s2_np = _to_numpy(s2l2a)
-    payload["s2"] = _serialize_array(s2_np)
-    payload["s2_shape"] = list(s2_np.shape)
+    if s2l2a is not None:
+        s2_np = _to_numpy(s2l2a)
+        payload["s2"] = _serialize_array(s2_np)
+        payload["s2_shape"] = list(s2_np.shape)
     if s1rtc is not None:
         s1_np = _to_numpy(s1rtc)
         payload["s1"] = _serialize_array(s1_np)

services/riprap-models/main.py

@@ -222,10 +222,54 @@ _TERRAMIND_SPECS = {
                    "labels": ["Trees", "Cropland", "Built", "Bare", "Water"]},
     "buildings": {"subdir": "buildings_nyc", "num_classes": 2,
                    "labels": ["Background", "Building"]},
+    # Synthesis is the IBM/NASA base TerraMind generative path
+    # (DEM -> LULC), not a NYC fine-tune. Listed here so the same
+    # /v1/terramind dispatch handles it.
+    "synthesis": {"subdir": None, "num_classes": None,
+                   "labels": ["Water", "Trees", "Grass", "Flooded vegetation",
+                              "Crops", "Scrub/Shrub", "Built", "Bare ground",
+                              "Snow/Ice", "Clouds"]},
 }
+_TERRAMIND_SYNTH_TIMESTEPS = int(os.environ.get(
+    "RIPRAP_TERRAMIND_SYNTH_TIMESTEPS", "10"))
+
+
+def _load_terramind_synthesis():
+    """Load the IBM/NASA base TerraMind v1 generative path
+    (DEM -> LULC) once. Different machinery from the LoRA adapters:
+    pulled via terratorch's FULL_MODEL_REGISTRY rather than
+    SemanticSegmentationTask + LoRA injection."""
+    key = "terramind_synthesis"
+    if key in _INSTANCES:
+        return _INSTANCES[key]
+    with _LOCKS.get(key, _LOCKS.get("terramind_lulc")):
+        if key in _INSTANCES:
+            return _INSTANCES[key]
+        log.info("terramind/synthesis: cold load (v1 base generate)")
+        import terratorch.models.backbones.terramind.model.terramind_register  # noqa
+        from terratorch.registry import FULL_MODEL_REGISTRY
+        m = FULL_MODEL_REGISTRY.build(
+            "terratorch_terramind_v1_base_generate",
+            modalities=["DEM"],
+            output_modalities=["LULC"],
+            pretrained=True,
+            timesteps=_TERRAMIND_SYNTH_TIMESTEPS,
+        )
+        try:
+            import torch
+            if _DEVICE == "cuda" and torch.cuda.is_available():
+                m = m.to("cuda")
+        except Exception:
+            log.exception("terramind/synthesis: cuda move failed")
+        m.eval()
+        _INSTANCES[key] = m
+        log.info("terramind/synthesis: ready")
+        return m
 
 
 def _load_terramind(adapter: str):
+    if adapter == "synthesis":
+        return _load_terramind_synthesis()
     key = f"terramind_{adapter}"
     if key in _INSTANCES:
         return _INSTANCES[key]
@@ -291,8 +335,10 @@ def _load_terramind(adapter: str):
 
 class TerramindIn(BaseModel):
     adapter: str  # "lulc" | "buildings" | "synthesis"
-    s2: str
-    s2_shape: list[int]
+    # All modality fields optional — `synthesis` adapter only needs DEM,
+    # while lulc / buildings need at minimum S2L2A.
+    s2: str | None = None
+    s2_shape: list[int] | None = None
     s1: str | None = None
     s1_shape: list[int] | None = None
     dem: str | None = None
@@ -319,14 +365,84 @@ def _build_chip_tensor(np_arr, n_timesteps: int = 4):
     raise ValueError(f"unexpected chip shape {tuple(t.shape)}")
 
 
-def _terramind_inference(payload: TerramindIn) -> dict[str, Any]:
+def _terramind_synthesis_inference(payload: TerramindIn) -> dict[str, Any]:
+    """DEM -> LULC generative path. Different machinery from the LoRA
+    adapters: model is the v1 base generate stack pulled from
+    terratorch's FULL_MODEL_REGISTRY, takes a single 4-D (B, 1, H, W)
+    DEM tensor, and emits a class-logit raster keyed by the ESRI
+    2020 LULC tokenizer codebook."""
     t0 = time.time()
+    if not payload.dem or not payload.dem_shape:
+        raise HTTPException(status_code=400,
+                            detail="synthesis requires `dem` + `dem_shape`")
+    model = _load_terramind_synthesis()
+    dem_np = _decode_array(payload.dem, payload.dem_shape)
+
+    import numpy as np
+    import torch
+    dem_t = torch.from_numpy(dem_np).float()
+    # Accept (H, W), (1, H, W), or (1, 1, H, W) — the local code builds
+    # (1, 1, H, W) so that's the most common.
+    while dem_t.ndim < 4:
+        dem_t = dem_t.unsqueeze(0)
+    dem_t = _to_device(dem_t)
+
+    spec = _TERRAMIND_SPECS["synthesis"]
+    with torch.no_grad():
+        out = model({"DEM": dem_t},
+                    timesteps=_TERRAMIND_SYNTH_TIMESTEPS,
+                    verbose=False)
+    lulc = out["LULC"]
+    if hasattr(lulc, "detach"):
+        lulc = lulc.detach().cpu().numpy()
+    if lulc.ndim == 4:
+        lulc = lulc[0]                      # (n_classes, H, W)
+    class_idx = lulc.argmax(axis=0)         # (H, W) per-pixel class
+    unique, counts = np.unique(class_idx, return_counts=True)
+    total = float(class_idx.size) or 1.0
+    fractions: dict[str, float] = {}
+    for u, c in zip(unique, counts):
+        u = int(u)
+        label = spec["labels"][u] if 0 <= u < len(spec["labels"]) else f"class_{u}"
+        fractions[label] = round(100.0 * c / total, 2)
+    ordered = dict(sorted(fractions.items(),
+                           key=lambda kv: kv[1], reverse=True))
+    dominant_class = next(iter(ordered)) if ordered else "unknown"
+    dominant_pct = ordered.get(dominant_class, 0.0)
+    return {
+        "ok": True,
+        "adapter": "synthesis",
+        "elapsed_s": round(time.time() - t0, 2),
+        "device": _DEVICE,
+        "synthetic_modality": True,
+        "tim_chain": ["DEM", "LULC_synthetic"],
+        "diffusion_steps": _TERRAMIND_SYNTH_TIMESTEPS,
+        "class_fractions": ordered,
+        "dominant_class": dominant_class,
+        "dominant_pct": dominant_pct,
+        "n_classes_observed": len(ordered),
+        "shape": list(lulc.shape),
+        "n_pixels": int(class_idx.size),
+        "label_schema": "ESRI 2020-2022 Land Cover (tentative — TerraMind "
+                         "tokenizer source confirms ESRI but not exact "
+                         "label-to-index mapping)",
+    }
+
+
+def _terramind_inference(payload: TerramindIn) -> dict[str, Any]:
     if payload.adapter not in _TERRAMIND_SPECS:
         raise HTTPException(status_code=400,
                             detail=f"unknown adapter {payload.adapter!r}")
+    if payload.adapter == "synthesis":
+        return _terramind_synthesis_inference(payload)
+    t0 = time.time()
     task = _load_terramind(payload.adapter)
     spec = _TERRAMIND_SPECS[payload.adapter]
 
+    if not payload.s2 or not payload.s2_shape:
+        raise HTTPException(status_code=400,
+                            detail=f"adapter {payload.adapter!r} requires "
+                                   f"`s2` + `s2_shape`")
     s2 = _decode_array(payload.s2, payload.s2_shape)
     chips = {"S2L2A": _to_device(_build_chip_tensor(s2))}
     if payload.s1 and payload.s1_shape: