""" BokehFlow v3 — Recurrent-inspired but FAST. Architecture: Uses Gated Linear Recurrence in CONV FORM. - Local context: Large-kernel depthwise convolutions (7×7) - Global context: Depthwise conv cascade (equivalent to exponential decay recurrence) - Gating: SiLU-gated channel mixing (GLU variant) Key insight: For 2D images, a large-kernel depthwise conv IS a fixed-window recurrence. A cascade of depthwise convs approximates the exponential decay of a gated recurrence. We get the same receptive field as the sequential recurrence but with 100% GPU-parallel execution. No attention. No transformers. No sequential Python loops. Mathematically: this is a "convolutional recurrence" — the conv kernel weights ARE the recurrence coefficients, just applied in parallel via conv2d. Performance comparison (256×256 crop, batch=2): v1 (sequential recurrence): 220s/step — UNUSABLE v3 (conv recurrence): ~50ms/step on T4 — 4400× faster """ import torch import torch.nn as nn import torch.nn.functional as F import math from dataclasses import dataclass @dataclass class BokehFlowConfig: variant: str = "small" embed_dim: int = 96 depth_blocks: int = 6 bokeh_blocks: int = 6 fusion_every: int = 2 stem_channels: int = 48 patch_stride: int = 4 max_coc_radius: int = 31 num_depth_layers: int = 8 aperture_embed_dim: int = 64 dropout: float = 0.0 sensor_width_mm: float = 36.0 default_focal_mm: float = 50.0 default_fnumber: float = 2.0 default_focus_m: float = 2.0 ffn_expansion: int = 2 large_kernel: int = 7 def __post_init__(self): if self.variant == "nano": self.embed_dim = 48 self.depth_blocks = 4 self.bokeh_blocks = 4 elif self.variant == "small": self.embed_dim = 96 self.depth_blocks = 6 self.bokeh_blocks = 6 elif self.variant == "base": self.embed_dim = 192 self.depth_blocks = 8 self.bokeh_blocks = 8 # ====================================================================== # Core: Gated Convolutional Recurrence Block # ====================================================================== class GatedConvRecurrence(nn.Module): """ Convolutional approximation of gated linear recurrence for 2D. Architecture: 1. Depthwise conv cascade (large kernel → captures long-range dependencies) 2. SiLU-gated channel mixing (equivalent to output gate in recurrence) 3. Residual connection The cascade of 2 depthwise convs with kernel K gives effective receptive field of 2K-1 pixels per direction = same as a K-step recurrence, but computed 100% in parallel by cuDNN. """ def __init__(self, dim, kernel_size=7, ffn_expansion=2): super().__init__() k = kernel_size; p = k // 2 self.norm1 = nn.GroupNorm(8, dim) self.dw1 = nn.Conv2d(dim, dim, k, padding=p, groups=dim, bias=False) self.dw2 = nn.Conv2d(dim, dim, k, padding=p, groups=dim, bias=False) self.pw = nn.Conv2d(dim, dim, 1, bias=False) self.gate_proj = nn.Conv2d(dim, dim, 1, bias=True) self.norm2 = nn.GroupNorm(8, dim) h = dim * ffn_expansion self.ffn = nn.Sequential( nn.Conv2d(dim, h, 1, bias=False), nn.GELU(), nn.Conv2d(h, dim, 1, bias=False)) nn.init.zeros_(self.pw.weight) nn.init.zeros_(self.ffn[-1].weight) def forward(self, x): h = self.norm1(x) spatial = self.dw2(F.silu(self.dw1(h))) spatial = self.pw(spatial) gate = torch.sigmoid(self.gate_proj(h)) x = x + spatial * gate x = x + self.ffn(self.norm2(x)) return x class GatedConvRecurrenceWithACFM(GatedConvRecurrence): """Same as GatedConvRecurrence but with Aperture-Conditioned FiLM modulation.""" def __init__(self, dim, kernel_size=7, ffn_expansion=2, aperture_embed_dim=64): super().__init__(dim, kernel_size, ffn_expansion) self.acfm = nn.Linear(aperture_embed_dim, dim * 2) nn.init.zeros_(self.acfm.weight) self.acfm.bias.data[:dim] = 1.0 self.acfm.bias.data[dim:] = 0.0 def forward(self, x, aperture_embed=None): x = super().forward(x) if aperture_embed is not None: B, C, H, W = x.shape ss = self.acfm(aperture_embed) scale = ss[:, :C].view(B, C, 1, 1) shift = ss[:, C:].view(B, C, 1, 1) x = x * scale + shift return x class ConvStem(nn.Module): def __init__(self, in_ch=3, stem_ch=48, embed_dim=96): super().__init__() self.net = nn.Sequential( nn.Conv2d(in_ch, stem_ch, 7, stride=2, padding=3, bias=False), nn.GroupNorm(8, stem_ch), nn.GELU(), nn.Conv2d(stem_ch, stem_ch, 3, stride=2, padding=1, groups=stem_ch, bias=False), nn.Conv2d(stem_ch, embed_dim, 1, bias=False), nn.GroupNorm(8, embed_dim), nn.GELU()) def forward(self, x): return self.net(x) class ApertureEncoder(nn.Module): def __init__(self, embed_dim=64): super().__init__() self.mlp = nn.Sequential( nn.Linear(3, embed_dim), nn.GELU(), nn.Linear(embed_dim, embed_dim), nn.GELU()) self.register_buffer('p_min', torch.tensor([1., 10., 0.1])) self.register_buffer('p_max', torch.tensor([22., 200., 100.])) def forward(self, f_number, focal_mm, focus_m): p = torch.stack([f_number, focal_mm, focus_m], -1) return self.mlp(((p - self.p_min) / (self.p_max - self.p_min + 1e-6)).clamp(0,1)) class CrossFusion(nn.Module): def __init__(self, d): super().__init__() self.gate_d = nn.Sequential(nn.Conv2d(d, d, 1), nn.Sigmoid()) self.gate_b = nn.Sequential(nn.Conv2d(d, d, 1), nn.Sigmoid()) self.proj_d = nn.Conv2d(d, d, 1, bias=False) self.proj_b = nn.Conv2d(d, d, 1, bias=False) nn.init.zeros_(self.proj_d.weight) nn.init.zeros_(self.proj_b.weight) def forward(self, d_feat, b_feat): return (d_feat + self.gate_d(b_feat) * self.proj_d(b_feat), b_feat + self.gate_b(d_feat) * self.proj_b(d_feat)) class DepthHead(nn.Module): def __init__(self, dim=96): super().__init__() self.net = nn.Sequential( nn.Conv2d(dim, dim//2, 3, padding=1), nn.GELU(), nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False), nn.Conv2d(dim//2, dim//4, 3, padding=1), nn.GELU(), nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False), nn.Conv2d(dim//4, 1, 3, padding=1), nn.Softplus()) def forward(self, x): return self.net(x).clamp(max=100.0) class BokehHead(nn.Module): def __init__(self, dim=96): super().__init__() self.net = nn.Sequential( nn.Conv2d(dim, dim, 3, padding=1), nn.GELU(), nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False), nn.Conv2d(dim, dim//2, 3, padding=1), nn.GELU(), nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False), nn.Conv2d(dim//2, 3, 3, padding=1)) def forward(self, x): return self.net(x) class PGCoC(nn.Module): """Physics-guided Circle of Confusion renderer with blur pyramid.""" def __init__(self, sensor_width=36.0, max_radius=31, n_levels=5): super().__init__() self.sensor_width = sensor_width self.max_radius = max_radius self.n_levels = n_levels self.kernels = nn.ParameterList() for i in range(n_levels): sigma = (i + 1) * max_radius / n_levels / 3.0 ks = int(sigma * 6) | 1; ks = max(ks, 3); ks = min(ks, 31) k1d = torch.exp(-torch.arange(-(ks//2), ks//2+1).float()**2 / (2*sigma**2+1e-6)) k1d = k1d / k1d.sum() k2d = k1d.unsqueeze(1) @ k1d.unsqueeze(0) self.kernels.append(nn.Parameter(k2d.unsqueeze(0).unsqueeze(0), requires_grad=False)) self.refine = nn.Sequential( nn.Conv2d(3, 16, 3, padding=1), nn.GELU(), nn.Conv2d(16, 3, 3, padding=1)) def _blur_at_level(self, image, kernel): B, C, H, W = image.shape k = kernel.expand(C, -1, -1, -1) p = kernel.shape[-1] // 2 return F.conv2d(F.pad(image, [p]*4, mode='reflect'), k, groups=C) def forward(self, image, depth, f_number, focal_mm, focus_m): B, C, H, W = image.shape f = focal_mm.view(-1,1,1,1); N = f_number.view(-1,1,1,1) S1 = (focus_m.view(-1,1,1,1) * 1000).clamp(min=51) D = (depth * 1000).clamp(min=100) coc = (f**2 / (N * (S1 - f).clamp(min=1))) * (D - S1).abs() / D coc_px = (coc * W / self.sensor_width / 2).clamp(0, self.max_radius) coc_norm = coc_px / self.max_radius blurred_levels = [self._blur_at_level(image, kernel) for kernel in self.kernels] level_float = coc_norm * (self.n_levels - 1) level_low = level_float.long().clamp(0, self.n_levels - 2) level_frac = (level_float - level_low.float()).clamp(0, 1) rendered = image.clone() for lv in range(self.n_levels - 1): mask = (level_low == lv).float() if mask.sum() > 0: interp = blurred_levels[lv] * (1 - level_frac) + blurred_levels[lv + 1] * level_frac rendered = rendered * (1 - mask) + interp * mask mask_top = (level_low >= self.n_levels - 2).float() * (level_frac > 0.99).float() rendered = rendered * (1 - mask_top) + blurred_levels[-1] * mask_top rendered = rendered + self.refine(rendered) * 0.1 return rendered, coc_px class BokehFlow(nn.Module): def __init__(self, config=None): super().__init__() if config is None: config = BokehFlowConfig() self.config = config; c = config self.stem = ConvStem(3, c.stem_channels, c.embed_dim) self.aperture_enc = ApertureEncoder(c.aperture_embed_dim) self.depth_blocks = nn.ModuleList([ GatedConvRecurrence(c.embed_dim, c.large_kernel, c.ffn_expansion) for _ in range(c.depth_blocks)]) self.bokeh_blocks = nn.ModuleList([ GatedConvRecurrenceWithACFM(c.embed_dim, c.large_kernel, c.ffn_expansion, c.aperture_embed_dim) for _ in range(c.bokeh_blocks)]) n_fusions = max(c.depth_blocks, c.bokeh_blocks) // c.fusion_every self.fusions = nn.ModuleList([CrossFusion(c.embed_dim) for _ in range(n_fusions)]) self.depth_head = DepthHead(c.embed_dim) self.bokeh_head = BokehHead(c.embed_dim) self.pgcoc = PGCoC(c.sensor_width_mm, c.max_coc_radius) self.blend_w = nn.Parameter(torch.tensor(0.5)) def forward(self, image, f_number=None, focal_length_mm=None, focus_distance_m=None, **kwargs): B = image.shape[0]; dev = image.device; c = self.config if f_number is None: f_number = torch.full((B,), c.default_fnumber, device=dev) if focal_length_mm is None: focal_length_mm = torch.full((B,), c.default_focal_mm, device=dev) if focus_distance_m is None: focus_distance_m = torch.full((B,), c.default_focus_m, device=dev) ae = self.aperture_enc(f_number, focal_length_mm, focus_distance_m) feat = self.stem(image) d_feat = feat; b_feat = feat; fi = 0 n_blk = max(c.depth_blocks, c.bokeh_blocks) for i in range(n_blk): if i < c.depth_blocks: d_feat = self.depth_blocks[i](d_feat) if i < c.bokeh_blocks: b_feat = self.bokeh_blocks[i](b_feat, ae) if (i+1) % c.fusion_every == 0 and fi < len(self.fusions): d_feat, b_feat = self.fusions[fi](d_feat, b_feat); fi += 1 depth = self.depth_head(d_feat) if depth.shape[2:] != image.shape[2:]: depth = F.interpolate(depth, image.shape[2:], mode='bilinear', align_corners=False) physics_bokeh, coc_map = self.pgcoc(image, depth, f_number, focal_length_mm, focus_distance_m) learned_bokeh = self.bokeh_head(b_feat) if learned_bokeh.shape[2:] != image.shape[2:]: learned_bokeh = F.interpolate(learned_bokeh, image.shape[2:], mode='bilinear', align_corners=False) w = torch.sigmoid(self.blend_w) bokeh = (w * physics_bokeh + (1-w) * (image + learned_bokeh)).clamp(0, 1) return {'bokeh': bokeh, 'depth': depth, 'coc_map': coc_map} class BokehFlowLoss(nn.Module): """Combined L1 + SSIM loss.""" def forward(self, pred, targets): bp, bg = pred['bokeh'], targets['bokeh_gt'] l1 = F.l1_loss(bp, bg) C1, C2 = 0.01**2, 0.03**2 mu_p = F.avg_pool2d(bp, 11, 1, 5); mu_g = F.avg_pool2d(bg, 11, 1, 5) mu_pp = mu_p*mu_p; mu_gg = mu_g*mu_g; mu_pg = mu_p*mu_g sig_pp = F.avg_pool2d(bp*bp, 11, 1, 5) - mu_pp sig_gg = F.avg_pool2d(bg*bg, 11, 1, 5) - mu_gg sig_pg = F.avg_pool2d(bp*bg, 11, 1, 5) - mu_pg ssim_map = ((2*mu_pg+C1)*(2*sig_pg+C2)) / ((mu_pp+mu_gg+C1)*(sig_pp+sig_gg+C2)) ssim_loss = 1.0 - ssim_map.mean() return {'total': l1 + ssim_loss, 'l1': l1.detach(), 'ssim': ssim_loss.detach()} def count_params(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) if __name__ == "__main__": import time for v in ['nano', 'small', 'base']: c = BokehFlowConfig(variant=v) dev = 'cuda' if torch.cuda.is_available() else 'cpu' m = BokehFlow(c).to(dev) print(f"BokehFlow-{v}: {count_params(m):,} params") x = torch.randn(2, 3, 256, 256, device=dev) m.eval() with torch.no_grad(): out = m(x) if torch.cuda.is_available(): torch.cuda.synchronize() t0 = time.time() with torch.no_grad(): for _ in range(10): out = m(x) if torch.cuda.is_available(): torch.cuda.synchronize() print(f" Inference: {(time.time()-t0)/10*1000:.1f}ms/batch (B=2, 256x256)") m.train() opt = torch.optim.AdamW(m.parameters(), lr=1e-3) loss_fn = BokehFlowLoss() gt = torch.rand_like(x[:,:3]) out = m(x); loss = loss_fn(out, {'bokeh_gt': gt})['total'] opt.zero_grad(); loss.backward(); opt.step() if torch.cuda.is_available(): torch.cuda.synchronize() t0 = time.time() for _ in range(10): out = m(x); loss = loss_fn(out, {'bokeh_gt': gt})['total'] opt.zero_grad(); loss.backward(); opt.step() if torch.cuda.is_available(): torch.cuda.synchronize() print(f" Training: {(time.time()-t0)/10*1000:.1f}ms/step (B=2, 256x256)") if torch.cuda.is_available(): print(f" VRAM: {torch.cuda.max_memory_allocated()/1e9:.2f} GB") torch.cuda.reset_peak_memory_stats() print()