drbh commited on 17 days ago

Commit

a80a7c6

unverified ·

0 Parent(s):

Migrated from kernels-community/tinygrad-rms

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +104 -0
benchmarks/benchmark.py +47 -0
build/torch210-cu128-x86_64-windows/__init__.py +63 -0
build/torch210-cu128-x86_64-windows/_ops.py +9 -0
build/torch210-cu128-x86_64-windows/_tinygrad_rms_cuda_6e9aef6.pyd +3 -0
build/torch210-cu128-x86_64-windows/metadata.json +21 -0
build/torch210-cu128-x86_64-windows/tinygrad_rms/__init__.py +26 -0
build/torch210-cxx11-cu126-aarch64-linux/__init__.py +63 -0
build/torch210-cxx11-cu126-aarch64-linux/_ops.py +9 -0
build/torch210-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch210-cxx11-cu126-aarch64-linux/metadata.json +18 -0
build/torch210-cxx11-cu126-aarch64-linux/tinygrad_rms/__init__.py +26 -0
build/torch210-cxx11-cu126-x86_64-linux/__init__.py +63 -0
build/torch210-cxx11-cu126-x86_64-linux/_ops.py +9 -0
build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch210-cxx11-cu126-x86_64-linux/metadata.json +18 -0
build/torch210-cxx11-cu126-x86_64-linux/tinygrad_rms/__init__.py +26 -0
build/torch210-cxx11-cu128-aarch64-linux/__init__.py +63 -0
build/torch210-cxx11-cu128-aarch64-linux/_ops.py +9 -0
build/torch210-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch210-cxx11-cu128-aarch64-linux/metadata.json +21 -0
build/torch210-cxx11-cu128-aarch64-linux/tinygrad_rms/__init__.py +26 -0
build/torch210-cxx11-cu128-x86_64-linux/__init__.py +63 -0
build/torch210-cxx11-cu128-x86_64-linux/_ops.py +9 -0
build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch210-cxx11-cu128-x86_64-linux/metadata.json +21 -0
build/torch210-cxx11-cu128-x86_64-linux/tinygrad_rms/__init__.py +26 -0
build/torch210-cxx11-cu130-aarch64-linux/__init__.py +63 -0
build/torch210-cxx11-cu130-aarch64-linux/_ops.py +9 -0
build/torch210-cxx11-cu130-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch210-cxx11-cu130-aarch64-linux/metadata.json +19 -0
build/torch210-cxx11-cu130-aarch64-linux/tinygrad_rms/__init__.py +26 -0
build/torch210-cxx11-cu130-x86_64-linux/__init__.py +63 -0
build/torch210-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch210-cxx11-cu130-x86_64-linux/metadata.json +19 -0
build/torch210-cxx11-cu130-x86_64-linux/tinygrad_rms/__init__.py +26 -0
build/torch211-cxx11-cu126-aarch64-linux/__init__.py +63 -0
build/torch211-cxx11-cu126-aarch64-linux/_ops.py +9 -0
build/torch211-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch211-cxx11-cu126-aarch64-linux/metadata.json +18 -0
build/torch211-cxx11-cu126-aarch64-linux/tinygrad_rms/__init__.py +26 -0
build/torch211-cxx11-cu126-x86_64-linux/__init__.py +63 -0
build/torch211-cxx11-cu126-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0
build/torch211-cxx11-cu126-x86_64-linux/metadata.json +18 -0
build/torch211-cxx11-cu126-x86_64-linux/tinygrad_rms/__init__.py +26 -0
build/torch211-cxx11-cu128-aarch64-linux/__init__.py +63 -0
build/torch211-cxx11-cu128-aarch64-linux/_ops.py +9 -0
build/torch211-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so +3 -0

.gitattributes ADDED Viewed

	@@ -0,0 +1,104 @@

+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu126-x86_64-linux/_tinygrad_rms_3102ae4.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu128-x86_64-linux/_tinygrad_rms_3102ae4.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu129-x86_64-linux/_tinygrad_rms_3102ae4.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/_tinygrad_rms_3102ae4.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/_tinygrad_rms_3102ae4.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/_tinygrad_rms_3102ae4.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu126-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu128-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu129-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/_tinygrad_rms_45fdbd5.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu126-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu128-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu129-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/_tinygrad_rms_490e187.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu126-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu128-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu129-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/_tinygrad_rms_9ad78ee.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu126-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu128-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch28-cxx11-cu129-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/_tinygrad_rms_eb79f2b.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-aarch64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-aarch64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cu128-x86_64-windows/_tinygrad_rms_cuda_6e9aef6.pyd filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_9a0d032.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu130-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu129-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch29-cxx11-cu129-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so filter=lfs diff=lfs merge=lfs -text

benchmarks/benchmark.py ADDED Viewed

	@@ -0,0 +1,47 @@

+import torch
+from kernels.benchmark import Benchmark
+def rmsnorm_reference(x: torch.Tensor, eps: float) -> torch.Tensor:
+    rms = torch.sqrt(torch.mean(x**2, dim=-1, keepdim=True) + eps)
+    return x / rms
+class TinygradRmsBenchmark(Benchmark):
+    seed: int = 42
+    def setup(self):
+        batch_size = 32
+        seq_len = 512
+        hidden_size = 1024
+        self.eps = 1e-6
+        self.x = torch.randn(
+            batch_size, seq_len, hidden_size, device=self.device, dtype=torch.float32
+        )
+        self.out = torch.empty_like(self.x)
+    def benchmark_base(self):
+        self.out = self.kernel.tinygrad_rms_norm_simple(self.x, self.eps)
+    def verify_base(self) -> torch.Tensor:
+        return rmsnorm_reference(self.x, self.eps)
+    def setup_large(self):
+        # Note: hidden_size must be 1024 (kernel constraint)
+        batch_size = 64
+        seq_len = 1024
+        hidden_size = 1024
+        self.eps = 1e-6
+        self.x = torch.randn(
+            batch_size, seq_len, hidden_size, device=self.device, dtype=torch.float32
+        )
+        self.out = torch.empty_like(self.x)
+    def benchmark_large(self):
+        self.out = self.kernel.tinygrad_rms_norm_simple(self.x, self.eps)
+    def verify_large(self) -> torch.Tensor:
+        return rmsnorm_reference(self.x, self.eps)

build/torch210-cu128-x86_64-windows/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cu128-x86_64-windows/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6e9aef6
+ops = torch.ops._tinygrad_rms_cuda_6e9aef6
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6e9aef6::{op_name}"

build/torch210-cu128-x86_64-windows/_tinygrad_rms_cuda_6e9aef6.pyd ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2015223a8a6040fa56544f699eab3c9f4723865f663aebf61fbd06d8fd250bea
+size 320000

build/torch210-cu128-x86_64-windows/metadata.json ADDED Viewed

	@@ -0,0 +1,21 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "10.1",
+      "12.0+PTX",
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch210-cu128-x86_64-windows/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import sys
+import importlib
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch210-cxx11-cu126-aarch64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cxx11-cu126-aarch64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch210-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9a26e93d9a6daa5741ae1b976db32aadf2a9342f67211cc6ab1caa5348406e75
+size 2241168

build/torch210-cxx11-cu126-aarch64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,18 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0+PTX"
+    ]
+  }
+}

build/torch210-cxx11-cu126-aarch64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch210-cxx11-cu126-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cxx11-cu126-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch210-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b6dc5ab336157c3e6c577ecc8dca3790d4ac9273ae4b38ebd45e7dee0c6c20a5
+size 2123424

build/torch210-cxx11-cu126-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,18 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0+PTX"
+    ]
+  }
+}

build/torch210-cxx11-cu126-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch210-cxx11-cu128-aarch64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cxx11-cu128-aarch64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch210-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e05735521a2de0896d311bbfdda83b8d5c813e1498597b6613728a029f969665
+size 2372360

build/torch210-cxx11-cu128-aarch64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,21 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "10.1",
+      "12.0+PTX",
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch210-cxx11-cu128-aarch64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch210-cxx11-cu128-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cxx11-cu128-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch210-cxx11-cu128-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:645b392a5deadd70a5405bbceeb2161c9b852cb0a303b826e8ee633ee4443883
+size 2244040

build/torch210-cxx11-cu128-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,21 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "10.1",
+      "12.0+PTX",
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch210-cxx11-cu128-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch210-cxx11-cu130-aarch64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cxx11-cu130-aarch64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch210-cxx11-cu130-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:dab93449e780276899a05ae1303d2a7c34c859ec5f596592246169999c6879b4
+size 2374320

build/torch210-cxx11-cu130-aarch64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,19 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "11.0",
+      "12.0+PTX",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch210-cxx11-cu130-aarch64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch210-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch210-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch210-cxx11-cu130-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:3bd783d80e077f2983793e5fdc8a0f5478836851435a8b77f00b186fea64c8f4
+size 2245848

build/torch210-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,19 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "11.0",
+      "12.0+PTX",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0"
+    ]
+  }
+}

build/torch210-cxx11-cu130-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu126-aarch64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch211-cxx11-cu126-aarch64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch211-cxx11-cu126-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:2b0b368d677277876427af5179c177f36d614fc7787e258b582bca5ac653b332
+size 2237384

build/torch211-cxx11-cu126-aarch64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,18 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0+PTX"
+    ]
+  }
+}

build/torch211-cxx11-cu126-aarch64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu126-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch211-cxx11-cu126-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch211-cxx11-cu126-x86_64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:f6394aee38bd05dd68b98ab6e87ff95748c1662dad1b85178c475cd76de7cbf4
+size 2112256

build/torch211-cxx11-cu126-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,18 @@

+{
+  "version": 1,
+  "license": "MIT",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "7.0",
+      "7.2",
+      "7.5",
+      "8.0",
+      "8.6",
+      "8.7",
+      "8.9",
+      "9.0+PTX"
+    ]
+  }
+}

build/torch211-cxx11-cu126-x86_64-linux/tinygrad_rms/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu128-aarch64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,63 @@

+from typing import Optional, Tuple
+import torch
+from ._ops import ops
+def tinygrad_rms_norm(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    RMSNorm(x) = x * (1 / sqrt(mean(x^2) + epsilon))
+    This implementation uses a two-kernel approach:
+    1. Compute 1/sqrt(mean(x^2) + epsilon) for each row
+    2. Multiply input by the computed factor
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Tuple of (output tensor, rms_inv tensor)
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    hidden_size = x.size(-1)
+    num_rows = x.numel() // hidden_size
+    rms_inv = torch.empty(num_rows, dtype=x.dtype, device=x.device)
+    ops.tinygrad_rms_norm(out, rms_inv, x, epsilon)
+    return out, rms_inv
+def tinygrad_rms_norm_simple(
+    x: torch.Tensor,
+    epsilon: float = 1e-6,
+    out: Optional[torch.Tensor] = None,
+) -> torch.Tensor:
+    """
+    Compute RMSNorm using tinygrad-style CUDA kernels.
+    This is a simpler interface that only returns the normalized output.
+    Args:
+        x: Input tensor of shape (..., hidden_size)
+        epsilon: Small constant for numerical stability
+        out: Optional pre-allocated output tensor
+    Returns:
+        Normalized output tensor
+    """
+    if out is None:
+        out = torch.empty_like(x)
+    ops.tinygrad_rms_norm_inplace(out, x, epsilon)
+    return out

build/torch211-cxx11-cu128-aarch64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _tinygrad_rms_cuda_6eb00dc
+ops = torch.ops._tinygrad_rms_cuda_6eb00dc
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_tinygrad_rms_cuda_6eb00dc::{op_name}"

build/torch211-cxx11-cu128-aarch64-linux/_tinygrad_rms_cuda_6eb00dc.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d7a77fa708df07fe1cd9181db9b445dd9a2535bef9968d170c437868f6cca5a3
+size 2368568