Perf/kernel optimizations

example/src/main.ts

@@ -9,7 +9,10 @@ import { tasks } from './tasks';
 /** Metrics tooltip lines shared by every task: `[token, description]` pairs. */
 const METRICS_TIP: readonly (readonly [token: string, description: string])[] = [
   ['FPS', 'live processing rate: video frames processed per second (camera-paced)'],
-  ['MS', 'per-frame inference time to GPU completion; 1000 / MS is the FPS ceiling'],
+  [
+    'MS',
+    'frame processing time including crop/upload, model, rendering and GPU completion notification',
+  ],
 ];
 
 const app = document.getElementById('app') as HTMLElement;

example/src/tasks/depth-estimation/loop.ts

@@ -31,7 +31,7 @@ function onNextVideoFrame(video: HTMLVideoElement, callback: (now: number) => vo
  *
  * @param video The playing camera video to source frames from.
  * @param canvas The canvas element to render the grayscale depth output to.
- * @param onMetrics Optional callback invoked with smoothed FPS and inference time in ms.
+ * @param onMetrics Optional callback invoked with smoothed FPS and frame processing time in ms.
  * @param precision Selected floating-point precision ('float32' or 'float16').
  * @returns A stop function that halts the frame loop and cleans up resources.
  */
@@ -88,8 +88,8 @@ export async function startFrameLoop(
   const vidTexture = device.createTexture({ size, format: 'rgba8unorm', usage: vidUsage });
   const resTexture = device.createTexture({ size, format: 'rgba8unorm', usage: resUsage });
 
-  // Frame-completion fence: a 16-byte copy whose mapAsync resolves exactly
-  // when the frame's GPU work has landed.
+  // Frame-completion fence: mapping this copy waits for the frame's GPU work.
+  // Promise delivery also includes browser scheduling latency.
   const fenceSrc = device.createBuffer({ size: 16, usage: GPUBufferUsage.COPY_SRC });
   const fenceDst = device.createBuffer({
     size: 16,
@@ -167,8 +167,8 @@ export async function startFrameLoop(
     encoder.copyBufferToBuffer(fenceSrc, 0, fenceDst, 0, 16);
     device.queue.submit([encoder.finish()]);
 
-    // Inference time runs to the fence: mapAsync resolves exactly when the
-    // copy lands, unlike onSubmittedWorkDone which can lag to a frame boundary.
+    // Frame time includes crop/upload, GPU work and completion notification.
+    // The fence wait alone is not a measurement of shader execution time.
     void fenceDst.mapAsync(GPUMapMode.READ).then(
       () => {
         if (!running) return;

example/src/tasks/yolo26-detection/loop.ts

@@ -31,7 +31,7 @@ function onNextVideoFrame(video: HTMLVideoElement, callback: (now: number) => vo
  *
  * @param video The playing camera video to source frames from.
  * @param canvas The canvas element to render the graded video output to.
- * @param onMetrics Optional callback invoked with smoothed FPS and inference time in ms.
+ * @param onMetrics Optional callback invoked with smoothed FPS and frame processing time in ms.
  * @param precision Selected floating-point precision ('float32' or 'float16').
  * @returns A stop function that halts the frame loop and cleans up resources.
  */
@@ -88,8 +88,8 @@ export async function startFrameLoop(
   const vidTexture = device.createTexture({ size, format: 'rgba8unorm', usage: vidUsage });
   const resTexture = device.createTexture({ size, format: 'rgba8unorm', usage: resUsage });
 
-  // Frame-completion fence: a 16-byte copy whose mapAsync resolves exactly
-  // when the frame's GPU work has landed.
+  // Frame-completion fence: mapping this copy waits for the frame's GPU work.
+  // Promise delivery also includes browser scheduling latency.
   const fenceSrc = device.createBuffer({ size: 16, usage: GPUBufferUsage.COPY_SRC });
   const fenceDst = device.createBuffer({
     size: 16,
@@ -171,8 +171,8 @@ export async function startFrameLoop(
     encoder.copyBufferToBuffer(fenceSrc, 0, fenceDst, 0, 16);
     device.queue.submit([encoder.finish()]);
 
-    // Inference time runs to the fence: mapAsync resolves exactly when the
-    // copy lands, unlike onSubmittedWorkDone which can lag to a frame boundary.
+    // Frame time includes crop/upload, GPU work and completion notification.
+    // The fence wait alone is not a measurement of shader execution time.
     void fenceDst.mapAsync(GPUMapMode.READ).then(
       () => {
         if (!running) return;

package.json

@@ -9,6 +9,12 @@
     "example": "vite example",
     "build:example": "vite build example",
     "benchmark:web": "vite utils/benchmark",
+    "benchmark:live": "node utils/benchmark/profile-live.mjs",
+    "benchmark:stages": "node utils/benchmark/profile-live.mjs --controlled",
+    "benchmark:specialized": "node utils/benchmark/profile-live.mjs --specialized",
+    "benchmark:runntime": "node utils/benchmark/profile-live.mjs --runntime",
+    "benchmark:hf": "node utils/benchmark/profile-live.mjs --hf",
+    "benchmark:narrow": "node utils/benchmark/profile-live.mjs --narrow",
     "format": "prettier --write .",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",

src/shaders/conv-with-clamp/conv-with-clamp-tile32-f16.wgsl.ts

@@ -41,6 +41,18 @@ struct Params {
 }
 @group(0) @binding(4) var<uniform> params: Params;
 
+// Geometry constants let every backend simplify division in the gather loop.
+override kernel_h: u32 = 0u;
+override kernel_w: u32 = 0u;
+
+// Zero preserves uniform-driven execution for callers using the raw template.
+fn kernel_height() -> u32 {
+  return select(params.Kh, kernel_h, kernel_h != 0u);
+}
+fn kernel_width() -> u32 {
+  return select(params.Kw, kernel_w, kernel_w != 0u);
+}
+
 var<workgroup> a_sub: array<array<vec4<f16>, 4>, 32>;
 var<workgroup> b_sub: array<array<vec4<f16>, 16>, 16>;
 
@@ -54,9 +66,9 @@ fn main(
   let tile_col0 = wg_id.x * 64u;
   let M = params.OC;
   let N = params.N * params.H_out * params.W_out;
-  let K = params.IC * params.Kh * params.Kw;
+  let K = params.IC * kernel_height() * kernel_width();
   let K_vecs = K >> 2u;
-  let khw = params.Kh * params.Kw;
+  let khw = kernel_height() * kernel_width();
   let ohw = params.H_out * params.W_out;
 
   let in_hw = params.H_in * params.W_in;
@@ -108,6 +120,33 @@ fn main(
     }
   }
 
+  // For 3x3 kernels, validity is invariant across input channels. Interior
+  // columns get all nine bits; borders and spatial tails get only valid taps.
+  var tap_mask: array<u32, 4>;
+  if (kernel_height() == 3u && kernel_width() == 3u) {
+    for (var col = 0u; col < 4u; col++) {
+      var mask = 0u;
+      if (col_valid0[col]) {
+        if (col_ih_base0[col] >= 0 && col_iw_base0[col] >= 0 &&
+            col_ih_base0[col] + 2 * dH < i32(params.H_in) &&
+            col_iw_base0[col] + 2 * dW < i32(params.W_in)) {
+          mask = 0x1ffu;
+        } else {
+          for (var ky = 0u; ky < 3u; ky++) {
+            let iy = col_ih_base0[col] + i32(ky) * dH;
+            for (var kx = 0u; kx < 3u; kx++) {
+              let ix = col_iw_base0[col] + i32(kx) * dW;
+              if (iy >= 0 && iy < i32(params.H_in) && ix >= 0 && ix < i32(params.W_in)) {
+                mask = mask | (1u << (ky * 3u + kx));
+              }
+            }
+          }
+        }
+      }
+      tap_mask[col] = mask;
+    }
+  }
+
   let num_tiles = (K + 15u) >> 4u;
   for (var t = 0u; t < num_tiles; t = t + 1u) {
     let k4_start = t << 2u;
@@ -123,18 +162,17 @@ fn main(
     if (bk0 < K) {
       let ic0 = bk0 / khw;
       let rem0 = bk0 % khw;
-      let kh0 = i32(rem0 / params.Kw) * dH;
-      let kw0 = i32(rem0 % params.Kw) * dW;
+      let kh0 = i32(rem0 / kernel_width()) * dH;
+      let kw0 = i32(rem0 % kernel_width()) * dW;
 
       var b_row0 = vec4<f16>(0.0h);
       for (var kk = 0u; kk < 4u; kk = kk + 1u) {
-        if (col_valid0[kk]) {
-          let ih0 = col_ih_base0[kk] + kh0;
-          let iw0 = col_iw_base0[kk] + kw0;
-          if (ih0 >= 0 && ih0 < i32(params.H_in) && iw0 >= 0 && iw0 < i32(params.W_in)) {
-            let in_idx0 = (col_b0[kk] * params.IC + ic0) * in_hw + u32(ih0) * in_w + u32(iw0);
-            b_row0[kk] = t_in[in_idx0];
-          }
+        let ih0 = col_ih_base0[kk] + kh0;
+        let iw0 = col_iw_base0[kk] + kw0;
+        if ((kernel_height() == 3u && kernel_width() == 3u && (tap_mask[kk] & (1u << rem0)) != 0u) ||
+            ((kernel_height() != 3u || kernel_width() != 3u) && col_valid0[kk] && ih0 >= 0 && ih0 < i32(params.H_in) && iw0 >= 0 && iw0 < i32(params.W_in))) {
+          let in_idx0 = (col_b0[kk] * params.IC + ic0) * in_hw + u32(ih0) * in_w + u32(iw0);
+          b_row0[kk] = t_in[in_idx0];
         }
       }
       b_sub[b_r0][b_c4_0] = b_row0;
@@ -146,18 +184,17 @@ fn main(
     if (bk1 < K) {
       let ic1 = bk1 / khw;
       let rem1 = bk1 % khw;
-      let kh1 = i32(rem1 / params.Kw) * dH;
-      let kw1 = i32(rem1 % params.Kw) * dW;
+      let kh1 = i32(rem1 / kernel_width()) * dH;
+      let kw1 = i32(rem1 % kernel_width()) * dW;
 
       var b_row1 = vec4<f16>(0.0h);
       for (var kk = 0u; kk < 4u; kk = kk + 1u) {
-        if (col_valid0[kk]) {
-          let ih1 = col_ih_base0[kk] + kh1;
-          let iw1 = col_iw_base0[kk] + kw1;
-          if (ih1 >= 0 && ih1 < i32(params.H_in) && iw1 >= 0 && iw1 < i32(params.W_in)) {
-            let in_idx1 = (col_b0[kk] * params.IC + ic1) * in_hw + u32(ih1) * in_w + u32(iw1);
-            b_row1[kk] = t_in[in_idx1];
-          }
+        let ih1 = col_ih_base0[kk] + kh1;
+        let iw1 = col_iw_base0[kk] + kw1;
+        if ((kernel_height() == 3u && kernel_width() == 3u && (tap_mask[kk] & (1u << rem1)) != 0u) ||
+            ((kernel_height() != 3u || kernel_width() != 3u) && col_valid0[kk] && ih1 >= 0 && ih1 < i32(params.H_in) && iw1 >= 0 && iw1 < i32(params.W_in))) {
+          let in_idx1 = (col_b0[kk] * params.IC + ic1) * in_hw + u32(ih1) * in_w + u32(iw1);
+          b_row1[kk] = t_in[in_idx1];
         }
       }
       b_sub[b_r1][b_c4_1] = b_row1;
@@ -210,12 +247,10 @@ fn main(
       for (var kk = 0u; kk < 4u; kk = kk + 1u) {
         let n = out_c_base + kk;
         if (n < N) {
-          let b_val = n / ohw;
-          let sp = n % ohw;
-          let oh = sp / params.W_out;
-          let ow = sp % params.W_out;
+          // Reuse the batch decoded for input gathering instead of dividing again.
+          let out_base = col_b0[kk] * (M - 1u) * ohw + n;
           let val = clamp(acc_v[kk], min_v, max_v);
-          t_out[((b_val * params.OC + m) * params.H_out + oh) * params.W_out + ow] = val;
+          t_out[m * ohw + out_base] = val;
         }
       }
     }

src/shaders/conv-with-clamp/conv-with-clamp-tile32.wgsl.ts

@@ -39,6 +39,18 @@ struct Params {
 }
 @group(0) @binding(4) var<uniform> params: Params;
 
+// Geometry constants let every backend simplify division in the gather loop.
+override kernel_h: u32 = 0u;
+override kernel_w: u32 = 0u;
+
+// Zero preserves uniform-driven execution for callers using the raw template.
+fn kernel_height() -> u32 {
+  return select(params.Kh, kernel_h, kernel_h != 0u);
+}
+fn kernel_width() -> u32 {
+  return select(params.Kw, kernel_w, kernel_w != 0u);
+}
+
 // a_sub: 32 output channels x 16 K elements (4 vec4s) = 128 vec4s (2 KB)
 var<workgroup> a_sub: array<array<vec4<f32>, 4>, 32>;
 // b_sub: 16 K elements x 64 spatial elements (16 vec4s) = 256 vec4s (4 KB)
@@ -54,9 +66,9 @@ fn main(
   let tile_col0 = wg_id.x * 64u;
   let M = params.OC;
   let N = params.N * params.H_out * params.W_out;
-  let K = params.IC * params.Kh * params.Kw;
+  let K = params.IC * kernel_height() * kernel_width();
   let K_vecs = K >> 2u;
-  let khw = params.Kh * params.Kw;
+  let khw = kernel_height() * kernel_width();
   let ohw = params.H_out * params.W_out;
 
   let in_hw = params.H_in * params.W_in;
@@ -108,6 +120,33 @@ fn main(
     }
   }
 
+  // For 3x3 kernels, validity is invariant across input channels. Interior
+  // columns get all nine bits; borders and spatial tails get only valid taps.
+  var tap_mask: array<u32, 4>;
+  if (kernel_height() == 3u && kernel_width() == 3u) {
+    for (var col = 0u; col < 4u; col++) {
+      var mask = 0u;
+      if (col_valid0[col]) {
+        if (col_ih_base0[col] >= 0 && col_iw_base0[col] >= 0 &&
+            col_ih_base0[col] + 2 * dH < i32(params.H_in) &&
+            col_iw_base0[col] + 2 * dW < i32(params.W_in)) {
+          mask = 0x1ffu;
+        } else {
+          for (var ky = 0u; ky < 3u; ky++) {
+            let iy = col_ih_base0[col] + i32(ky) * dH;
+            for (var kx = 0u; kx < 3u; kx++) {
+              let ix = col_iw_base0[col] + i32(kx) * dW;
+              if (iy >= 0 && iy < i32(params.H_in) && ix >= 0 && ix < i32(params.W_in)) {
+                mask = mask | (1u << (ky * 3u + kx));
+              }
+            }
+          }
+        }
+      }
+      tap_mask[col] = mask;
+    }
+  }
+
   let num_tiles = (K + 15u) >> 4u;
   for (var t = 0u; t < num_tiles; t = t + 1u) {
     let k4_start = t << 2u;
@@ -124,18 +163,17 @@ fn main(
     if (bk0 < K) {
       let ic0 = bk0 / khw;
       let rem0 = bk0 % khw;
-      let kh0 = i32(rem0 / params.Kw) * dH;
-      let kw0 = i32(rem0 % params.Kw) * dW;
+      let kh0 = i32(rem0 / kernel_width()) * dH;
+      let kw0 = i32(rem0 % kernel_width()) * dW;
 
       var b_row0 = vec4<f32>(0.0);
       for (var kk = 0u; kk < 4u; kk = kk + 1u) {
-        if (col_valid0[kk]) {
-          let ih0 = col_ih_base0[kk] + kh0;
-          let iw0 = col_iw_base0[kk] + kw0;
-          if (ih0 >= 0 && ih0 < i32(params.H_in) && iw0 >= 0 && iw0 < i32(params.W_in)) {
-            let in_idx0 = (col_b0[kk] * params.IC + ic0) * in_hw + u32(ih0) * in_w + u32(iw0);
-            b_row0[kk] = t_in[in_idx0];
-          }
+        let ih0 = col_ih_base0[kk] + kh0;
+        let iw0 = col_iw_base0[kk] + kw0;
+        if ((kernel_height() == 3u && kernel_width() == 3u && (tap_mask[kk] & (1u << rem0)) != 0u) ||
+            ((kernel_height() != 3u || kernel_width() != 3u) && col_valid0[kk] && ih0 >= 0 && ih0 < i32(params.H_in) && iw0 >= 0 && iw0 < i32(params.W_in))) {
+          let in_idx0 = (col_b0[kk] * params.IC + ic0) * in_hw + u32(ih0) * in_w + u32(iw0);
+          b_row0[kk] = t_in[in_idx0];
         }
       }
       b_sub[b_r0][b_c4_0] = b_row0;
@@ -148,18 +186,17 @@ fn main(
     if (bk1 < K) {
       let ic1 = bk1 / khw;
       let rem1 = bk1 % khw;
-      let kh1 = i32(rem1 / params.Kw) * dH;
-      let kw1 = i32(rem1 % params.Kw) * dW;
+      let kh1 = i32(rem1 / kernel_width()) * dH;
+      let kw1 = i32(rem1 % kernel_width()) * dW;
 
       var b_row1 = vec4<f32>(0.0);
       for (var kk = 0u; kk < 4u; kk = kk + 1u) {
-        if (col_valid0[kk]) {
-          let ih1 = col_ih_base0[kk] + kh1;
-          let iw1 = col_iw_base0[kk] + kw1;
-          if (ih1 >= 0 && ih1 < i32(params.H_in) && iw1 >= 0 && iw1 < i32(params.W_in)) {
-            let in_idx1 = (col_b0[kk] * params.IC + ic1) * in_hw + u32(ih1) * in_w + u32(iw1);
-            b_row1[kk] = t_in[in_idx1];
-          }
+        let ih1 = col_ih_base0[kk] + kh1;
+        let iw1 = col_iw_base0[kk] + kw1;
+        if ((kernel_height() == 3u && kernel_width() == 3u && (tap_mask[kk] & (1u << rem1)) != 0u) ||
+            ((kernel_height() != 3u || kernel_width() != 3u) && col_valid0[kk] && ih1 >= 0 && ih1 < i32(params.H_in) && iw1 >= 0 && iw1 < i32(params.W_in))) {
+          let in_idx1 = (col_b0[kk] * params.IC + ic1) * in_hw + u32(ih1) * in_w + u32(iw1);
+          b_row1[kk] = t_in[in_idx1];
         }
       }
       b_sub[b_r1][b_c4_1] = b_row1;
@@ -212,12 +249,10 @@ fn main(
       for (var kk = 0u; kk < 4u; kk = kk + 1u) {
         let n = out_c_base + kk;
         if (n < N) {
-          let b_val = n / ohw;
-          let sp = n % ohw;
-          let oh = sp / params.W_out;
-          let ow = sp % params.W_out;
+          // Reuse the batch decoded for input gathering instead of dividing again.
+          let out_base = col_b0[kk] * (M - 1u) * ohw + n;
           let val = clamp(acc_v[kk], min_v, max_v);
-          t_out[((b_val * params.OC + m) * params.H_out + oh) * params.W_out + ow] = val;
+          t_out[m * ohw + out_base] = val;
         }
       }
     }