Add option to use fragment shaders rather than compute with WebGPU

blur.html

@@ -29,6 +29,10 @@
       display: block;
       margin-bottom: 10px;
     }
+
+    .inline label {
+      display: inline-block
+    }
   </style>
 </head>
 
@@ -53,6 +57,15 @@ <h3 style="margin-top: 0;">Renderer Selection</h3>
       <label>
         <input type="radio" name="renderer" value="webgpu" id="webgpuRadio" /> WebGPU (Advanced, requires compatible browser)
       </label>
+      <div style="padding-left: 20px;" class='inline'>
+        <b>Shader Type</b>:
+        <label id="shaderTypeComputeLabel">
+          <input type="radio" name="shaderType" value="compute" id="shaderTypeCompute" /> Compute
+        </label>
+        <label id="shaderTypeFragLabel">
+          <input type="radio" name="shaderType" value="fragment" id="shaderTypeFrag" checked /> Fragment
+        </label>
+      </div>
       <label style="padding-left: 20px;" id="zeroCopyLabel">
         <input type="checkbox" id="zeroCopy" name="zeroCopy" unchecked /> Zero-copy import
       </label>

blur.js

@@ -54,8 +54,9 @@ async function initializeBlurRenderer(webGpuDevice) {
       const zeroCopy = zeroCopyCheckbox.checked;
       const directOutput = directOutputCheckbox.checked;
       const zeroCopyTensor = webnnZeroCopyCheckbox.checked;
+      const useFragment = shaderTypeFrag.checked;
       const useWebNN = webnnRadio.checked;
-      appBlurRenderer = await createWebGPUBlurRenderer(webGpuDevice, segmenter, zeroCopy, directOutput, useWebNN, zeroCopyTensor);
+      appBlurRenderer = await createWebGPUBlurRenderer(webGpuDevice, segmenter, zeroCopy, directOutput, useWebNN, zeroCopyTensor, useFragment);
       appStatus.innerText = 'Renderer: WebGPU';
       console.log('Using WebGPU for blur rendering');
     } else {
@@ -145,6 +146,7 @@ async function runInWorker(trackProcessor, trackGenerator) {
       segmenterType: document.querySelector('input[name="segmenter"]:checked').value,
       zeroCopy: zeroCopyCheckbox ? zeroCopyCheckbox.checked : false,
       directOutput: directOutputCheckbox ? directOutputCheckbox.checked : false,
+      useFragment: shaderTypeFrag ? shaderTypeFrag.checked : false,
     };
 
     // Transfer the readable and writable streams to the worker for zero-copy data handling.
@@ -213,6 +215,10 @@ const zeroCopyCheckbox = document.getElementById('zeroCopy');
 const zeroCopyLabel = document.getElementById('zeroCopyLabel');
 const directOutputCheckbox = document.getElementById('directOutput');
 const directOutputLabel = document.getElementById('directOutputLabel');
+const shaderTypeComputeLabel = document.getElementById('shaderTypeComputeLabel');
+const shaderTypeCompute = document.getElementById('shaderTypeCompute');
+const shaderTypeFragLabel = document.getElementById('shaderTypeFragLabel');
+const shaderTypeFrag = document.getElementById('shaderTypeFrag');
 const webnnRadio = document.getElementById('segmenter-webnn');
 const webnnZeroCopyCheckbox = document.getElementById('webnnZeroCopy');
 const webrtcSink = document.getElementById('webrtcSink');
@@ -305,6 +311,10 @@ function updateOptionState() {
   zeroCopyLabel.style.color = isWebGPU ? '' : '#aaa';
   directOutputCheckbox.disabled = !isWebGPU;
   directOutputLabel.style.color = isWebGPU ? '' : '#aaa';
+  shaderTypeCompute.disabled = !isWebGPU;
+  shaderTypeComputeLabel.style.color = isWebGPU ? '' : '#aaa';
+  shaderTypeFrag.disabled = !isWebGPU;
+  shaderTypeFragLabel.style.color = isWebGPU ? '' : '#aaa';
 
   const useWebRTCSink = webrtcSink.checked;
   webrtcCodec.style.display = useWebRTCSink ? 'inline-block' : 'none';
@@ -384,7 +394,7 @@ async function initializeApp() {
     updateOptionState();
 
     // If the app is running, and a core pipeline option changed, restart the pipeline.
-    const restartNeededOptions = ['renderer', 'useWorker', 'segmenter', 'zeroCopy', 'directOutput'];
+    const restartNeededOptions = ['renderer', 'useWorker', 'segmenter', 'zeroCopy', 'directOutput', 'shaderType'];
     if (isRunning && restartNeededOptions.includes(event.target.name)) {
       console.log(`Restarting pipeline due to change in '${event.target.name}'`);
       stopVideoProcessing();

blur4/shaders/blend.fragment.wgsl

@@ -0,0 +1,39 @@
+struct VertexOut {
+  @builtin(position) pos: vec4f,
+  @location(0) uv: vec2f,
+}
+
+@vertex
+fn vertMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOut {
+  var pos = array<vec2f, 4>(
+    vec2f(-1.0, -1.0), vec2f(1.0, -1.0), vec2f(-1.0, 1.0), vec2f(1.0, 1.0),
+  );
+  var uv = pos[vertexIndex] * vec2f(0.5, -0.5) + 0.5;
+  return VertexOut(vec4f(pos[vertexIndex], 0, 1), uv);
+}
+
+struct ImageSize {
+  width : i32,
+  height : i32,
+  texel_size : vec2<f32>,
+};
+
+@group(0) @binding(0) var input : ${inputTextureType};
+@group(0) @binding(1) var blurred : texture_2d<f32>;
+@group(0) @binding(2) var mask : texture_2d<f32>;
+@group(0) @binding(3) var s : sampler;
+@group(0) @binding(4) var<uniform> size : ImageSize;
+
+const k00 = f32(${k00});
+const kTileSize = ${tileSize}u;
+
+@fragment
+fn fragMain(@location(0) uv: vec2f) -> @location(0) vec4f {
+  let coord_norm = uv + (0.5 * size.texel_size);
+  var m = textureSampleLevel(mask, s, coord_norm, 0.0).r;
+  var c = ${textureSampleCall};
+  var b = textureSampleLevel(blurred, s, coord_norm, 0.0);
+  b = b + (k00 * m) * c;
+  b = b / b.a;
+  return mix(b, c, m);
+}

blur4/shaders/blur.fragment.wgsl

@@ -0,0 +1,78 @@
+struct VertexOut {
+  @builtin(position) pos: vec4f,
+  @location(0) uv: vec2f,
+}
+
+@vertex
+fn vertMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOut {
+  var pos = array<vec2f, 4>(
+    vec2f(-1.0, -1.0), vec2f(1.0, -1.0), vec2f(-1.0, 1.0), vec2f(1.0, 1.0),
+  );
+  var uv = pos[vertexIndex] * vec2f(0.5, -0.5) + 0.5;
+  return VertexOut(vec4f(pos[vertexIndex], 0, 1), uv);
+}
+
+struct ImageSize {
+  width : i32,
+  height : i32,
+  texel_size : vec2<f32>,
+};
+
+@group(0) @binding(0) var input : ${inputTextureType};
+@group(0) @binding(1) var mask : texture_2d<f32>;
+@group(0) @binding(2) var s : sampler;
+@group(0) @binding(3) var<uniform> size : ImageSize;
+
+const kRadius = ${radius}u;
+const kTileSize = ${tileSize}u;
+
+fn blur(sample_coordinate : vec2<i32>, dir : vec2<f32>, pass_no : i32) -> vec4f {
+  let sample_coordinate_norm =
+      (vec2<f32>(sample_coordinate) + vec2<f32>(0.5)) * size.texel_size;
+
+  var kKernel = array<f32, ${kernelSize}>(${kernelInitializer});
+
+  let alpha = 1.0 - textureSampleLevel(mask, s, sample_coordinate_norm, 0.0).r;
+  var color = ${textureSampleCall};
+  var w = kKernel[0];
+  if (pass_no == 0) {
+    w = w * alpha;
+  }
+  color = color * w;
+
+  let step = dir * alpha;
+  var offset = step;
+
+  var coord : vec2<f32>;
+  for (var i = 1u; i <= kRadius; i=i+1u) {
+      coord = sample_coordinate_norm + offset;
+      w = kKernel[i];
+      if (pass_no == 0) {
+        w = w * (1.0 - textureSampleLevel(mask, s, coord, 0.0).r);
+      }
+      color = color + (w * ${loopTextureSampleCall});
+
+      coord = sample_coordinate_norm - offset;
+      w = kKernel[i];
+      if (pass_no == 0) {
+        w = w * (1.0 - textureSampleLevel(mask, s, coord, 0.0).r);
+      }
+      color = color + (w * ${loopTextureSampleCall});
+
+      offset = offset + step;
+  }
+
+  return color;
+}
+
+@fragment
+fn main_horizontal(@builtin(position) pos: vec4f) -> @location(0) vec4f {
+  let dir = vec2<f32>(size.texel_size.x, 0.0);
+  return blur(vec2i(pos.xy), dir, 0);
+}
+
+@fragment
+fn main_vertical(@builtin(position) pos: vec4f) -> @location(0) vec4f {
+  let dir = vec2<f32>(0.0, size.texel_size.y);
+  return blur(vec2i(pos.xy), dir, 1);
+}

blur4/shaders/downscale.fragment.wgsl

@@ -0,0 +1,21 @@
+struct VertexOut {
+  @builtin(position) pos: vec4f,
+  @location(0) uv: vec2f,
+}
+
+@vertex
+fn vertMain(@builtin(vertex_index) vertexIndex: u32) -> VertexOut {
+  var pos = array<vec2f, 4>(
+    vec2f(-1.0, -1.0), vec2f(1.0, -1.0), vec2f(-1.0, 1.0), vec2f(1.0, 1.0),
+  );
+  var uv = pos[vertexIndex] * vec2f(0.5, -0.5) + 0.5;
+  return VertexOut(vec4f(pos[vertexIndex], 0, 1), uv);
+}
+
+@group(0) @binding(0) var inputTexture: ${inputTextureType};
+@group(0) @binding(1) var textureSampler: sampler;
+
+@fragment
+fn fragMain(@location(0) uv: vec2f) -> @location(0) vec4f {
+    return textureSampleBaseClampToEdge(inputTexture, textureSampler, uv) + vec4f(0, 1, 0, 0);
+}