Trouble Rendering Dynamic Texture from OffscreenCanvas in Three.js Shader

Question

I'm working on a React Three Fiber project where I'm trying to render a dynamic texture from an OffscreenCanvas that's being updated in a WebWorker. However, all I'm getting is a black render on my mesh, and I'm struggling to pinpoint the issue.

I'm using THREE.CanvasTexture for the texture, a custom shader for the material, and a useFrame loop to update the texture. Below are the simplified and relevant parts of my code:

Component (component.ts):

import { useRef, useEffect, useState } from "react";
import { useFrame } from "@react-three/fiber";
import * as THREE from 'three';
import TextureShader from "./TextureShader";

const MyComponent = () => {
  const [texture, setTexture] = useState<THREE.Texture>();
  const meshRef = useRef<THREE.Mesh>(null);

  useEffect(() => {
    const offscreenCanvas = new OffscreenCanvas(100, 100);
    const canvasTexture = new THREE.CanvasTexture(offscreenCanvas);
    setTexture(canvasTexture);

    // Initialize and post message to worker (not shown for brevity)

    // ... other initialization code ...
  }, []);

  useFrame(() => {
    if (texture) {
      texture.needsUpdate = true;
    }
  });

  return (
    <mesh ref={meshRef}>
      <boxBufferGeometry args={[1, 1, 1]} />
      <TextureShader uTexture={texture} />
    </mesh>
  );
};

export default MyComponent;

Shader (TextureShader.ts):

import { shaderMaterial } from '@react-three/drei';
import * as THREE from 'three';

const vert = /* glsl */`
varying vec2 vUv;
void main() {
  vUv = uv;
  gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
}
`;

const frag = /* glsl */`
uniform sampler2D uTexture;
varying vec2 vUv;
void main() {
  gl_FragColor = texture2D(uTexture, vUv);
}
`;

const TextureShader = shaderMaterial({ uTexture: new THREE.Texture() }, vert, frag);

export default TextureShader;

WebWorker (worker.ts):

// Simplified worker logic for drawing on the OffscreenCanvas
self.onmessage = (event) => {
  if (event.data.canvas) {
    const canvas = event.data.canvas;
    const ctx = canvas.getContext('2d');

    // Drawing logic (e.g., ctx.fillRect(...))
  }
};

The texture should be dynamically updated by the WebWorker, but when applied to the mesh using the custom shader, it only renders black. The shader works fine with static colors, so I suspect the issue lies in either the dynamic texture update process or how the OffscreenCanvas is managed.

What I've tried:

Ensuring needsUpdate is set to true for the texture in the useFrame loop. Verifying that the WebWorker is correctly drawing on the OffscreenCanvas. Using a simpler material to rule out shader issues. Questions:

Could the issue be related to how the OffscreenCanvas is handled in the WebWorker? Is there a specific way to update a THREE.CanvasTexture from an OffscreenCanvas being drawn in a WebWorker? Any insights or suggestions would be appreciated

Answer 1

Just like when we transfer an ArrayBuffer instance it gets detached and its length is set to 0 and you can basically not do anything with it anymore, when transferring an OffscreenCanvas instance, it is also detached, its width and height are set to 0 and you can't access its drawing buffer anymore.

const canvas = new OffscreenCanvas(50, 50);
console.log("before", canvas.width, canvas.height); // 50, 50
// Transfer the OffscreenCanvas (even to the same JS context)
(new MessageChannel()).port1.postMessage("", [canvas]);
console.log("after", canvas.width, canvas.height); // 0, 0

What you could do, is to get your OffscreenCanvas from a <canvas> element, using its transferControlToOffscreen() method. Doing so, the placeholder <canvas> content would get updated every painting frame (basically at the same rate as requestAnimationFrame, and you'll be able to draw it from your main context.

(Using 2D contexts for ease of demonstration, but CanvasTexture should work the same.)

const placeholder = document.createElement("canvas");
const offscreen = placeholder.transferControlToOffscreen();
const worker = new Worker(
  URL.createObjectURL(new Blob(
    [document.querySelector("[type=worker-script]").textContent],
    { type: "text/javascript" }
  ))
);
worker.postMessage(offscreen, [offscreen]);
worker.onmessage = (evt) => {
  const ctx = document.querySelector("canvas").getContext("2d");
  ctx.drawImage(placeholder, 0, 0);
};

<script type=worker-script>
onmessage = ({data: canvas}) => {
  const ctx = canvas.getContext("2d");
  ctx.fillStyle = "green"
  ctx.fillRect(0, 0, 300, 150);
  // wait 'till next paint befre saying we're done
  requestAnimationFrame(() => postMessage("done"));
};
</script>
<canvas></canvas>

However, doing so, you're always one frame late, and you may face browser bugs (for instance my Firefox fails to use the placeholder canvas as source correctly).

So instead you will probably want to transfer ImageBitmap objects from your worker thread, calling canvas.transferToImageBitmap() and remember to close() it when you're done loading it as CanvasTexture.

<script type=worker-script>
  const canvas = new OffscreenCanvas(600, 600);
  const ctx = canvas.getContext("2d");
  const image = new ImageData(600, 600);
  const data = new Uint32Array(image.data.buffer);
  const update = async () => {
    for (let i = 0; i<data.length; i++) {
      data[i] = Math.random() * 0xFFFFFF + 0xFF000000;
    }
    ctx.putImageData(image, 0, 0);
    const bmp = await canvas.transferToImageBitmap();
    postMessage(bmp, [bmp]);
    requestAnimationFrame(update);
  };
  requestAnimationFrame(update);
</script>
<script type=importmap>
 {
  "imports": {
   "three/": "https://cdn.jsdelivr.net/npm/[email protected]/",
   "three": "https://cdn.jsdelivr.net/npm/[email protected]/build/three.module.js"
  }
 }
</script>
<canvas></canvas>
<script type="module">
import * as THREE from 'three';
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';

const worker = new Worker(
  URL.createObjectURL(
    new Blob(
      [document.querySelector("[type=worker-script]").textContent],
      { type: "text/javascript" }
    )
  )
);
// Scene based on the code from https://medium.com/@ashabb/threejs-texture-example-9f87953c8023
const canvas = document.querySelector("canvas");
const renderer = new THREE.WebGLRenderer({ canvas });
renderer.setSize(window.innerWidth, window.innerHeight);
const scene = new THREE.Scene();
scene.background = new THREE.Color("red");
const cubeTextureLoader = new THREE.CubeTextureLoader();
scene.background = new THREE.Color("green");
const ambientLight = new THREE.AmbientLight(0xffffff, 0.4)
scene.add(ambientLight)
const directionalLight = new THREE.DirectionalLight(0xffffff, 0.5);
scene.add(directionalLight);
const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 1, 1000);
camera.position.y = 100;
camera.position.z = 250;

const controls = new OrbitControls(camera, renderer.domElement);
controls.enableDamping = true;
controls.dampingFactor = 0.6;
controls.screenSpacePanning = false;
controls.update();

// Initialize the texture without image
const map = new THREE.CanvasTexture();

const floorGeometry = new THREE.PlaneGeometry(500, 500, 128, 128);
const floorMaterial = new THREE.MeshStandardMaterial({
  color: '#777777',
  metalness: 0.2,
  roughness: 0.6,
  envMapIntensity: 0.5,
  side: THREE.DoubleSide,
  map: map,
});
const floor = new THREE.Mesh(floorGeometry, floorMaterial)
floor.receiveShadow = true;
floor.rotation.x = - Math.PI * 0.5;
floor.position.set(0, 0, 0);
scene.add(floor);

function animate() {
  requestAnimationFrame(animate)
  renderer.render(scene, camera);
}
let animated = false;
// Update the texture from the worker's response
worker.onmessage = ({data}) => {
  map.image = data;
  map.needsUpdate = true;
  if (!animated) {
    animate();
    animated = true;
  }
};


</script>