Building a WebXR App with Three.js

WebXR Hit Testing & Depth Sensing

Comprehensive guide to WebXR hit testing and depth sensing — ray intersection, transient hit tests, reading the depth buffer, occlusion rendering, and mesh reconstruction for real-world interaction.

A beginner-friendly introduction to WebXR — building virtual and augmented reality experiences that run directly in the browser without plugins.

This guide walks through building a complete WebXR application from scratch — a VR scene with controller interaction, teleportation movement, and world-space UI. By the end, you'll have a deployable app that incorporates the APIs covered throughout this series.

Project Setup

We'll use Vite for fast development and TypeScript for type safety:

npm create vite@latest my-xr-app -- --template vanilla-ts
cd my-xr-app
npm install three @webxr-input-profiles/motion-controllers
npm install --save-dev @types/three

TypeScript Configuration

Enable WebXR types in tsconfig.json:

{
  "compilerOptions": {
    "target": "ES2020",
    "module": "ESNext",
    "moduleResolution": "bundler",
    "strict": true,
    "types": ["three", "webxr"]
  }
}

Add the WebXR type reference at the top of your entry file:

/// <reference types="webxr" />

You may also need to add the types glob in a *.d.ts file:

// src/xr-types.d.ts
import 'webxr';

Application Architecture

my-xr-app/
├── src/
│   ├── main.ts          # Entry point — session setup, loop
│   ├── scene.ts         # Scene, camera, renderer setup
│   ├── controllers.ts   # Controller model loading, input events
│   ├── teleport.ts      # Teleportation movement system
│   ├── ui.ts            # World-space UI
│   └── xr-types.d.ts    # WebXR type declarations
├── public/
│   └── models/          # GLTF models
├── index.html
├── package.json
└── tsconfig.json

Session Setup (main.ts)

import { setupScene } from './scene';
import { setupControllers } from './controllers';
import { setupTeleport } from './teleport';

async function main() {
  // Check WebXR support
  if (!navigator.xr) {
    document.body.innerHTML = '<h1>WebXR not supported in this browser</h1>';
    return;
  }

  // Request immersive VR session
  const session = await navigator.xr.requestSession('immersive-vr', {
    requiredFeatures: ['local-floor', 'hand-tracking']
  });

  // Set up the Three.js scene
  const { renderer, scene, camera } = setupScene(session);

  // Set up controllers
  const controllers = setupControllers(renderer, scene);

  // Set up teleportation
  const teleport = setupTeleport(renderer, scene, session);

  // Handle session end
  session.addEventListener('end', () => {
    renderer.setAnimationLoop(null);
  });

  // Start the render loop
  renderer.setAnimationLoop((time: number, frame: XRFrame | null) => {
    if (frame) {
      controllers.update(frame, renderer.xr.getReferenceSpace()!);
      teleport.update(frame);
    }
    renderer.render(scene, camera);
  });
}

main().catch(console.error);

The renderer.setAnimationLoop() method replaces requestAnimationFrame() when XR is active. The callback receives an XRFrame object each frame.

Scene Setup (scene.ts)

import * as THREE from 'three';

let _session: XRSession | null = null;

export function setupScene(session: XRSession) {
  _session = session;

  const renderer = new THREE.WebGLRenderer({
    antialias: true,
    alpha: true
  });
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.setSize(window.innerWidth, window.innerHeight);
  renderer.xr.enabled = true;
  renderer.xr.setFoveation(0.5); // Balance quality/performance
  document.body.appendChild(renderer.domElement);

  const scene = new THREE.Scene();
  scene.background = new THREE.Color(0x111122);

  // Lighting
  const ambient = new THREE.AmbientLight(0x404060);
  scene.add(ambient);

  const directional = new THREE.DirectionalLight(0xffffff, 1);
  directional.position.set(1, 3, 2);
  scene.add(directional);

  // Floor grid
  const grid = new THREE.GridHelper(10, 10, 0x4444ff, 0x222244);
  scene.add(grid);

  // Interactive objects
  const geometry = new THREE.BoxGeometry(0.2, 0.2, 0.2);
  const material = new THREE.MeshStandardMaterial({ color: 0x44aaff });
  for (let i = 0; i < 10; i++) {
    const cube = new THREE.Mesh(geometry, material);
    cube.position.set(
      (Math.random() - 0.5) * 4,
      0.1,
      (Math.random() - 0.5) * 4 - 1
    );
    scene.add(cube);
  }

  // Camera (handled by Three.js XR manager)
  const camera = new THREE.PerspectiveCamera(70, window.innerWidth / window.innerHeight, 0.1, 100);
  camera.position.set(0, 1.6, 0);

  return { renderer, scene, camera };
}

Key points:

• renderer.xr.enabled = true — tells Three.js to use XR camera transforms instead of the scene camera
• setFoveation() reduces peripheral resolution for performance
• The scene camera's initial position becomes the seated/standing origin

Controller Input (controllers.ts)

Three.js provides XRRenderer.getController() to represent each tracked input:

import * as THREE from 'three';

export interface ControllerState {
  position: THREE.Vector3;
  quaternion: THREE.Quaternion;
  selectPressed: boolean;
  squeezePressed: boolean;
}

export function setupControllers(renderer: THREE.WebGLRenderer, scene: THREE.Scene) {
  const controllers: { [key: string]: ControllerState } = {};
  const controllerMeshes: THREE.Object3D[] = [];

  // Create a controller visual (simple ray + tip)
  function createControllerVisual(): THREE.Group {
    const group = new THREE.Group();

    // Ray
    const rayGeometry = new THREE.BufferGeometry();
    const rayVertices = new Float32Array([0, 0, 0, 0, 0, -1]);
    rayGeometry.setAttribute('position', new THREE.BufferAttribute(rayVertices, 3));
    const rayMaterial = new THREE.LineBasicMaterial({ color: 0x8888ff });
    const ray = new THREE.Line(rayGeometry, rayMaterial);
    group.add(ray);

    // Tip
    const tipGeometry = new THREE.SphereGeometry(0.01, 8, 8);
    const tipMaterial = new THREE.MeshBasicMaterial({ color: 0xffffff });
    const tip = new THREE.Mesh(tipGeometry, tipMaterial);
    tip.position.set(0, 0, -1);
    group.add(tip);

    return group;
  }

  // Left controller
  const leftGroup = createControllerVisual();
  const leftController = renderer.xr.getController(0);
  leftController.add(leftGroup);
  scene.add(leftController);
  controllerMeshes.push(leftGroup);

  // Right controller
  const rightGroup = createControllerVisual();
  const rightController = renderer.xr.getController(1);
  rightController.add(rightGroup);
  scene.add(rightController);
  controllerMeshes.push(rightGroup);

  // Input events
  function setupInput(hand: number, state: ControllerState) {
    const controller = renderer.xr.getController(hand);

    controller.addEventListener('selectstart', () => {
      state.selectPressed = true;
    });
    controller.addEventListener('selectend', () => {
      state.selectPressed = false;
    });
    controller.addEventListener('squeezestart', () => {
      state.squeezePressed = true;
    });
    controller.addEventListener('squeezeend', () => {
      state.squeezePressed = false;
    });

    controllers[`hand-${hand}`] = state;
  }

  setupInput(0, { position: new THREE.Vector3(), quaternion: new THREE.Quaternion(), selectPressed: false, squeezePressed: false });
  setupInput(1, { position: new THREE.Vector3(), quaternion: new THREE.Quaternion(), selectPressed: false, squeezePressed: false });

  // Update controller state each frame
  return {
    controllers,
    getController: (index: number) => renderer.xr.getController(index),
    update: (frame: XRFrame, refSpace: XRReferenceSpace) => {
      for (const inputSource of session.inputSources) {
        const handIndex = inputSource.handedness === 'left' ? 0 : 1;
        const pose = frame.getPose(inputSource.gripSpace!, refSpace);
        if (pose && controllers[`hand-${handIndex}`]) {
          const state = controllers[`hand-${handIndex}`];
          state.position.copy(pose.transform.position);
          state.quaternion.copy(pose.transform.orientation);
        }
      }
    }
  };
}

Controller Events

Teleportation (teleport.ts)

Teleportation is the standard VR locomotion pattern — point where you want to go, release to move:

import * as THREE from 'three';

export function setupTeleport(
  renderer: THREE.WebGLRenderer,
  scene: THREE.Scene,
  session: XRSession
) {
  // Arc visualization
  const arcPoints = 20;
  const arcGeometry = new THREE.BufferGeometry();
  const arcPositions = new Float32Array((arcPoints + 1) * 3);
  arcGeometry.setAttribute('position', new THREE.BufferAttribute(arcPositions, 3));
  const arcMaterial = new THREE.LineBasicMaterial({
    color: 0x44ff44,
    transparent: true,
    opacity: 0.6
  });
  const arcLine = new THREE.Line(arcGeometry, arcMaterial);
  arcLine.visible = false;
  scene.add(arcLine);

  // Target ring
  const ringGeometry = new THREE.RingGeometry(0.1, 0.15, 32);
  const ringMaterial = new THREE.MeshBasicMaterial({
    color: 0x44ff44,
    transparent: true,
    opacity: 0.8,
    side: THREE.DoubleSide
  });
  const targetRing = new THREE.Mesh(ringGeometry, ringMaterial);
  targetRing.visible = false;
  scene.add(targetRing);

  const camera = renderer.xr.getCamera();

  return {
    update: (frame: XRFrame) => {
      const refSpace = renderer.xr.getReferenceSpace();
      if (!refSpace) return;

      // Use right controller for aiming
      const rightController = renderer.xr.getController(1);
      const controllerPose = frame.getPose(rightController.targetRaySpace, refSpace);
      if (!controllerPose) {
        arcLine.visible = false;
        targetRing.visible = false;
        return;
      }

      // Compute parabolic arc
      const start = controllerPose.transform.position;
      const forward = new THREE.Vector3(0, 0, -1);
      forward.applyQuaternion(controllerPose.transform.orientation);

      // Physics for arc: simulate velocity with gravity
      const velocity = forward.clone().multiplyScalar(2);
      velocity.y += 0.5; // Upward arc

      const points: THREE.Vector3[] = [];
      const gravity = new THREE.Vector3(0, -9.8, 0);
      const step = 0.05; // 50ms per step

      for (let t = 0; t < arcPoints; t++) {
        const time = t * step;
        const point = start.clone()
          .add(velocity.clone().multiplyScalar(time))
          .add(gravity.clone().multiplyScalar(0.5 * time * time));

        if (point.y < 0) {
          point.y = 0; // Hit floor level
          points.push(point);
          break;
        }
        points.push(point);
      }

      // Update arc visual
      const positions = arcLine.geometry.attributes.position.array as Float32Array;
      for (let i = 0; i < points.length && i < arcPoints + 1; i++) {
        positions[i * 3] = points[i].x;
        positions[i * 3 + 1] = points[i].y;
        positions[i * 3 + 2] = points[i].z;
      }
      arcLine.geometry.attributes.position.needsUpdate = true;
      arcLine.visible = points.length > 0;

      // Update target ring
      if (points.length > 1) {
        const target = points[points.length - 1];
        targetRing.position.copy(target);
        targetRing.visible = true;
      }

      // Teleport on trigger release
      // (Handled via controller 'selectend' event in main.ts)
    },

    teleportTo: (position: THREE.Vector3) => {
      // Offset camera position
      const offset = position.clone().sub(camera.position);
      camera.position.add(offset);
    }
  };
}

The parabolic arc uses simple physics simulation — gravity + initial velocity. The selectend event triggers the teleport, moving the camera to the target position.

World-Space UI (ui.ts)

For simple UIs in VR, use sprite-based or plane-based panels:

import * as THREE from 'three';

export function createVrPanel(
  scene: THREE.Scene,
  text: string,
  width = 0.5,
  height = 0.15
): THREE.Group {
  const group = new THREE.Group();

  // Background
  const bgGeometry = new THREE.PlaneGeometry(width, height);
  const bgMaterial = new THREE.MeshBasicMaterial({
    color: 0x222244,
    transparent: true,
    opacity: 0.8,
    side: THREE.DoubleSide
  });
  const background = new THREE.Mesh(bgGeometry, bgMaterial);
  group.add(background);

  // Border
  const edgeGeometry = new THREE.EdgesGeometry(bgGeometry);
  const edgeMaterial = new THREE.LineBasicMaterial({ color: 0x6666ff });
  const border = new THREE.LineSegments(edgeGeometry, edgeMaterial);
  group.add(border);

  // Text using canvas texture (simple approach)
  const canvas = document.createElement('canvas');
  canvas.width = 256;
  canvas.height = 64;
  const ctx = canvas.getContext('2d')!;
  ctx.fillStyle = '#ffffff';
  ctx.font = '24px monospace';
  ctx.textAlign = 'center';
  ctx.textBaseline = 'middle';
  ctx.fillText(text, 128, 32);

  const texture = new THREE.CanvasTexture(canvas);
  const textGeometry = new THREE.PlaneGeometry(width * 0.9, height * 0.6);
  const textMaterial = new THREE.MeshBasicMaterial({
    map: texture,
    transparent: true,
    side: THREE.DoubleSide
  });
  const textMesh = new THREE.Mesh(textGeometry, textMaterial);
  textMesh.position.z = 0.001; // Slightly in front of background
  group.add(textMesh);

  scene.add(group);
  return group;
}

For more complex UIs, consider:

• drei HTML — React Three Fiber's HTML overlay
• CSS2DRenderer — Three.js addon for 2D overlays in 3D space
• IMGUI-style — Immediate mode GUIs with ray intersection

Loading GLTF Models

import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';

const loader = new GLTFLoader();
loader.load('/models/my-model.glb', (gltf) => {
  const model = gltf.scene;
  model.scale.set(0.1, 0.1, 0.1);
  scene.add(model);
});

Performance Optimization

// Adjust foveation level based on scene complexity
renderer.xr.setFoveation(complexScene ? 0.7 : 0.3);

// Enable XR multisampling (if available)
renderer.xr.setFramebufferScaleFactor(0.8); // Lower = better performance

// Use instancing for repeated objects
const matrix = new THREE.InstancedMesh(geometry, material, count);

// Limit draw calls
renderer.sortObjects = true; // Enable depth sorting

Frame Budget Guide

Profile with the browser's about://inspect tool or Meta's Performance HUD overlay.

Deployment

WebXR requires HTTPS to function (camera and tracking APIs).

Static Hosting

npm run build

# Deploy to any static host
npx serve dist          # Local test
# OR: upload to Vercel, Netlify, or your own server

Docker Deployment

FROM nginx:alpine
COPY dist/ /usr/share/nginx/html/
COPY nginx.conf /etc/nginx/conf.d/default.conf

With HTTPS via your reverse proxy (Caddy, Traefik, nginx + certbot):

server {
    listen 443 ssl;
    server_name xr.mydomain.com;

    ssl_certificate /etc/letsencrypt/live/xr.mydomain.com/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/xr.mydomain.com/privkey.pem;

    location / {
        root /usr/share/nginx/html;
        try_files $uri $uri/ /index.html;
        add_header 'Cross-Origin-Embedder-Policy' 'require-corp';
        add_header 'Cross-Origin-Opener-Policy' 'same-origin';
    }
}

The Cross-Origin headers are important for WebXR performance features (SharedArrayBuffer).

Tying It All Together

In main.ts, wire everything together:

import { setupScene } from './scene';
import { setupControllers } from './controllers';
import { setupTeleport } from './teleport';
import { createVrPanel } from './ui';

async function main() {
  if (!navigator.xr) { /* fallback */ return; }

  const session = await navigator.xr.requestSession('immersive-vr', {
    requiredFeatures: ['local-floor']
  });

  const { renderer, scene, camera } = setupScene(session);
  const controllers = setupControllers(renderer, scene);
  const teleport = setupTeleport(renderer, scene, session);

  // Title panel
  const titlePanel = createVrPanel(scene, 'My WebXR App');
  titlePanel.position.set(0, 1.8, -1);

  // Teleport on trigger release
  renderer.xr.getController(1).addEventListener('selectend', () => {
    const refSpace = renderer.xr.getReferenceSpace();
    if (refSpace) {
      // teleport.teleportTo(currentTarget);
    }
  });

  session.addEventListener('end', () => renderer.setAnimationLoop(null));

  // Handle window resize
  window.addEventListener('resize', () => {
    camera.aspect = window.innerWidth / window.innerHeight;
    camera.updateProjectionMatrix();
    renderer.setSize(window.innerWidth, window.innerHeight);
  });

  renderer.setAnimationLoop((time: number, frame: XRFrame | null) => {
    if (frame) {
      const refSpace = renderer.xr.getReferenceSpace();
      if (refSpace) controllers.update(frame, refSpace);
      teleport.update(frame);
    }
    renderer.render(scene, camera);
  });
}

main();

This is your foundation. From here, you can add any of the APIs from this series:

• Hand tracking from Article 1 — natural gesture interaction
• Anchors from Article 2 — persistent world-locked objects
• Composition layers from Article 3 — high-performance video and UI
• Hit testing & depth from Article 4 — real-world interaction

Series Cross-References

• WebXR Hand Tracking — Add hand tracking to your app
• WebXR Anchors & Plane Detection — Persistent objects in user space
• WebXR Composition Layers — High-performance video and UI layers
• WebXR Hit Testing & Depth Sensing — Real-world interaction and occlusion

References

• Three.js XR Documentation
• WebXR Device API Spec (W3C)
• Meta Quest Development Hub
• Vite Documentation