Add preliminary support for WebXR Layers (#22060)

* Add preliminary support for WebXR Layers * WebXR Layers: rename sample * Resolve merge conflicts for WebXR Layers * Update WebXRManager.js Clean up. * Update webxr_vr_layers.html Correct title. Co-authored-by: N Rik Cabanier <cabanier@fb.com> Co-authored-by: N Michael Herzog <michael.herzog@human-interactive.org>

Add preliminary support for WebXR Layers (#22060)
* Add preliminary support for WebXR Layers * WebXR Layers: rename sample * Resolve merge conflicts for WebXR Layers * Update WebXRManager.js Clean up. * Update webxr_vr_layers.html Correct title. Co-authored-by: N Rik Cabanier <cabanier@fb.com> Co-authored-by: N Michael Herzog <michael.herzog@human-interactive.org>
0938236d · Rik Cabanier · GitHub · 81d12dda · 0938236d · 0938236d
5 changed file
--- a/examples/files.json
+++ b/examples/files.json
@@ -355,7 +355,8 @@
 		"webxr_vr_rollercoaster",
 		"webxr_vr_sandbox",
 		"webxr_vr_sculpt",
-		"webxr_vr_video"
+		"webxr_vr_video",
+		"webxr_vr_layers"
 	],
 	"games": [
 		"games_fps"

--- a/examples/jsm/webxr/VRButton.js
+++ b/examples/jsm/webxr/VRButton.js
@@ -68,7 +68,7 @@ class VRButton {
 					// ('local' is always available for immersive sessions and doesn't need to
 					// be requested separately.)

-					const sessionInit = { optionalFeatures: [ 'local-floor', 'bounded-floor', 'hand-tracking' ] };
+					const sessionInit = { optionalFeatures: [ 'local-floor', 'bounded-floor', 'hand-tracking', 'layers' ] };
 					navigator.xr.requestSession( 'immersive-vr', sessionInit ).then( onSessionStarted );

 				} else {

--- a/examples/screenshots/webxr_vr_layers.jpg
+++ b/examples/screenshots/webxr_vr_layers.jpg
--- a/examples/webxr_vr_layers.html
+++ b/examples/webxr_vr_layers.html
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>three.js vr - layers</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+	<body>
+
+		<div id="info">
+			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> media and projection layers<br/>
+			(Oculus Browser with #webxr-hands and #webxr-layers flags enabled)
+		</div>
+
+		<script type="module">
+
+			import * as THREE from '../build/three.module.js';
+			import { OrbitControls } from './jsm/controls/OrbitControls.js';
+			import { VRButton } from './jsm/webxr/VRButton.js';
+			import { XRControllerModelFactory } from './jsm/webxr/XRControllerModelFactory.js';
+			import { XRHandModelFactory } from './jsm/webxr/XRHandModelFactory.js';
+
+			let container;
+			let camera, scene, renderer;
+			let hand1, hand2;
+			let controller1, controller2;
+			let controllerGrip1, controllerGrip2;
+
+			let controls;
+			let video;
+			let video2;
+
+			init();
+			animate();
+
+			function init() {
+
+				container = document.createElement( 'div' );
+				document.body.appendChild( container );
+
+				scene = new THREE.Scene();
+
+				camera = new THREE.PerspectiveCamera( 50, window.innerWidth / window.innerHeight, 0.1, 10 );
+				camera.position.set( 0, 1.6, 3 );
+
+				controls = new OrbitControls( camera, container );
+				controls.target.set( 0, 1.6, 0 );
+				controls.update();
+
+				scene.add( new THREE.HemisphereLight( 0x808080, 0x606060 ) );
+
+				const light = new THREE.DirectionalLight( 0xffffff );
+				light.position.set( 0, 6, 0 );
+				light.castShadow = true;
+				light.shadow.camera.top = 2;
+				light.shadow.camera.bottom = - 2;
+				light.shadow.camera.right = 2;
+				light.shadow.camera.left = - 2;
+				light.shadow.mapSize.set( 4096, 4096 );
+				scene.add( light );
+
+				//
+
+				renderer = new THREE.WebGLRenderer( { antialias: true, alpha: true } );
+				renderer.setPixelRatio( window.devicePixelRatio );
+				renderer.setClearAlpha(1);
+				renderer.setClearColor(new THREE.Color(0), 0);
+				renderer.setSize( window.innerWidth, window.innerHeight );
+				renderer.outputEncoding = THREE.sRGBEncoding;
+				renderer.shadowMap.enabled = true;
+				renderer.xr.enabled = true;
+
+				container.appendChild( renderer.domElement );
+
+				document.body.appendChild( VRButton.createButton( renderer ) );
+
+				// controllers
+
+				controller1 = renderer.xr.getController( 0 );
+				scene.add( controller1 );
+
+				controller2 = renderer.xr.getController( 1 );
+				scene.add( controller2 );
+
+				const controllerModelFactory = new XRControllerModelFactory();
+				const handModelFactory = new XRHandModelFactory().setPath( "./models/fbx/" );
+
+				// Hand 1
+				controllerGrip1 = renderer.xr.getControllerGrip( 0 );
+				controllerGrip1.add( controllerModelFactory.createControllerModel( controllerGrip1 ) );
+				scene.add( controllerGrip1 );
+
+				hand1 = renderer.xr.getHand( 0 );
+				hand1.add( handModelFactory.createHandModel( hand1 ) );
+
+				scene.add( hand1 );
+
+				// Hand 2
+				controllerGrip2 = renderer.xr.getControllerGrip( 1 );
+				controllerGrip2.add( controllerModelFactory.createControllerModel( controllerGrip2 ) );
+				scene.add( controllerGrip2 );
+
+				hand2 = renderer.xr.getHand( 1 );
+				hand2.add( handModelFactory.createHandModel( hand2 ) );
+				scene.add( hand2 );
+
+				//
+
+				const geometry = new THREE.BufferGeometry().setFromPoints( [ new THREE.Vector3( 0, 0, 0 ), new THREE.Vector3( 0, 0, - 1 ) ] );
+
+				const line = new THREE.Line( geometry );
+				line.name = 'line';
+				line.scale.z = 5;
+
+				controller1.add( line.clone() );
+				controller2.add( line.clone() );
+
+				//
+
+				window.addEventListener( 'resize', onWindowResize, false );
+
+				video = document.createElement('video');
+				video.loop = true;
+				video.src = 'textures/pano.webm';
+
+				video2 = document.createElement('video');
+				video2.loop = true;
+				video2.src = 'textures/MaryOculus.webm';
+			}
+
+			function onWindowResize() {
+
+				camera.aspect = window.innerWidth / window.innerHeight;
+				camera.updateProjectionMatrix();
+
+				renderer.setSize( window.innerWidth, window.innerHeight );
+
+			}
+			//
+
+			function animate() {
+
+				renderer.setAnimationLoop( render );
+
+			}
+
+			function render() {
+				const xr = renderer.xr;
+				const session = xr.getSession();
+
+				if ( session && session.renderState.layers !== undefined && session.hasMediaLayer === undefined && video.readyState >= 2 && video2.readyState >= 2) {
+					session.hasMediaLayer = true;
+					session.requestReferenceSpace('local').then((refSpace) => {
+						const mediaBinding = new XRMediaBinding(session);
+						const equirectLayer = mediaBinding.createEquirectLayer(video, {space: refSpace, layout: "mono"});
+						const quadLayer = mediaBinding.createQuadLayer(video2, {space: refSpace, layout: "stereo-left-right"});
+						quadLayer.transform = new XRRigidTransform({x: 1.5, y: 1.0, z: -2.0});
+						session.updateRenderState( { layers: [ equirectLayer, quadLayer, session.renderState.layers[0] ] } );
+						video.play();
+						video2.play();
+					});
+				}
+
+				renderer.render( scene, camera );
+
+			}
+
+		</script>
+	</body>
+</html>
--- a/src/renderers/webxr/WebXRManager.js
+++ b/src/renderers/webxr/WebXRManager.js
@@ -16,13 +16,15 @@ class WebXRManager extends EventDispatcher {
 		const state = renderer.state;

 		let session = null;
-
 		let framebufferScaleFactor = 1.0;

 		let referenceSpace = null;
 		let referenceSpaceType = 'local-floor';

 		let pose = null;
+		let glBinding = null;
+		let glFramebuffer = null;
+		let glProjLayer = null;

 		const controllers = [];
 		const inputSourcesMap = new Map();
@@ -199,18 +201,47 @@ class WebXRManager extends EventDispatcher {

 				}

-				const layerInit = {
-					antialias: attributes.antialias,
-					alpha: attributes.alpha,
-					depth: attributes.depth,
-					stencil: attributes.stencil,
-					framebufferScaleFactor: framebufferScaleFactor
-				};
+				if ( session.renderState.layers === undefined ) {
+
+					const layerInit = {
+						antialias: attributes.antialias,
+						alpha: attributes.alpha,
+						depth: attributes.depth,
+						stencil: attributes.stencil,
+						framebufferScaleFactor: framebufferScaleFactor
+					};
+
+					// eslint-disable-next-line no-undef
+					const baseLayer = new XRWebGLLayer( session, gl, layerInit );
+
+					session.updateRenderState( { baseLayer: baseLayer } );
+
+				} else {
+
+					let depthFormat = 0;
+
+					if ( attributes.depth ) {
+
+						depthFormat = attributes.stencil ? gl.DEPTH_STENCIL : gl.DEPTH_COMPONENT;
+
+					}
+
+					const projectionlayerInit = {
+						colorFormat: attributes.alpha ? gl.RGBA : gl.RGB,
+						depthFormat: depthFormat,
+						scaleFactor: framebufferScaleFactor
+					};
+
+					// eslint-disable-next-line no-undef
+					glBinding = new XRWebGLBinding( session, gl );
+
+					glProjLayer = glBinding.createProjectionLayer( projectionlayerInit );

-				// eslint-disable-next-line no-undef
-				const baseLayer = new XRWebGLLayer( session, gl, layerInit );
+					glFramebuffer = gl.createFramebuffer();

-				session.updateRenderState( { baseLayer: baseLayer } );
+					session.updateRenderState( { layers: [ glProjLayer ] } );
+
+				}

 				referenceSpace = await session.requestReferenceSpace( referenceSpaceType );

@@ -429,9 +460,14 @@ class WebXRManager extends EventDispatcher {
 			if ( pose !== null ) {

 				const views = pose.views;
+
 				const baseLayer = session.renderState.baseLayer;

-				state.bindXRFramebuffer( baseLayer.framebuffer );
+				if ( session.renderState.layers === undefined ) {
+
+					state.bindXRFramebuffer( baseLayer.framebuffer );
+
+				}

 				let cameraVRNeedsUpdate = false;

@@ -440,18 +476,50 @@ class WebXRManager extends EventDispatcher {
 				if ( views.length !== cameraVR.cameras.length ) {

 					cameraVR.cameras.length = 0;
+
 					cameraVRNeedsUpdate = true;

+
 				}

 				for ( let i = 0; i < views.length; i ++ ) {

 					const view = views[ i ];
-					const viewport = baseLayer.getViewport( view );
+
+					let viewport = null;
+
+					if ( session.renderState.layers === undefined ) {
+
+						viewport = baseLayer.getViewport( view );
+
+					} else {
+
+						const glSubImage = glBinding.getViewSubImage( glProjLayer, view );
+
+						gl.bindFramebuffer( gl.FRAMEBUFFER, glFramebuffer );
+
+						gl.framebufferTexture2D( gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, glSubImage.colorTexture, 0 );
+
+						if ( glSubImage.depthStencilTexture !== undefined ) {
+
+							gl.framebufferTexture2D( gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.TEXTURE_2D, glSubImage.depthStencilTexture, 0 );
+
+						}
+
+						gl.bindFramebuffer( gl.FRAMEBUFFER, null );
+
+						state.bindXRFramebuffer( glFramebuffer );
+
+						viewport = glSubImage.viewport;
+
+					}

 					const camera = cameras[ i ];
+
 					camera.matrix.fromArray( view.transform.matrix );
+
 					camera.projectionMatrix.fromArray( view.projectionMatrix );
+
 					camera.viewport.set( viewport.x, viewport.y, viewport.width, viewport.height );

 					if ( i === 0 ) {