more work

README.md

@@ -183,9 +183,10 @@ I exited to see what people will do with AR.js :)
 # Futures
 - add webworkers
 - add the nft
-- handle sensor fusion with the IMU ?
+- handle sensor fusion with the IMU ? Assume that the marker is fixed in space
 - marker removal in the video - https://twitter.com/jerome_etienne/status/838584931009835009
 - dynamic multiple markers
+- put the video in the webgl (and not the dom), as an options
 
 # Ideas
 - AR Gaming - https://www.youtube.com/watch?v=EmGGGzibGok
@@ -202,6 +203,7 @@ I exited to see what people will do with AR.js :)
   - find the kind of matrix it is.
   - just put each marker in front of the camera - using image / photo of screen
   - and go thru each and every matrix type detection
+- Nice effect with video texture - https://www.youtube.com/watch?v=Y9HMn6bd-v8&feature=youtu.be&t=172
 
 # How To Release ?
 

TODO.md

@@ -47,6 +47,13 @@
 - do that in a single html. markercache.html 
   - threex.markercache.js - this is just the plane with the special shader
   - attached to the markerRoot, provide a texture where to take the pixels
+- apply the model view matrix on the UV, then the projection matrix
+  - do that first in javascript, manually
+  - clone markerCache.object3d.geometry.faceVertexUvs
+  - change the UV on every fps
+  - take the 2d vectors, convert it in 3d, apply the matrices
+  - set markerCache.object3d.geometry.uvsNeedUpdate = true
+  - test the matrices on the vertices - so require to have orthogonal camera and render vertices on it
 
 # webvr-polyfill
 - GOAL: works well using only the positional tracking, not the stereo display

three.js/examples/markercache.html

@@ -35,6 +35,8 @@
 	renderer.domElement.style.left = '0px'
 	document.body.appendChild( renderer.domElement );
 
+	renderer.autoClear = false;
+
 	// array of functions for the rendering loop
 	var onRenderFcts= [];
 
@@ -55,6 +57,13 @@
 	// Create a camera
 	var camera = new THREE.Camera();
 	scene.add(camera);
+	
+	//////////////////////////////////////////////////////////////////////////////
+	//		Code Separator
+	//////////////////////////////////////////////////////////////////////////////
+
+        var orthoCamera = new THREE.OrthographicCamera(-1, 1, 1, -1, -100, +100);
+        var orthoScene = new THREE.Scene();
 
 	////////////////////////////////////////////////////////////////////////////////
 	//          handle arToolkitSource
@@ -69,8 +78,8 @@
 		// sourceUrl : '../../data/images/img.jpg',		
 
 		// to read from a video
-		sourceType : 'video',
-		sourceUrl : '../../data/videos/headtracking.mp4',		
+		// sourceType : 'video',
+		// sourceUrl : '../../data/videos/headtracking.mp4',		
 	})
 
 	arToolkitSource.init(function onReady(){
@@ -118,44 +127,147 @@
 	scene.add(markerRoot)
 	var artoolkitMarker = new THREEx.ArMarkerControls(arToolkitContext, markerRoot, {
 		type : 'pattern',
-		// patternUrl : '../../data/data/patt.hiro',
-		patternUrl : '../../data/data/patt.kanji',
+		patternUrl : '../../data/data/patt.hiro',
+		// patternUrl : '../../data/data/patt.kanji',
 	})
 
 	//////////////////////////////////////////////////////////////////////////////////
 	//		add an object in the scene
 	//////////////////////////////////////////////////////////////////////////////////
 
-	arToolkitSource.domElement.pause()
-
+	// arToolkitSource.domElement.pause()
 	var videoTexture = new THREE.VideoTexture(arToolkitSource.domElement)
 	videoTexture.minFilter =  THREE.NearestFilter
+	// videoTexture.wrapS = videoTexture.wrapT = THREE.RepeatWrapping;	
+	// videoTexture.repeat.set( 1, 5 )
 
-	var markerCache =  new THREEx.ArMarkerCache(videoTexture);
-	markerRoot.add(markerCache.object3d)
-
-
-
-	// // add a torus knot	
-	// var geometry	= new THREE.CubeGeometry(1,1,1);
-	// var material	= new THREE.MeshNormalMaterial({
-	// 	transparent : true,
-	// 	opacity: 0.5,
-	// 	side: THREE.DoubleSide
-	// }); 
-	// var mesh	= new THREE.Mesh( geometry, material );
-	// mesh.position.z	= geometry.parameters.height/2
-	// markerRoot.add( mesh );
-	// 
-	// var geometry	= new THREE.TorusKnotGeometry(0.3,0.1,32,32);
-	// var material	= new THREE.MeshNormalMaterial(); 
-	// var mesh	= new THREE.Mesh( geometry, material );
-	// mesh.position.z	= 0.5
-	// markerRoot.add( mesh );
+	// var markerCache =  new THREEx.ArMarkerCache(videoTexture);
+	// markerRoot.add(markerCache.object3d)
 	
-	// onRenderFcts.push(function(){
-	// 	mesh.rotation.x += 0.1
-	// })
+	
+	;(function(){
+	        var material = new THREE.MeshNormalMaterial({
+			transparent : true,
+			opacity: 0.5,
+			side: THREE.DoubleSide
+		});
+	        var geometry = new THREE.PlaneGeometry(1,1);
+	        var orthoMesh = new THREE.Mesh(geometry, material);
+	        orthoScene.add(orthoMesh);
+
+		var geometry	= new THREE.PlaneGeometry(1.3,1.85);
+		var material	= new THREE.MeshBasicMaterial({
+			// transparent : true,
+			// opacity: 0.5,
+			side: THREE.DoubleSide,
+			map: videoTexture,
+		}); 
+		var cacheMesh = new THREE.Mesh( geometry, material );
+		cacheMesh.position.y = -0.3
+		markerRoot.add(cacheMesh)
+window.cacheMesh = cacheMesh
+		//////////////////////////////////////////////////////////////////////////////
+		//		Code Separator
+		//////////////////////////////////////////////////////////////////////////////
+
+		var xMin = -0.65
+		var xMax =  0.65
+		var yMin = -0.5
+		var yMin =  0.65
+		var yMax =  0.95
+
+		var originalUvs = []
+		originalUvs.push( new THREE.Vector3(xMin, yMax, 0))
+		originalUvs.push( new THREE.Vector3(xMax, yMax, 0))
+		originalUvs.push( new THREE.Vector3(xMin, yMin, 0))
+		originalUvs.push( new THREE.Vector3(xMax, yMin, 0))
+
+		onRenderFcts.push(function(){
+			var transformedUvs = []
+			originalUvs.forEach(function(originalUvs, index){
+				var transformedUv = originalUvs.clone()
+				// apply modelViewMatrix and projectionMatrix
+				transformedUv.applyMatrix4( markerRoot.matrix )
+				transformedUv.applyMatrix4( camera.projectionMatrix )
+
+				// apply perspective
+				transformedUv.x /= transformedUv.z
+				transformedUv.y /= transformedUv.z
+				// store it
+				transformedUvs.push(transformedUv)
+			})
+
+			// change orthoMesh vertices
+			for(var i = 0; i < transformedUvs.length; i++){
+				orthoMesh.geometry.vertices[i].copy(transformedUvs[i])
+			}
+			orthoMesh.geometry.computeBoundingSphere()
+			orthoMesh.geometry.verticesNeedUpdate = true
+			
+			// change cacheMesh UVs
+			cacheMesh.geometry.faceVertexUvs[0][0][0].copy( buildUvs(0, 1) )
+			cacheMesh.geometry.faceVertexUvs[0][0][1].copy( buildUvs(0, 0) )
+			cacheMesh.geometry.faceVertexUvs[0][0][2].copy( buildUvs(1, 1) )
+			
+			cacheMesh.geometry.faceVertexUvs[0][1][0].copy( buildUvs(0, 0) )
+			cacheMesh.geometry.faceVertexUvs[0][1][1].copy( buildUvs(1, 0) )
+			cacheMesh.geometry.faceVertexUvs[0][1][2].copy( buildUvs(1, 1) )
+
+			cacheMesh.geometry.uvsNeedUpdate = true
+			function buildUvs(x, y){
+				var Uv = new THREE.Vector2()
+						// originalUvs.push( new THREE.Vector3(xMin, yMax, 0))
+						// originalUvs.push( new THREE.Vector3(xMax, yMax, 0))
+						// originalUvs.push( new THREE.Vector3(xMin, yMin, 0))
+						// originalUvs.push( new THREE.Vector3(xMax, yMin, 0))
+
+				if( x === 0 && y === 0 ){
+					var transformedUv = transformedUvs[2]
+				}else if( x === 0 && y === 1 ){
+					var transformedUv = transformedUvs[0]
+				}else if( x === 1 && y === 1 ){
+					var transformedUv = transformedUvs[1]
+				}else if( x === 1 && y === 0 ){
+					var transformedUv = transformedUvs[3]
+				}else {
+					console.assert(false)
+				}
+
+				var Uv = new THREE.Vector2()
+				Uv.x = transformedUv.x / 2 + 0.5
+				Uv.y = transformedUv.y / 2 + 0.5
+
+				// Uv.y *= 5
+				
+				return Uv
+			}
+		})
+	})()
+	
+	//////////////////////////////////////////////////////////////////////////////////
+	//		add an object in the scene
+	//////////////////////////////////////////////////////////////////////////////////
+
+	// add a torus knot	
+	var geometry	= new THREE.CubeGeometry(1,1,1);
+	var material	= new THREE.MeshNormalMaterial({
+		transparent : true,
+		opacity: 0.5,
+		side: THREE.DoubleSide
+	}); 
+	var mesh	= new THREE.Mesh( geometry, material );
+	mesh.position.z	= geometry.parameters.height/2
+	markerRoot.add( mesh );
+	
+	var geometry	= new THREE.TorusKnotGeometry(0.3,0.1,32,32);
+	var material	= new THREE.MeshNormalMaterial(); 
+	var mesh	= new THREE.Mesh( geometry, material );
+	mesh.position.z	= 0.5
+	markerRoot.add( mesh );
+	
+	onRenderFcts.push(function(){
+		mesh.rotation.x += 0.1
+	})
 
 	//////////////////////////////////////////////////////////////////////////////////
 	//		render the whole thing on the page
@@ -164,7 +276,14 @@
 	document.body.appendChild( stats.dom );
 	// render the scene
 	onRenderFcts.push(function(){
+		
+		renderer.clear();
+		
 		renderer.render( scene, camera );
+		
+        	// renderer.render( orthoScene, camera );
+        	renderer.render( orthoScene, orthoCamera );
+
 		stats.update();
 	})
 

three.js/examples/markercache0.html

+<!DOCTYPE html>
+<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
+<!-- three.js library -->
+<script src='vendor/three.js/build/three.js'></script>
+<script src="vendor/three.js/examples/js/libs/stats.min.js"></script>
+<!-- jsartookit -->
+<script src="../vendor/jsartoolkit5/build/artoolkit.min.js"></script>
+<script src="../vendor/jsartoolkit5/js/artoolkit.api.js"></script>
+<!-- include threex.artoolkit -->
+<script src="../threex-artoolkitsource.js"></script>
+<script src="../threex-artoolkitcontext.js"></script>
+<script src="../threex-artoolkitprofile.js"></script>
+<script src="../threex-armarkercontrols.js"></script>
+<script src="../threex-armarkercache.js"></script>
+
+<body style='margin : 0px; overflow: hidden; font-family: Monospace;'><div style='position: absolute; top: 10px; width:100%; text-align: center;z-index:1';>
+	<a href="https://github.com/jeromeetienne/AR.js/" target="_blank">AR.js</a> - developement playground
+	<br/>
+	Contact me any time at <a href='https://twitter.com/jerome_etienne' target='_blank'>@jerome_etienne</a>
+</div><script>
+	//////////////////////////////////////////////////////////////////////////////////
+	//		Init
+	//////////////////////////////////////////////////////////////////////////////////
+
+	// init renderer
+	var renderer	= new THREE.WebGLRenderer({
+		// antialias	: true,
+		alpha: true
+	});
+	renderer.setClearColor(new THREE.Color('lightgrey'), 0)
+	// renderer.setPixelRatio( 2 );
+	renderer.setSize( window.innerWidth, window.innerHeight );
+	renderer.domElement.style.position = 'absolute'
+	renderer.domElement.style.top = '0px'
+	renderer.domElement.style.left = '0px'
+	document.body.appendChild( renderer.domElement );
+
+	// array of functions for the rendering loop
+	var onRenderFcts= [];
+
+	// init scene and camera
+	var scene	= new THREE.Scene();
+
+	var ambient = new THREE.AmbientLight( 0x666666 );
+	scene.add( ambient );
+
+	var directionalLight = new THREE.DirectionalLight( 0x887766 );
+	directionalLight.position.set( -1, 1, 1 ).normalize();
+	scene.add( directionalLight );
+	
+	//////////////////////////////////////////////////////////////////////////////////
+	//		Initialize a basic camera
+	//////////////////////////////////////////////////////////////////////////////////
+
+	// Create a camera
+	var camera = new THREE.Camera();
+	scene.add(camera);
+
+	////////////////////////////////////////////////////////////////////////////////
+	//          handle arToolkitSource
+	////////////////////////////////////////////////////////////////////////////////
+
+	var arToolkitSource = new THREEx.ArToolkitSource({
+		// to read from the webcam 
+		sourceType : 'webcam',
+		
+		// to read from an image
+		// sourceType : 'image',
+		// sourceUrl : '../../data/images/img.jpg',		
+
+		// to read from a video
+		sourceType : 'video',
+		sourceUrl : '../../data/videos/headtracking.mp4',		
+	})
+
+	arToolkitSource.init(function onReady(){
+		console.log('source is ready')
+		// handle resize of renderer
+		arToolkitSource.onResize(renderer.domElement)		
+	})
+	
+	// handle resize
+	window.addEventListener('resize', function(){
+		// handle arToolkitSource resize
+		arToolkitSource.onResize(renderer.domElement)		
+	})
+
+	////////////////////////////////////////////////////////////////////////////////
+	//          initialize arToolkitContext
+	////////////////////////////////////////////////////////////////////////////////	
+
+	// create atToolkitContext
+	var arToolkitContext = new THREEx.ArToolkitContext({
+		cameraParametersUrl: '../../data/data/camera_para.dat',
+		detectionMode: 'mono',
+		sourceWidth: arToolkitSource.parameters.sourceWidth,
+		sourceHeight: arToolkitSource.parameters.sourceHeight,
+	})
+	// initialize it
+	arToolkitContext.init(function onCompleted(){
+		// copy projection matrix to camera
+		camera.projectionMatrix.copy( arToolkitContext.getProjectionMatrix() );
+	})
+
+	// update artoolkit on every frame
+	onRenderFcts.push(function(){
+		if( arToolkitSource.ready === false )	return
+
+		arToolkitContext.update( arToolkitSource.domElement )
+	})
+	
+	
+	////////////////////////////////////////////////////////////////////////////////
+	//          Create a ArMarkerControls
+	////////////////////////////////////////////////////////////////////////////////
+	
+	var markerRoot = new THREE.Group
+	scene.add(markerRoot)
+	var artoolkitMarker = new THREEx.ArMarkerControls(arToolkitContext, markerRoot, {
+		type : 'pattern',
+		// patternUrl : '../../data/data/patt.hiro',
+		patternUrl : '../../data/data/patt.kanji',
+	})
+
+	//////////////////////////////////////////////////////////////////////////////////
+	//		add an object in the scene
+	//////////////////////////////////////////////////////////////////////////////////
+
+	arToolkitSource.domElement.pause()
+
+	var videoTexture = new THREE.VideoTexture(arToolkitSource.domElement)
+	videoTexture.minFilter =  THREE.NearestFilter
+
+	var markerCache =  new THREEx.ArMarkerCache(videoTexture);
+	markerRoot.add(markerCache.object3d)
+
+	// // add a torus knot	
+	// var geometry	= new THREE.CubeGeometry(1,1,1);
+	// var material	= new THREE.MeshNormalMaterial({
+	// 	transparent : true,
+	// 	opacity: 0.5,
+	// 	side: THREE.DoubleSide
+	// }); 
+	// var mesh	= new THREE.Mesh( geometry, material );
+	// mesh.position.z	= geometry.parameters.height/2
+	// markerRoot.add( mesh );
+	// 
+	// var geometry	= new THREE.TorusKnotGeometry(0.3,0.1,32,32);
+	// var material	= new THREE.MeshNormalMaterial(); 
+	// var mesh	= new THREE.Mesh( geometry, material );
+	// mesh.position.z	= 0.5
+	// markerRoot.add( mesh );
+	
+	// onRenderFcts.push(function(){
+	// 	mesh.rotation.x += 0.1
+	// })
+
+	//////////////////////////////////////////////////////////////////////////////////
+	//		render the whole thing on the page
+	//////////////////////////////////////////////////////////////////////////////////
+	var stats = new Stats();
+	document.body.appendChild( stats.dom );
+	// render the scene
+	onRenderFcts.push(function(){
+		renderer.render( scene, camera );
+		stats.update();
+	})
+
+	// run the rendering loop
+	var lastTimeMsec= null
+	requestAnimationFrame(function animate(nowMsec){
+		// keep looping
+		requestAnimationFrame( animate );
+		// measure time
+		lastTimeMsec	= lastTimeMsec || nowMsec-1000/60
+		var deltaMsec	= Math.min(200, nowMsec - lastTimeMsec)
+		lastTimeMsec	= nowMsec
+		// call each update function
+		onRenderFcts.forEach(function(onRenderFct){
+			onRenderFct(deltaMsec/1000, nowMsec/1000)
+		})
+	})
+</script></body>