Add puppeteer checks to eliminate freezing

.travis.yml

@@ -9,7 +9,7 @@ jobs:
     - &lint-n-unit
       stage: lint & unit
       name: lint
-      script: npm run lint
+      script: npm run test-lint
     - <<: *lint-n-unit
       name: unit
       script: npm run test-unit
@@ -17,11 +17,11 @@ jobs:
     - &diff
       stage: diff
       name: diff
-      script: FORCE_COLOR=0 npm run test-diff
-      env: CI=0
+      script: npm run test-diff
+      env: FORCE_COLOR=0 CI=0
     - <<: *diff
-      env: CI=1
+      env: FORCE_COLOR=0 CI=1
     - <<: *diff
-      env: CI=2
+      env: FORCE_COLOR=0 CI=2
     - <<: *diff
-      env: CI=3
+      env: FORCE_COLOR=0 CI=3

package.json

@@ -52,9 +52,9 @@
     "build-examples": "rollup -c utils/build/rollup-examples.config.js",
     "dev": "concurrently --names \"ROLLUP,HTTP\" -c \"bgBlue.bold,bgGreen.bold\" \"rollup -c utils/build/rollup.config.js -w -m inline\" \"serve\"",
     "dev-test": "concurrently --names \"ROLLUP,ROLLUPTEST,HTTP\" -c \"bgBlue.bold,bgRed.bold,bgGreen.bold\" \"rollup -c utils/build/rollup.config.js -w -m inline\" \"rollup -c test/rollup.unit.config.js -w -m inline\" \"serve\"",
-    "lint": "eslint src --ext js --ext ts && tsc -p utils/build/tsconfig.lint.json",
-    "lint-examples": "eslint examples/jsm --ext js --ext ts --ignore-pattern libs && tsc -p utils/build/tsconfig-examples.lint.json",
-    "test": "npm run test-unit && npm run test-diff",
+    "test": "npm run test-lint && npm run test-unit",
+    "test-lint": "eslint src --ext js --ext ts && tsc -p utils/build/tsconfig.lint.json",
+    "test-lint-examples": "eslint examples/jsm --ext js --ext ts --ignore-pattern libs && tsc -p utils/build/tsconfig-examples.lint.json",
     "test-unit": "npm run build-test && qunit -r failonlyreporter test/unit/three.source.unit.js",
     "test-diff": "node --expose-gc test/diff/puppeteer.js",
     "make-screenshot": "cross-env MAKE=true npm run test-diff"

test/diff/README.md

@@ -15,6 +15,15 @@ npx cross-env VISIBLE=ture npm run test-diff
 ```
 
 ### How it works
+
+|           Travis                        |               Attempts               |
+|-----------------------------------------|--------------------------------------|
+| 61 from 362 failed, time=21:14          | networkidle0 timeout                 |
+| 26 from 362 failed, time=16:22          | with rAF hook                        |
+| 13=1+1+7+4 failed, time=4:26            | with render promise and parallelism  |
+| 4=0+0+2+2 failed, time=5:13             | with network tax and other settings  |
+| 4=0+0+2+2 failed, time=3:26             | with progressive attempts            |
+
 - ci configs with parallelism
 - deterministic random/timer/rAF/video for screenshots
 - increased robustness with hided text, datgui, different flags and timeouts.
@@ -22,7 +31,7 @@ npx cross-env VISIBLE=ture npm run test-diff
 - added 3 progressive attempts for robustness
 
 ### Status
-97% examples are covered with tests. Random robusness in CI ~85%. Robustness on different machines ~97%. For example in Windows webgl_effects_ascii example always fails or on integrated GPU you will have additional artifacts: webgl_materials_texture_anisotropy, webgl_postprocessing_procedural, webgl_shaders_tonemapping.
+97% examples are covered with tests. Random robusness in CI >93%. Robustness on different machines ~97%. For example in Windows webgl_effects_ascii example always fails or on integrated GPU you will have additional artifacts: webgl_materials_texture_anisotropy, webgl_postprocessing_procedural, webgl_shaders_tonemapping.
 
 ### Probably wrong screenshots
 webgl2_multisampled_renderbuffers, webgl_simple_gi, webgl_postprocessing_dof2, webgl_loader_texture_pvrtc

test/diff/puppeteer.js

@@ -11,14 +11,8 @@ const png = require( 'pngjs' ).PNG;
 const fs = require( 'fs' );
 
 const port = 1234;
-const pixelThreshold = 0.2;
-const maxFailedPixels = 0.05;
-const networkTimeout = 600;
-const networkTax = 2000;                   // additional timeout for resources size
-const pageSizeMinTax = 1.0;                // in mb, when networkTax = 0
-const pageSizeMaxTax = 5.0;                // in mb, when networkTax = networkTax
-const renderTimeout = 1200;
-const maxAttemptId = 3;                    // progresseve attempts
+const pixelThreshold = 0.2;                // threshold error in one pixel
+const maxFailedPixels = 0.05;              // total failed pixels
 
 const exceptionList = [
 
@@ -34,6 +28,14 @@ const exceptionList = [
 
 ] : [] );
 
+const networkTimeout = 600;
+const networkTax = 2000;                   // additional timeout for resources size
+const pageSizeMinTax = 1.0;                // in mb, when networkTax = 0
+const pageSizeMaxTax = 5.0;                // in mb, when networkTax = networkTax
+const renderTimeout = 1200;
+const maxAttemptId = 3;                    // progresseve attempts
+const progressFunc = n => 1 + n;
+
 console.green = ( msg ) => console.log( `\x1b[32m${ msg }\x1b[37m` );
 console.red = ( msg ) => console.log( `\x1b[31m${ msg }\x1b[37m` );
 console.null = ( msg ) => {};
@@ -114,7 +116,7 @@ const pup = puppeteer.launch( {
 
 	/* Loop for each file, with CI parallelism */
 
-	let pageSize, file;
+	let pageSize, file, attemptProgress;
 	let failedScreenshots = 0;
 	const isParallel = 'CI' in process.env;
 	const beginId = isParallel ? Math.floor( parseInt( process.env.CI.slice( 0, 1 ) ) * files.length / 4 ) : 0;
@@ -133,6 +135,7 @@ const pup = puppeteer.launch( {
 			/* Load target page */
 
 			file = files[ id ];
+			attemptProgress = progressFunc( attemptId );
 			pageSize = 0;
 			global.gc();
 			global.gc();
@@ -141,7 +144,7 @@ const pup = puppeteer.launch( {
 
 				await page.goto( `http://localhost:${ port }/examples/${ file }.html`, {
 					waitUntil: 'networkidle2',
-					timeout: networkTimeout * ( 1 + attemptId )
+					timeout: networkTimeout * attemptProgress
 				} );
 
 			} catch {
@@ -153,7 +156,7 @@ const pup = puppeteer.launch( {
 
 			try {
 
-				await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptId ) => {
+				await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ) => {
 
 
 					/* Prepare page */
@@ -169,7 +172,12 @@ const pup = puppeteer.launch( {
 					style.type = 'text/css';
 					style.innerHTML = `body { font size: 0 !important; }
 							#info, button, input, body > div.dg.ac, body > div.lbl { display: none !important; }`;
-					document.getElementsByTagName( 'head' )[ 0 ].appendChild( style );
+					let head = document.getElementsByTagName( 'head' );
+					if ( head.length > 0 ) {
+
+						head[ 0 ].appendChild( style );
+
+					}
 
 					let canvas = document.getElementsByTagName( 'canvas' );
 					for ( let i = 0; i < canvas.length; ++ i ) {
@@ -183,7 +191,7 @@ const pup = puppeteer.launch( {
 					}
 
 					let resourcesSize = Math.min( 1, ( pageSize / 1024 / 1024 - pageSizeMinTax ) / pageSizeMaxTax );
-					await new Promise( resolve => setTimeout( resolve, networkTax * resourcesSize * ( 1 + attemptId ) ) );
+					await new Promise( resolve => setTimeout( resolve, networkTax * resourcesSize * attemptProgress ) );
 
 
 					/* Resolve render promise */
@@ -191,10 +199,13 @@ const pup = puppeteer.launch( {
 					window.chromeRenderStarted = true;
 					await new Promise( function( resolve ) {
 
+						if ( typeof performance.wow === 'undefined' ) {
+							performance.wow = performance.now;
+						}
 						let renderStart = performance.wow();
 						let waitingLoop = setInterval( function() {
 
-							let renderEcceded = ( performance.wow() - renderStart > renderTimeout * window.chromeMaxFrameId  * ( 1 + attemptId ) );
+							let renderEcceded = ( performance.wow() - renderStart > renderTimeout * window.chromeMaxFrameId  * attemptProgress );
 							if ( window.chromeRenderFinished || renderEcceded ) {
 
 								if ( renderEcceded ) {
@@ -211,7 +222,7 @@ const pup = puppeteer.launch( {
 
 					} );
 
-				}, pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptId );
+				}, pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress );
 
 			} catch ( e ) {
 
@@ -224,7 +235,7 @@ const pup = puppeteer.launch( {
 				} else {
 
 					console.log( 'Another attempt..' );
-					await new Promise( resolve => setTimeout( resolve, networkTimeout * ( 1 + attemptId ) ) );
+					await new Promise( resolve => setTimeout( resolve, networkTimeout * attemptProgress ) );
 
 				}