Merge pull request #19075 from munrocket/e2e-cov

Tests: e2e coverage

.travis.yml

@@ -25,3 +25,12 @@ jobs:
       env: FORCE_COLOR=0 CI=2
     - <<: *e2e
       env: FORCE_COLOR=0 CI=3
+
+    - &e2e-cov
+      stage: e2e-cov
+      name: e2e-cov
+      script: npm run test-e2e-cov
+
+  allow_failures:
+
+    - stage: e2e-cov

package.json

@@ -57,6 +57,7 @@
     "test-lint-examples": "eslint examples/jsm --ext js --ext ts --ignore-pattern libs && tsc -p utils/build/tsconfig-examples.lint.json",
     "test-unit": "npm run build-test && qunit -r failonlyreporter test/unit/three.source.unit.js",
     "test-e2e": "node --expose-gc test/e2e/puppeteer.js",
+    "test-e2e-cov": "node test/e2e/check-coverage.js",
     "make-screenshot": "cross-env MAKE=true npm run test-e2e"
   },
   "keywords": [
@@ -87,7 +88,7 @@
     "image-output": "2.4.1",
     "pixelmatch": "5.1.0",
     "pngjs": "3.4.0",
-    "puppeteer": "^2.1.1",
+    "puppeteer": "2.1.1",
     "qunit": "^2.9.3",
     "rollup": "^2.3.2",
     "rollup-plugin-buble": "^0.19.8",

test/e2e/README.md

 # Three.js end-to-end testing
 
-You probably shouldn't run this tests on PC because it's not optimized for local usage and
-you can get different results on different GPUs. Goal is to make quick automated testing
-inside CI and keep screenshot pack updated for it.
+### Motivation
+Simplify code reviews with quick automated testing inside CI.
 
 ### Local usage
+If you get an error in e2e test after PR in repo and you sure that all is right, just make a new screenshot.
+Sometime you can get wrong results on different GPUs, especially on macbook's. Keep screenshots pack updated for tests.
+
 ```shell
-# generate new screenshots
+# generate new screenshots for exact examples
 npm run make-screenshot <example1_name> ... <exampleN_name>
 
-# check examples
+# check exact examples
 npm run test-e2e <example1_name> ... <exampleN_name>
 
+# check all examples
+npm run test-e2e
+
 # check all examples in browser
 npx cross-env VISIBLE=ture npm run test-e2e
 ```
@@ -34,7 +39,7 @@ npx cross-env VISIBLE=ture npm run test-e2e
 | 4=0+0+2+2 failed, time=3:26             | with progressive attempts            |
 
 ### Status
-97% examples are covered with tests. Random robusness in CI >93%. Robustness on different machines ~97%.
+95% examples are covered with tests. Random robusness in CI >85%. Robustness on different machines ~97%.
 For example on integrated GPU you can have additional artifacts: webgl_materials_texture_anisotropy,
 webgl_postprocessing_procedural, webgl_shaders_tonemapping.
 
@@ -42,6 +47,4 @@ webgl_postprocessing_procedural, webgl_shaders_tonemapping.
 webgl_loader_bvh, webgl_simple_gi, webgl_postprocessing_dof2, webgl_loader_texture_pvrtc, webgl_physics_volume
 
 ### Contribution
-Proper determinism for video/audio is welcome. You can cover more examples: `dof2` can be rendered with 2 rAF,
-`offsceencanvas`/`webgl_test_memory2` with additional puppeteer flags, `webgl_simple_gi` with
-[beginFrame](https://chromedevtools.github.io/devtools-protocol/tot/HeadlessExperimental) CDP API.
+You can help with MacOS support.

test/e2e/check-coverage.js

+/**
+ * @author munrocket / https://github.com/munrocket
+ */
+
+const fs = require( 'fs' );
+
+// examples
+const E = fs.readdirSync( './examples' )
+		.filter( s => s.slice( - 5 ) === '.html' )
+		.map( s => s.slice( 0, s.length - 5 ) )
+		.filter( f => f !== 'index' );
+
+// screenshots
+const S = fs.readdirSync( './examples/screenshots' )
+		.filter( s => s.slice( - 4 ) === '.png' )
+		.map( s => s.slice( 0, s.length - 4 ) )
+
+// files.js
+const F = [];
+eval( fs.readFileSync( './examples/files.js' ).toString() );
+for ( var key in files ) {
+
+	var section = files[ key ];
+	for ( var i = 0, len = section.length; i < len; i ++ ) {
+
+		F.push( section[ i ] );
+
+	}
+
+}
+
+let subES = E.filter( x => ! S.includes( x ) );
+let subSE = S.filter( x => ! E.includes( x ) );
+let subEF = E.filter( x => ! F.includes( x ) );
+let subFE = F.filter( x => ! E.includes( x ) );
+
+console.green = ( msg ) => console.log( `\x1b[32m${ msg }\x1b[37m` );
+console.red = ( msg ) => console.log( `\x1b[31m${ msg }\x1b[37m` );
+
+if ( subES.length + subSE.length + subEF.length + subFE.length === 0 ) {
+
+	console.green( 'TEST PASSED! All examples is covered with screenshots and descriptions in files.js!' );
+
+} else {
+
+	if ( subES.length > 0 ) console.red( 'Add screenshots for example(s): ' + subES.join(' ') );
+	if ( subSE.length > 0 ) console.red( 'Remove unnecessary screenshot(s): ' + subSE.join(' ') );
+	if ( subEF.length > 0 ) console.red( 'Add description in file.js for example(s): ' + subEF.join(' ') );
+	if ( subFE.length > 0 ) console.red( 'Remove description in file.js for example(s): ' + subFE.join(' ') );
+
+	console.red( 'TEST FAILED!' );
+
+	process.exit( 1 );
+
+}

test/e2e/puppeteer.js

@@ -47,28 +47,8 @@ console.null = () => {};
 
 /* Launch server */
 
-const server = http.createServer( ( request, response ) => {
-
-	return handler( request, response );
-
-} );
-server.listen( port, async () => {
-
-	try {
-
-		await pup;
-
-	} catch ( e ) {
-
-		console.error( e );
-
-	} finally {
-
-		server.close();
-
-	}
-
-} );
+const server = http.createServer( ( req, resp ) => handler( req, resp ) );
+server.listen( port, async () => await pup );
 server.on( 'SIGINT', () => process.exit( 1 ) );
 
 
@@ -122,7 +102,7 @@ const pup = puppeteer.launch( {
 	/* Loop for each file, with CI parallelism */
 
 	let pageSize, file, attemptProgress;
-	let failedScreenshots = 0;
+	let failedScreenshots = [];
 	const isParallel = 'CI' in process.env;
 	const beginId = isParallel ? Math.floor( parseInt( process.env.CI.slice( 0, 1 ) ) * files.length / 4 ) : 0;
 	const endId = isParallel ? Math.floor( ( parseInt( process.env.CI.slice( - 1 ) ) + 1 ) * files.length / 4 ) : files.length;
@@ -212,7 +192,7 @@ const pup = puppeteer.launch( {
 				if ( ++ attemptId === maxAttemptId ) {
 
 					console.red( `WTF? 'Network timeout' is small for your machine. file: ${ file } \n${ e }` );
-					++ failedScreenshots;
+					failedScreenshots.push( file );
 					continue;
 
 				} else {
@@ -234,6 +214,7 @@ const pup = puppeteer.launch( {
 
 				attemptId = maxAttemptId;
 				await page.screenshot( { path: `./examples/screenshots/${ file }.png` } );
+				printImage( png.sync.read( fs.readFileSync( `./examples/screenshots/${ file }.png` ) ), console );
 				console.green( `file: ${ file } generated` );
 
 
@@ -260,7 +241,7 @@ const pup = puppeteer.launch( {
 
 					attemptId = maxAttemptId;
 					console.red( `ERROR! Image sizes does not match in file: ${ file }` );
-					++ failedScreenshots;
+					failedScreenshots.push( file );
 					continue;
 
 				}
@@ -280,7 +261,7 @@ const pup = puppeteer.launch( {
 
 						printImage( diff, console );
 						console.red( `ERROR! Diff wrong in ${ numFailedPixels.toFixed( 3 ) } of pixels in file: ${ file }` );
-						++ failedScreenshots;
+						failedScreenshots.push( file );
 						continue;
 
 					} else {
@@ -294,8 +275,7 @@ const pup = puppeteer.launch( {
 			} else {
 
 				attemptId = maxAttemptId;
-				console.red( `ERROR! Screenshot not exists: ${ file }` );
-				++ failedScreenshots;
+				console.null( `Warning! Screenshot not exists: ${ file }` );
 				continue;
 
 			}
@@ -307,10 +287,19 @@ const pup = puppeteer.launch( {
 
 	/* Finish */
 
-	if ( failedScreenshots ) {
+	if ( failedScreenshots.length ) {
+
+		if ( failedScreenshots.length > 1 ) {
+
+			console.red( 'List of failed screenshots: ' + failedScreenshots.join(' ') );
+
+		} else {
+
+			console.red( `If you sure that all is right, try to run \`npm run make-screenshot ${ failedScreenshots[ 0 ] }\`` );
+
+		}
 
-		console.red( `TEST FAILED! ${ failedScreenshots } from ${ endId - beginId } screenshots not pass.` );
-		process.exit( 1 );
+		console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ endId - beginId } screenshots not pass.` );
 
 	} else if ( ! process.env.MAKE ) {
 
@@ -318,6 +307,8 @@ const pup = puppeteer.launch( {
 
 	}
 
-	await browser.close();
+	browser.close();
+	server.close();
+	process.exit( failedScreenshots.length );
 
 } );