test(visual): store results of each two versions

test/runTest/cli.js

@@ -38,7 +38,8 @@ program
     .option('--expected <expected>', 'Expected version')
     .option('--actual <actual>', 'Actual version')
     .option('--renderer <renderer>', 'svg/canvas renderer')
-    .option('--no-save', 'Don\'t save result');
+    .option('--no-save', 'Don\'t save result')
+    .option('--dir <dir>', 'Out dir');
 
 program.parse(process.argv);
 
@@ -46,13 +47,14 @@ program.speed = +program.speed || 1;
 program.actual = program.actual || 'local';
 program.expected = program.expected || '4.2.1';
 program.renderer = (program.renderer || 'canvas').toLowerCase();
+program.dir = program.dir || (__dirname + '/tmp');
 
 if (!program.tests) {
     throw new Error('Tests are required');
 }
 
 function getScreenshotDir() {
-    return 'tmp/__screenshot__';
+    return `${program.dir}/__screenshot__`;
 }
 
 function sortScreenshots(list) {
@@ -98,9 +100,6 @@ async function convertToWebP(filePath, lossless) {
 
 async function takeScreenshot(page, fullPage, fileUrl, desc, isExpected, minor) {
     let screenshotName = testNameFromFile(fileUrl);
-    if (program.renderer === 'svg') {
-        screenshotName += '-_svg_render_';
-    }
     if (desc) {
         screenshotName += '-' + slugify(desc, { replacement: '-', lower: true });
     }
@@ -108,8 +107,8 @@ async function takeScreenshot(page, fullPage, fileUrl, desc, isExpected, minor)
         screenshotName += '-' + minor;
     }
     let screenshotPrefix = isExpected ? 'expected' : 'actual';
-    fse.ensureDirSync(path.join(__dirname, getScreenshotDir()));
-    let screenshotPath = path.join(__dirname, `${getScreenshotDir()}/${screenshotName}-${screenshotPrefix}.png`);
+    fse.ensureDirSync(getScreenshotDir());
+    let screenshotPath = path.join(getScreenshotDir(), `${screenshotName}-${screenshotPrefix}.png`);
     await page.screenshot({
         path: screenshotPath,
         fullPage
@@ -277,7 +276,7 @@ async function runTest(browser, testOpt, runtimeCode, expectedVersion, actualVer
                     actual.rawScreenshotPath
                 );
 
-                const diffPath = `${path.resolve(__dirname, getScreenshotDir())}/${shot.screenshotName}-diff.png`;
+                const diffPath = `${getScreenshotDir()}/${shot.screenshotName}-diff.png`;
                 await writePNG(diffPNG, diffPath);
                 const diffWebpPath = await convertToWebP(diffPath);
 

test/runTest/client/client.js

@@ -127,11 +127,16 @@ const app = new Vue({
 
     mounted() {
         this.fetchVersions(false);
-        this.fetchVersions(true);
-
-        setTimeout(() => {
-            this.scrollToCurrent();
-        }, 500);
+        this.fetchVersions(true).then(() => {
+            socket.emit('setTestVersions', {
+                expectedVersion: app.runConfig.expectedVersion,
+                actualVersion: app.runConfig.actualVersion,
+                renderer: app.runConfig.renderer,
+            });
+            setTimeout(() => {
+                this.scrollToCurrent();
+            }, 500);
+        })
     },
 
     computed: {
@@ -318,7 +323,7 @@ const app = new Vue({
             const url = this.runConfig[isActual ? 'isActualNightly' : 'isExpectedNightly']
                 ? 'https://data.jsdelivr.com/v1/package/npm/echarts-nightly'
                 : 'https://data.jsdelivr.com/v1/package/npm/echarts'
-            fetch(url, {
+            return fetch(url, {
                 mode: 'cors'
             }).then(res => res.json()).then(json => {
                 this[prop] = json.versions;
@@ -366,15 +371,15 @@ function runTests(tests) {
 
 socket.on('connect', () => {
     console.log('Connected');
-
-    app.$el.style.display = 'block';
 });
 
 let firstUpdate = true;
 socket.on('update', msg => {
+    app.$el.style.display = 'block';
+
     let hasFinishedTest = !!msg.tests.find(test => test.status === 'finished');
     if (!hasFinishedTest && firstUpdate) {
-        app.$confirm('It seems you haven\'t run any test yet!<br />Do you want to start now?', 'Tip', {
+        app.$confirm('You haven\'t run any test on these two versions yet!<br />Do you want to start now?', 'Tip', {
             confirmButtonText: 'Yes',
             cancelButtonText: 'No',
             dangerouslyUseHTMLString: true,
@@ -409,16 +414,6 @@ socket.on('abort', res => {
     app.running = false;
 });
 
-// function handleUrlChanged() {
-//     const params = parseParams(window.location.search.substr(1));
-//     app.currentTestName = params.test;
-//     try {
-//         Object.assign(app.runConfig, JSON.parse(params.runConfig));
-//     }
-//     catch (e) {}
-// }
-
-
 function updateUrl(notRefresh) {
     const searchUrl = assembleParams({
         test: app.currentTestName,

test/runTest/server.js

@@ -24,12 +24,14 @@ const path = require('path');
 const {fork} = require('child_process');
 const semver = require('semver');
 const {port, origin} = require('./config');
-const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults, updateActionsMeta} = require('./store');
+const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults, updateActionsMeta, getResultBaseDir} = require('./store');
 const {prepareEChartsLib, getActionsFullPath} = require('./util');
 const fse = require('fs-extra');
 const fs = require('fs');
 const open = require('open');
 
+const TEST_HASH_SPLITTER = '__';
+
 function serve() {
     const server = http.createServer((request, response) => {
         return handler(request, response, {
@@ -52,7 +54,7 @@ function serve() {
 
 let runningThreads = [];
 let pendingTests;
-let aborted = false;
+let aborted = true;
 
 function stopRunningTests() {
     if (runningThreads) {
@@ -132,7 +134,10 @@ function startTests(testsNameList, socket, {
             });
 
             if (!aborted) {
-                socket.emit('update', {tests: getTestsList(), running: true});
+                socket.emit('update', {
+                    tests: getTestsList(),
+                    running: true
+                });
             }
         }
         let runningCount = 0;
@@ -146,7 +151,10 @@ function startTests(testsNameList, socket, {
         function onUpdate() {
             // Merge tests.
             if (!aborted && !noSave) {
-                socket.emit('update', {tests: getTestsList(), running: true});
+                socket.emit('update', {
+                    tests: getTestsList(),
+                    running: true
+                });
             }
         }
         threadsCount = Math.min(threadsCount, pendingTests.length);
@@ -163,6 +171,7 @@ function startTests(testsNameList, socket, {
                 '--actual', actualVersion,
                 '--expected', expectedVersion,
                 '--renderer', renderer || '',
+                '--dir', getResultBaseDir(),
                 ...(noHeadless ? ['--no-headless'] : []),
                 ...(noSave ? ['--no-save'] : [])
             ]);
@@ -186,6 +195,14 @@ function checkPuppeteer() {
     }
 }
 
+function getTestHash(params) {
+    return [
+        params.expectedVersion,
+        params.actualVersion,
+        params.renderer
+    ].join(TEST_HASH_SPLITTER);
+}
+
 async function start() {
     if (!checkPuppeteer()) {
         // TODO Check version.
@@ -202,26 +219,27 @@ async function start() {
     let {io} = serve();
 
     io.of('/client').on('connect', async socket => {
-        await updateTestsList();
-
         function abortTests() {
+            if (aborted) {
+                return;
+            }
             stopRunningTests();
             io.of('/client').emit('abort');
             aborted = true;
         }
 
-        function emitUpdatedList() {
+        socket.on('setTestVersions', async (params) => {
+            abortTests();
+
+            await updateTestsList(
+                getTestHash(params),
+                true
+            );
+
             socket.emit('update', {
                 tests: getTestsList(),
                 running: runningThreads.length > 0
             });
-        }
-
-        emitUpdatedList();
-
-        socket.on('fetch', () => {
-            abortTests();
-            emitUpdatedList();
         });
 
         socket.on('run', async data => {

test/runTest/store.js

@@ -27,6 +27,7 @@ const {blacklist, SVGBlacklist} = require('./blacklist');
 
 let _tests = [];
 let _testsMap = {};
+let _testHash = '';
 
 class Test {
     constructor(fileUrl) {
@@ -58,8 +59,14 @@ class Test {
     }
 }
 
-function getCacheFilePath() {
-    return path.join(__dirname, 'tmp/__cache__.json');;
+function getResultBaseDir() {
+    return path.join(__dirname, 'tmp', 'result', _testHash);
+}
+
+module.exports.getResultBaseDir = getResultBaseDir;
+
+function getCacheFilePath(baseDir) {
+    return path.join(getResultBaseDir(), '__result__.json');;
 }
 
 module.exports.getTestsList = function () {
@@ -70,15 +77,21 @@ module.exports.getTestByFileUrl = function (url) {
     return _testsMap[url];
 };
 
-module.exports.updateTestsList = async function (setPendingTestToUnsettled) {
-    let tmpFolder = path.join(__dirname, 'tmp');
-    fse.ensureDirSync(tmpFolder);
+module.exports.updateTestsList = async function (
+    testHash,
+    setPendingTestToUnsettled
+) {
+    _testHash = testHash;
     _tests = [];
     _testsMap = {};
+    _testsExists = {};
+
+    fse.ensureDirSync(getResultBaseDir());
+
     try {
         let cachedStr = fs.readFileSync(getCacheFilePath(), 'utf-8');
-        _tests = JSON.parse(cachedStr);
-        _tests.forEach(test => {
+        const tests = JSON.parse(cachedStr);
+        tests.forEach(test => {
             // In somehow tests are stopped and leave the status pending.
             // Set the status to unsettled again.
             if (setPendingTestToUnsettled) {
@@ -90,7 +103,6 @@ module.exports.updateTestsList = async function (setPendingTestToUnsettled) {
         });
     }
     catch(e) {
-        _tests = [];
     }
     // Find if there is new html file
     const files = await util.promisify(glob)('**.html', { cwd: path.resolve(__dirname, '../') });
@@ -98,6 +110,8 @@ module.exports.updateTestsList = async function (setPendingTestToUnsettled) {
         if (blacklist.includes(fileUrl)) {
             return;
         }
+        _testsExists[fileUrl] = true;
+
         if (_testsMap[fileUrl]) {
             return;
         }
@@ -105,10 +119,14 @@ module.exports.updateTestsList = async function (setPendingTestToUnsettled) {
         const test = new Test(fileUrl);
         test.ignoreSVG = SVGBlacklist.includes(fileUrl);
 
-        _tests.push(test);
         _testsMap[fileUrl] = test;
     });
 
+    // Exclude tests that there is no HTML files.
+    Object.keys(_testsExists).forEach(key => {
+        _tests.push(_testsMap[key]);
+    });
+
     const actionsMetaData = {};
     const metaPath = path.join(__dirname, 'actions/__meta__.json');
     try {