test: add version selection in visual regression testing tool

test/runTest/blacklist.js

@@ -29,5 +29,10 @@ module.exports = [
 
     'finished-gl.html',
     'scatter-gps.html',
-    'webkit-dep.html'
+    'webkit-dep.html',
+
+    // This case will have timeout
+    'visualMap-performance1.html',
+    'lines-bus.html',
+    'lines-stream-not-large.html'
 ];

test/runTest/cli.js

@@ -32,11 +32,15 @@ const Timeline = require('./Timeline');
 program
     .option('-t, --tests <tests>', 'Tests names list')
     .option('--no-headless', 'Not headless')
-    .option('-s, --speed <speed>', 'Playback speed');
+    .option('-s, --speed <speed>', 'Playback speed')
+    .option('--expected <expected>', 'Expected version')
+    .option('--actual <actual>', 'Actual version');
 
 program.parse(process.argv);
 
 program.speed = +program.speed || 1;
+program.actual = program.actual || 'local';
+program.expected = program.expected || '4.2.1';
 
 if (!program.tests) {
     throw new Error('Tests are required');
@@ -144,6 +148,9 @@ async function runTestPage(browser, testOpt, version, runtimeCode) {
     page.on('pageerror', error => {
         errors.push(error.toString());
     });
+    page.on('dialog', async dialog => {
+        await dialog.dismiss();
+    });
 
     try {
         await page.setViewport({width: 800, height: 600});
@@ -151,19 +158,20 @@ async function runTestPage(browser, testOpt, version, runtimeCode) {
             waitUntil: 'networkidle2',
             timeout: 10000
         });
+
+        await waitTime(200);  // Wait for animation or something else. Pending
+        // Final shot.
+        await page.mouse.move(0, 0);
+        let desc = 'Full Shot';
+        const {screenshotName, screenshotPath} = await takeScreenshot(page, true, fileUrl, desc, version);
+        screenshots.push({screenshotName, desc, screenshotPath});
+
+        await runActions(page, testOpt, version, screenshots);
     }
     catch(e) {
         console.error(e);
     }
 
-    await waitTime(200);  // Wait for animation or something else. Pending
-    // Final shot.
-    let desc = 'Full Shot';
-    const {screenshotName, screenshotPath} = await takeScreenshot(page, true, fileUrl, desc, version);
-    screenshots.push({screenshotName, desc, screenshotPath});
-
-    await runActions(page, testOpt, version, screenshots);
-
     await page.close();
 
     return {
@@ -181,10 +189,10 @@ async function writePNG(diffPNG, diffPath) {
     });
 };
 
-async function runTest(browser, testOpt, runtimeCode) {
+async function runTest(browser, testOpt, runtimeCode, expectedVersion, actualVersion) {
     testOpt.status === 'running';
-    const expectedResult = await runTestPage(browser, testOpt, '4.2.1', runtimeCode);
-    const actualResult = await runTestPage(browser, testOpt, '', runtimeCode);
+    const expectedResult = await runTestPage(browser, testOpt, expectedVersion, runtimeCode);
+    const actualResult = await runTestPage(browser, testOpt, actualVersion, runtimeCode);
 
     // sortScreenshots(expectedResult.screenshots);
     // sortScreenshots(actualResult.screenshots);
@@ -218,6 +226,8 @@ async function runTest(browser, testOpt, runtimeCode) {
     testOpt.expectedLogs = expectedResult.logs;
     testOpt.actualErrors = actualResult.errors;
     testOpt.expectedErrors = expectedResult.errors;
+    testOpt.actualVersion = actualVersion;
+    testOpt.expectedVersion = expectedVersion;
     testOpt.lastRun = Date.now();
 }
 
@@ -235,11 +245,11 @@ async function runTests(pendingTests) {
         for (let testOpt of pendingTests) {
             console.log('Running Test', testOpt.name);
             try {
-                await runTest(browser, testOpt, runtimeCode);
+                await runTest(browser, testOpt, runtimeCode, program.expected, program.actual);
             }
             catch (e) {
                 // Restore status
-                testOpt.status = 'pending';
+                testOpt.status = 'unsettled';
                 console.log(e);
             }
 

test/runTest/client/client.css

@@ -79,6 +79,11 @@
     margin-left: 5px;
     cursor: pointer;
 }
+.nav-toolbar .el-button {
+    padding-left: 8px;
+    padding-right: 8px;
+}
+
 .run-config-item {
     margin: 5px 0;
 }

test/runTest/client/client.js

@@ -67,9 +67,13 @@ const app = new Vue({
         allSelected: false,
         lastSelectedIndex: -1,
 
+        versions: [],
+
         runConfig: {
             noHeadless: false,
             replaySpeed: 5,
+            actualVersion: 'local',
+            expectedVersion: null,
             threads: 1
         }
     },
@@ -162,15 +166,20 @@ const app = new Vue({
                 this.tests[i].selected = selected;
             }
         },
-        refreshList() {
-
-        },
-        runTest(testName) {
+        runSingleTest(testName) {
             runTests([testName]);
         },
-        runSelectedTests() {
+        run(runTarget) {
             const tests = this.fullTests.filter(test => {
-                return test.selected;
+                if (runTarget === 'selected') {
+                    return test.selected;
+                }
+                else if (runTarget === 'unfinished') {
+                    return test.status !== 'finished';
+                }
+                else {  // Run all
+                    return true;
+                }
             }).map(test => {
                 return test.name;
             });
@@ -184,23 +193,31 @@ const app = new Vue({
 });
 
 function runTests(tests) {
-    if (tests.length > 0) {
-        app.running = true;
-        socket.emit('run', {
-            tests,
-            threads: app.runConfig.threads,
-            noHeadless: app.runConfig.noHeadless,
-            replaySpeed: app.runConfig.noHeadless
-                ? app.runConfig.replaySpeed
-                : 5 // Force run at 5x speed
+    if (!tests.length) {
+        app.$notify({
+            title: 'No test selected.',
+            position: 'top-right'
         });
+        return;
     }
-    else {
+    if (!app.runConfig.expectedVersion || !app.runConfig.actualVersion) {
         app.$notify({
-            title: 'No test selected.',
+            title: 'No echarts version selected.',
             position: 'top-right'
         });
+        return;
     }
+    app.running = true;
+    socket.emit('run', {
+        tests,
+        expectedVersion: app.runConfig.expectedVersion,
+        actualVersion: app.runConfig.actualVersion,
+        threads: app.runConfig.threads,
+        noHeadless: app.runConfig.noHeadless,
+        replaySpeed: app.runConfig.noHeadless
+            ? app.runConfig.replaySpeed
+            : 5 // Force run at 5x speed
+    });
 }
 
 
@@ -240,6 +257,17 @@ socket.on('finish', res => {
     console.log(`${res.count} test complete, Cost: ${(res.time / 1000).toFixed(1)} s. Threads: ${res.threads}`);
     app.running = false;
 });
+socket.on('versions', versions => {
+    app.versions = versions.filter(version => {
+        return !version.startsWith('2.')
+            && !version.match('beta')
+            && !version.match('rc');
+    }).reverse();
+    if (!app.runConfig.expectedVersion) {
+        app.runConfig.expectedVersion = app.versions[0];
+    }
+    app.versions.unshift('local');
+});
 
 function updateTestHash() {
     app.currentTestName = window.location.hash.slice(1);

test/runTest/client/index.html

@@ -40,21 +40,26 @@ under the License.
                     <el-input v-model="searchString" size="mini" placeholder="Filter Tests"></el-input>
                     <div class="controls">
                         <el-checkbox :indeterminate="isSelectAllIndeterminate" v-model="allSelected" @change="handleSelectAllChange"></el-checkbox>
-                        <el-button
-                            title="Sort By Failue Percentage" @click="toggleSort" circle size="mini" type="primary" icon="el-icon-sort"
-                        ></el-button>
+                        <el-button title="Sort By Failue Percentage" @click="toggleSort" size="mini" type="primary" icon="el-icon-sort">Sort</el-button>
 
-                        <el-button-group style="margin-left: 10px">
-                            <el-button title="Run Selected" @click="runSelectedTests" :loading="running" circle size="mini" type="primary" icon="el-icon-caret-right"></el-button>
-                            <el-button v-if="running" title="Run Selected" @click="stopTests" circle size="mini" type="primary" icon="el-icon-close"></el-button>
+                        <el-dropdown v-if="!running" split-button type="primary" size="mini" title="Run"
+                            @click="run('selected')"
+                            @command="run"
+                        >
+                            <i class="el-icon-caret-right"></i> Run selected
+                            <el-dropdown-menu slot="dropdown" >
+                                <el-dropdown-item command="unfinished">Run unfinished</el-dropdown-item>
+                                <el-dropdown-item command="all">Run all</el-dropdown-item>
+                            </el-dropdown-menu>
+                        </el-dropdown>
+
+                        <el-button-group v-else>
+                            <el-button type="primary" size="mini" :loading="true">Stop</el-button>
+                            <el-button title="Run Selected" @click="stopTests" size="mini" type="primary" icon="el-icon-close" style="padding-left: 3px;padding-right:3px;"></el-button>
                         </el-button-group>
 
 
                         <el-popover title="Configuration" class="run-configuration">
-                            <div class="run-config-item">
-                                <span>Threads</span>
-                                <el-slider style="width: 140px;" v-model="runConfig.threads" :step="1" :min="1" :max="8" show-stops></el-slider>
-                            </div>
                             <div class="run-config-item">
                                 <el-checkbox v-model="runConfig.noHeadless">Replay</el-checkbox>
                                 <el-slider
@@ -66,15 +71,28 @@ under the License.
                                     :disabled="!runConfig.noHeadless"
                                 ></el-slider>
                             </div>
+                            <div class="run-config-item">
+                                <span>Threads</span>
+                                <el-slider style="width: 140px;" v-model="runConfig.threads" :step="1" :min="1" :max="8" show-stops></el-slider>
+                            </div>
+                            <div class="run-config-item">
+                                <span>Version </span>
+                                <span style="font-size: 12px; color:#afafaf">Expected</span>
+                                <el-select size="mini" v-model="runConfig.expectedVersion" placeholder="Select Version"
+                                    style="width: 80px;"
+                                >
+                                    <el-option v-for="version in versions" :key="version" :label="version" :value="version"></el-option>
+                                </el-select>
+                                <span style="font-size: 12px; color: #afafaf">Actual</span>
+                                <el-select size="mini" v-model="runConfig.actualVersion" placeholder="Select Version"
+                                    style="width: 80px;"
+                                >
+                                    <el-option v-for="version in versions" :key="version" :label="version" :value="version"></el-option>
+                                </el-select>
+                            </div>
                             <i slot="reference" class="el-icon-setting"></i>
                         </el-popover>
 
-
-                        <!-- <el-button-group>
-                            <el-button title="Select All" @click="selectAllTests" circle size="mini" type="primary" icon="el-icon-check"></el-button>
-                            <el-button title="Unselect All" @click="unselectAllTests" circle size="mini" type="primary" icon="el-icon-close"></el-button>
-                        </el-button-group> -->
-                        <!-- <el-button title="Refresh List" @click="refreshList" circle size="mini" type="primary" icon="el-icon-refresh"></el-button> -->
                     </div>
                 </div>
                 <ul class="test-list">
@@ -130,7 +148,7 @@ under the License.
                         ></el-progress>
                         <h3>{{currentTest.name}}</h3>
                         <el-button-group style="margin-left: 10px">
-                            <el-button title="Run Selected" @click="runTest(currentTest.name)" :loading="running" circle type="primary" icon="el-icon-caret-right"></el-button>
+                            <el-button title="Run Selected" @click="runSingleTest(currentTest.name)" :loading="running" circle type="primary" icon="el-icon-caret-right"></el-button>
                             <el-button v-if="running" title="Run Selected" @click="stopTests" circle type="primary" icon="el-icon-close"></el-button>
                         </el-button-group>
                         <a target="_blank" :href="currentTestUrl"><i class="el-icon-link"></i>Open Demo</a>
@@ -146,7 +164,7 @@ under the License.
                             <el-col :span="8">
                                 <el-card shadow="hover">
                                     <div slot="header" class="clearfix">
-                                        <span>Expected</span>
+                                        <span>Expected - {{currentTest.expectedVersion || ''}}</span>
                                     </div>
                                     <el-image :src="result.expected"></el-image>
                                 </el-card>
@@ -155,7 +173,7 @@ under the License.
                             <el-col :span="8">
                                 <el-card shadow="hover">
                                     <div slot="header" class="clearfix">
-                                        <span>Actual</span>
+                                        <span>Actual - {{currentTest.actualVersion || ''}}</span>
                                     </div>
                                     <el-image :src="result.actual"></el-image>
                                 </el-card>
@@ -164,7 +182,7 @@ under the License.
                             <el-col :span="8">
                                 <el-card shadow="hover">
                                     <div slot="header" class="clearfix">
-                                        <span>Diff({{result.diffRatio.toFixed(4)}})</span>
+                                        <span>Diff - {{result.diffRatio.toFixed(4)}}</span>
                                     </div>
                                     <el-image :src="result.diff" :preview-src-list="[result.diff]"></el-image>
                                 </el-card>

test/runTest/server.js

@@ -25,7 +25,7 @@ const {fork} = require('child_process');
 const semver = require('semver');
 const {port, origin} = require('./config');
 const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults, updateActionsMeta} = require('./store');
-const {prepareEChartsVersion, getActionsFullPath} = require('./util');
+const {prepareEChartsLib, getActionsFullPath, fetchVersions} = require('./util');
 const fse = require('fs-extra');
 const fs = require('fs');
 
@@ -75,12 +75,16 @@ class Thread {
         this.onUpdate;
     }
 
-    fork(noHeadless, replaySpeed) {
+    fork(noHeadless, replaySpeed, actualVersion, expectedVersion) {
         let p = fork(path.join(__dirname, 'cli.js'), [
             '--tests',
             this.tests.map(testOpt => testOpt.name).join(','),
             '--speed',
             replaySpeed || 5,
+            '--actual',
+            actualVersion,
+            '--expected',
+            expectedVersion,
             ...(noHeadless ? ['--no-headless'] : [])
         ]);
         this.p = p;
@@ -108,7 +112,9 @@ class Thread {
 function startTests(testsNameList, socket, {
     noHeadless,
     threadsCount,
-    replaySpeed
+    replaySpeed,
+    actualVersion,
+    expectedVersion
 }) {
     console.log('Received: ', testsNameList.join(','));
 
@@ -147,7 +153,7 @@ function startTests(testsNameList, socket, {
         for (let i = 0; i < threadsCount; i++) {
             runningThreads[i].onExit = onExit;
             runningThreads[i].onUpdate = onUpdate;
-            runningThreads[i].fork(noHeadless, replaySpeed);
+            runningThreads[i].fork(noHeadless, replaySpeed, actualVersion, expectedVersion);
             runningCount++;
         }
         // If something bad happens and no proccess are started successfully
@@ -174,8 +180,7 @@ async function start() {
         return;
     }
 
-    await prepareEChartsVersion('4.2.1'); // Expected version.
-    await prepareEChartsVersion(); // Version to test
+    let versions = await fetchVersions();
 
     // let runtimeCode = await buildRuntimeCode();
     // fse.outputFileSync(path.join(__dirname, 'tmp/testRuntime.js'), runtimeCode, 'utf-8');
@@ -189,7 +194,12 @@ async function start() {
         socket.emit('update', {tests: getTestsList()});
 
         socket.on('run', async data => {
+
             let startTime = Date.now();
+
+            await prepareEChartsLib(data.expectedVersion); // Expected version.
+            await prepareEChartsLib(data.actualVersion); // Version to test
+
             // TODO Should broadcast to all sockets.
             try {
                 await startTests(
@@ -198,7 +208,9 @@ async function start() {
                     {
                         noHeadless: data.noHeadless,
                         threadsCount: data.threads,
-                        replaySpeed: data.replaySpeed
+                        replaySpeed: data.replaySpeed,
+                        actualVersion: data.actualVersion,
+                        expectedVersion: data.expectedVersion
                     }
                 );
             }
@@ -213,6 +225,8 @@ async function start() {
         socket.on('stop', () => {
             stopRunningTests();
         });
+
+        socket.emit('versions', versions);
     });
 
     io.of('/recorder').on('connect', async socket => {
@@ -245,7 +259,9 @@ async function start() {
                 await startTests([data.testName], socket, {
                     noHeadless: true,
                     threadsCount: 1,
-                    replaySpeed: 2
+                    replaySpeed: 2,
+                    actualVersion: data.actualVersion,
+                    expectedVersion: data.expectedVersion
                 });
             }
             catch (e) { console.error(e); }
@@ -264,7 +280,7 @@ async function start() {
     });
 
     console.log(`Dashboard: ${origin}/test/runTest/client/index.html`);
-    console.log(`Interaction Recorder: ${origin}/test/runTest/recorder/index.html`);
+    // console.log(`Interaction Recorder: ${origin}/test/runTest/recorder/index.html`);
     // open(`${origin}/test/runTest/client/index.html`);
 
 }

test/runTest/util.js

@@ -34,7 +34,7 @@ module.exports.fileNameFromTest = function (testName) {
 };
 
 function getVersionDir(version) {
-    version = version || 'developing';
+    version = version || 'local';
     return `tmp/__version__/${version}`;
 };
 module.exports.getVersionDir = getVersionDir;
@@ -44,10 +44,10 @@ module.exports.getActionsFullPath = function (testName) {
 };
 
 
-module.exports.prepareEChartsVersion = function (version) {
+module.exports.prepareEChartsLib = function (version) {
     let versionFolder = path.join(__dirname, getVersionDir(version));
     fse.ensureDirSync(versionFolder);
-    if (!version) {
+    if (!version || version === 'local') {
         // Developing version, make sure it's new build
         return fse.copy(
             path.join(__dirname, '../../dist/echarts.js'),
@@ -58,7 +58,7 @@ module.exports.prepareEChartsVersion = function (version) {
         if (!fs.existsSync(`${versionFolder}/echarts.js`)) {
             const file = fs.createWriteStream(`${versionFolder}/echarts.js`);
 
-            console.log('Downloading echarts4.2.1 from ', `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`);
+            console.log(`Downloading echarts@${version} from `, `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`);
             https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => {
                 response.pipe(file);
 
@@ -73,6 +73,29 @@ module.exports.prepareEChartsVersion = function (version) {
     });
 };
 
+module.exports.fetchVersions = function () {
+    return new Promise((resolve, reject) => {
+        https.get(`https://registry.npmjs.org/echarts`, res => {
+            if (res.statusCode !== 200) {
+                res.destroy();
+                reject('Failed fetch versions from https://registry.npmjs.org/echarts');
+                return;
+            }
+            var buffers = [];
+            res.on('data', buffers.push.bind(buffers));
+            res.on('end', function () {
+                try {
+                    var data = Buffer.concat(buffers);
+                    resolve(Object.keys(JSON.parse(data).versions));
+                }
+                catch (e) {
+                    reject(e.toString());
+                }
+            });
+        });
+    });
+};
+
 module.exports.buildRuntimeCode = async function () {
     const bundle = await rollup.rollup({
         input: path.join(__dirname, 'runtime/main.js'),