test: Use child process to run test. Add logs in dashboard

package.json

@@ -14,7 +14,7 @@
   },
   "scripts": {
     "prepublish": "node build/build.js --prepublish",
-    "test:visual": "node build/build.js && node test/runTest/cli.js",
+    "test:visual": "node test/runTest/server.js",
     "test": "node build/build.js"
   },
   "dependencies": {
@@ -34,15 +34,18 @@
     "estraverse": "4.1.1",
     "fs-extra": "^0.26.7",
     "glob": "7.0.0",
+    "lolex": "^4.2.0",
     "open": "^6.4.0",
     "pixelmatch": "^5.0.2",
     "pngjs": "^3.4.0",
     "puppeteer": "^1.19.0",
     "rollup": "0.50.0",
+    "rollup-plugin-commonjs": "^8.4.1",
     "rollup-plugin-node-resolve": "3.0.0",
     "rollup-plugin-uglify": "2.0.1",
     "seedrandom": "^3.0.3",
     "serve-handler": "^6.1.1",
-    "slugify": "^1.3.4"
+    "slugify": "^1.3.4",
+    "socket.io": "^2.2.0"
   }
 }

test/runTest/cli.js

@@ -2,38 +2,15 @@ const puppeteer = require('puppeteer');
 const slugify = require('slugify');
 const fse = require('fs-extra');
 const fs = require('fs');
-const https = require('https');
 const path = require('path');
-const open = require('open');
-const util = require('util');
-const glob = require('glob');
-const {serve, origin} = require('./serve');
 const compareScreenshot = require('./compareScreenshot');
-const blacklist = require('./blacklist');
-
-const seedrandomCode = fs.readFileSync(
-    path.join(__dirname, '../../node_modules/seedrandom/seedrandom.js'),
-    'utf-8'
-);
-const runtimeCode = fs.readFileSync(path.join(__dirname, './runtime.js'), 'utf-8');
-
-function getVersionDir(version) {
-    version = version || 'developing';
-    return `tmp/__version__/${version}`;
-}
+const {getTestName, getVersionDir} = require('./util');
+const {origin} = require('./config');
 
 function getScreenshotDir() {
     return 'tmp/__screenshot__';
 }
 
-function getTestName(fileUrl) {
-    return path.basename(fileUrl, '.html');
-}
-
-function getCacheFilePath() {
-    return path.join(__dirname, 'tmp/__cache__.json');
-}
-
 function sortScreenshots(list) {
     return list.sort((a, b) => {
         return a.testName.localeCompare(b.testName);
@@ -56,35 +33,6 @@ function replaceEChartsVersion(interceptedRequest, version) {
     }
 }
 
-function prepareEChartsVersion(version) {
-    let versionFolder = path.join(__dirname, getVersionDir(version));
-    fse.ensureDirSync(versionFolder);
-    if (!version) {
-        // Developing version, make sure it's new build
-        return fse.copy(
-            path.join(__dirname, '../../dist/echarts.js'),
-            `${versionFolder}/echarts.js`
-        );
-    }
-    return new Promise(resolve => {
-        if (!fs.existsSync(`${versionFolder}/echarts.js`)) {
-            const file = fs.createWriteStream(`${versionFolder}/echarts.js`);
-
-            console.log('Downloading echarts4.2.1 from ', `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`);
-            https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => {
-                response.pipe(file);
-
-                file.on('finish', () => {
-                    resolve();
-                })
-            });
-        }
-        else {
-            resolve();
-        }
-    });
-}
-
 function waitPageForFinish(page) {
     return new Promise(resolve => {
         page.exposeFunction('puppeteerFinishTest', () => {
@@ -130,6 +78,7 @@ async function takeScreenshot(page, elementQuery, fileUrl, desc, version) {
         testName += '-' + slugify(desc, { replacement: '-', lower: true })
     }
     let screenshotPrefix = version ? 'expected' : 'actual';
+    fse.ensureDirSync(path.join(__dirname, getScreenshotDir()));
     let screenshotPath = path.join(__dirname, `${getScreenshotDir()}/${testName}-${screenshotPrefix}.png`);
     await target.screenshot({
         path: screenshotPath,
@@ -139,15 +88,16 @@ async function takeScreenshot(page, elementQuery, fileUrl, desc, version) {
     return {testName, screenshotPath};
 }
 
-async function runTestPage(browser, fileUrl, version) {
-    const {keepWait, waitTimeout} = createWaitTimeout(3200);
-    const testResults = [];
+async function runTestPage(browser, fileUrl, version, runtimeCode) {
+    const {keepWait, waitTimeout} = createWaitTimeout(1500);
+    const screenshots = [];
+    const logs = [];
+    const errors = [];
     let screenshotPromises = [];
 
     const page = await browser.newPage();
     page.setRequestInterception(true);
     page.on('request', replaceEChartsVersion);
-    await page.evaluateOnNewDocument(seedrandomCode);
     await page.evaluateOnNewDocument(runtimeCode);
 
     let descAutoCounter = 0;
@@ -160,24 +110,25 @@ async function runTestPage(browser, fileUrl, version) {
                 return;
             }
             const {testName, screenshotPath} = result;
-            testResults.push({testName, desc, screenshotPath});
+            screenshots.push({testName, desc, screenshotPath});
         });
         screenshotPromises.push(promise);
 
         return promise;
     });
-    // page.on('console', msg => {
-    //     console.log(msg.text());
-    // });
-    // page.on('pageerror', error => {
-    //     console.error(error);
-    // })
+    page.on('console', msg => {
+        logs.push(msg.text());
+    });
+    page.on('pageerror', error => {
+        errors.push(error);
+    });
 
     let pageFinishPromise = waitPageForFinish(page);
 
     try {
         await page.goto(`${origin}/test/${fileUrl}`, {
             waitUntil: 'networkidle2',
+            // waitUntil: 'domcontentloaded',
             timeout: 10000
         });
     }
@@ -186,16 +137,17 @@ async function runTestPage(browser, fileUrl, version) {
         console.error(e);
     }
 
-    // Do auto screenshot for every 1 second.
-    let count = 1;
-    let autoSnapshotInterval = setInterval(async () => {
-        let desc = `autogen-${count++}`;
-        let promise = takeScreenshot(page, '', fileUrl, desc, version)
-            .then(({testName, screenshotPath}) => {
-                testResults.push({testName, desc, screenshotPath});
-            });
-        screenshotPromises.push(promise);
-    }, 1000);
+    // TODO Animation
+
+    // Do auto screenshot after 100ms for animation.
+    // let autoSnapshotTimeout = setTimeout(async () => {
+    //     let desc = `Animation Interval`;
+    //     let promise = takeScreenshot(page, '', fileUrl, desc, version)
+    //         .then(({testName, screenshotPath}) => {
+    //             screenshots.push({testName, desc, screenshotPath});
+    //         });
+    //     screenshotPromises.push(promise);
+    // }, 100);
 
 
     // Wait for puppeteerFinishTest() is called
@@ -204,33 +156,43 @@ async function runTestPage(browser, fileUrl, version) {
         pageFinishPromise,
         waitTimeout().then(() => {
             // console.warn('Test timeout after 3 seconds.');
+            // Final shot.
+            let desc = 'Final Shot';
+            return takeScreenshot(page, '', fileUrl, desc, version)
+                .then(({testName, screenshotPath}) => {
+                    screenshots.push({testName, desc, screenshotPath});
+                });
         })
     ]);
 
-    clearInterval(autoSnapshotInterval);
+    // clearTimeout(autoSnapshotTimeout);
 
     // Wait for screenshot finished.
     await Promise.all(screenshotPromises);
 
     await page.close();
 
-    return testResults;
+    return {
+        logs,
+        errors,
+        screenshots: screenshots
+    };
 }
 
-async function runTest(browser, testOpt) {
+async function runTest(browser, testOpt, runtimeCode) {
     testOpt.status === 'running';
     const fileUrl = testOpt.fileUrl;
-    const expectedShots = await runTestPage(browser, fileUrl, '4.2.1');
-    const actualShots = await runTestPage(browser, fileUrl);
+    const expectedResult = await runTestPage(browser, fileUrl, '4.2.1', runtimeCode);
+    const actualResult = await runTestPage(browser, fileUrl, '', runtimeCode);
 
-    sortScreenshots(expectedShots);
-    sortScreenshots(actualShots);
+    sortScreenshots(expectedResult.screenshots);
+    sortScreenshots(actualResult.screenshots);
 
-    const results = [];
+    const screenshots = [];
     let idx = 0;
-    for (let shot of expectedShots) {
+    for (let shot of expectedResult.screenshots) {
         let expected = shot;
-        let actual = actualShots[idx++];
+        let actual = actualResult.screenshots[idx++];
         let {diffRatio, diffPNG} = await compareScreenshot(
             expected.screenshotPath,
             actual.screenshotPath
@@ -239,7 +201,7 @@ async function runTest(browser, testOpt) {
         let diffPath = `${path.resolve(__dirname, getScreenshotDir())}/${shot.testName}-diff.png`;
         diffPNG.pack().pipe(fs.createWriteStream(diffPath));
 
-        results.push({
+        screenshots.push({
             actual: getClientRelativePath(actual.screenshotPath),
             expected: getClientRelativePath(expected.screenshotPath),
             diff: getClientRelativePath(diffPath),
@@ -249,88 +211,47 @@ async function runTest(browser, testOpt) {
         });
     }
 
-    testOpt.results = results;
+    testOpt.results = screenshots;
     testOpt.status = 'finished';
-}
+    testOpt.actualLogs = actualResult.logs;
+    testOpt.expectedLogs = expectedResult.logs;
+    testOpt.actualErrors = actualResult.errors;
+    testOpt.expectedErrors = expectedResult.errors;
 
-function writeTestsToCache(tests) {
-    fs.writeFileSync(getCacheFilePath(), JSON.stringify(tests, null, 2), 'utf-8');
 }
 
-async function getTestsList() {
-    let tmpFolder = path.join(__dirname, 'tmp');
-    fse.ensureDirSync(tmpFolder);
-    try {
-        let cachedStr = fs.readFileSync(getCacheFilePath(), 'utf-8');
-        let tests = JSON.parse(cachedStr);
-        return tests;
-    }
-    catch(e) {
-        let files = await util.promisify(glob)('**.html', { cwd: path.resolve(__dirname, '../') });
-        let tests = files.filter(fileUrl => {
-            return blacklist.includes(fileUrl);
-        }).map(fileUrl => {
-            return {
-                fileUrl,
-                name: getTestName(fileUrl),
-                status: 'pending',
-                results: []
-            };
-        });
-        return tests;
-    }
-}
-
-async function start() {
-    await prepareEChartsVersion('4.2.1'); // Expected version.
-    await prepareEChartsVersion(); // Version to test
-
-    fse.ensureDirSync(path.join(__dirname, getScreenshotDir()));
-    // Start a static server for puppeteer open the html test cases.
-    let {broadcast, io} = serve();
-
-    const browser = await puppeteer.launch({ /* headless: false */ });
-
-    const tests = await getTestsList();
-
-    io.on('connect', socket => {
-        socket.emit('update', {tests});
-        // TODO Stop previous?
-        socket.on('run', async testsNameList => {
-            console.log(testsNameList.join(','));
-
-            const pendingTests = tests.filter(testOpt => {
-                return testsNameList.includes(testOpt.name);
-            });
-
-            for (let testOpt of pendingTests) {
-                // Reset all tests results
-                testOpt.status = 'pending';
-                testOpt.results = [];
-            }
-
-            socket.emit('update', {tests});
+async function runTests(pendingTests) {
+    const browser = await puppeteer.launch({ headless: true });
+    // TODO Not hardcoded.
+    let runtimeCode = fs.readFileSync(path.join(__dirname, 'tmp/testRuntime.js'), 'utf-8');
 
+    try {
+        for (let testOpt of pendingTests) {
+            console.log('Running Test', testOpt.name);
             try {
-                for (let testOpt of pendingTests) {
-                    console.log('Running Test', testOpt.name);
-                    await runTest(browser, testOpt);
-                    socket.emit('update', {tests});
-                    writeTestsToCache(tests);
-                }
+                await runTest(browser, testOpt, runtimeCode);
             }
-            catch(e) {
+            catch (e) {
+                // Restore status
+                testOpt.status = 'pending';
                 console.log(e);
             }
 
-            socket.emit('finish');
-        });
-    });
-
-    console.log(`Dashboard: ${origin}/test/runTest/client/index.html`);
-    // open(`${origin}/test/runTest/client/index.html`);
-
-    // runTests(browser, tests, tests);
+            process.send(testOpt);
+        }
+    }
+    catch(e) {
+        console.log(e);
+    }
 }
 
-start()
\ No newline at end of file
+// Handling input arguments.
+const testsFileUrlList = process.argv[2] || '';
+runTests(testsFileUrlList.split(',').map(fileUrl => {
+    return {
+        fileUrl,
+        name: getTestName(fileUrl),
+        results: [],
+        status: 'pending'
+    };
+}));
\ No newline at end of file

test/runTest/client/client.css

@@ -96,6 +96,21 @@
     margin-left: -20px;
 }
 
+.test-errors, .test-logs {
+    margin-top: 20px;
+    padding: 0 50px;
+}
+
+.test-logs .log-item {
+    margin: 10px 20px;
+    color: #909399;
+}
+
+.test-errors .error-item {
+    margin: 10px 20px;
+    color: #f56c6c
+}
+
 
 ::-webkit-scrollbar {
     height:8px;

test/runTest/client/client.js

@@ -50,7 +50,12 @@ socket.on('connect', () => {
             tests() {
                 let sortFunc = this.sortBy === 'name'
                     ? (a, b) => a.name.localeCompare(b.name)
-                    : (a, b) => a.percentage - b.percentage;
+                    : (a, b) => {
+                        if (a.percentage === b.percentage) {
+                            return a.name.localeCompare(b.name);
+                        }
+                        return a.percentage - b.percentage;
+                    }
 
                 if (!this.searchString) {
                     return this.fullTests.sort(sortFunc);
@@ -133,10 +138,31 @@ socket.on('connect', () => {
     });
     app.$el.style.display = 'block';
 
+    let firstUpdate = true;
     socket.on('update', msg => {
+        let hasFinishedTest = !!msg.tests.find(test => test.status === 'finished');
+        if (!hasFinishedTest && firstUpdate) {
+            app.$confirm("It seems you haven't run any test yet!<br />Do you want to start now?", 'Tip', {
+                confirmButtonText: 'Yes',
+                cancelButtonText: 'No',
+                dangerouslyUseHTMLString: true,
+                center: true
+            }).then(value => {
+                app.running = true;
+                socket.emit('run', msg.tests.map(test => test.name));
+            }).catch(() => {})
+        }
+        // TODO
+        // app.running = !!msg.running;
         app.fullTests = processTestsData(msg.tests, app.fullTests);
+
+        firstUpdate = false;
     });
     socket.on('finish', () => {
+        app.$notify({
+            title: 'Test Complete',
+            position: 'bottom-right'
+        })
         app.running = false;
     });
 

test/runTest/client/index.html

@@ -20,6 +20,12 @@
                     <el-input v-model="searchString" size="mini" placeholder="Filter Tests"></el-input>
                     <div class="controls">
                         <el-checkbox :indeterminate="isSelectAllIndeterminate" v-model="allSelected" @change="handleSelectAllChange"></el-checkbox>
+                        <el-button-group>
+                            <el-button title="Run Selected" @click="runSelectedTests" :loading="running" circle size="mini" type="primary" icon="el-icon-caret-right"></el-button>
+                            <el-button v-if="running" title="Run Selected" @click="stopTests" circle size="mini" type="primary" icon="el-icon-close"></el-button>
+                        </el-button-group>
+
+
                         <el-button
                             style="margin-left: 10px"
                             title="Sort By Failue Percentage" @click="toggleSort" circle size="mini" type="primary" icon="el-icon-sort"
@@ -28,10 +34,6 @@
                             <el-button title="Select All" @click="selectAllTests" circle size="mini" type="primary" icon="el-icon-check"></el-button>
                             <el-button title="Unselect All" @click="unselectAllTests" circle size="mini" type="primary" icon="el-icon-close"></el-button>
                         </el-button-group> -->
-                        <el-button-group>
-                            <el-button title="Run Selected" @click="runSelectedTests" :loading="running" circle size="mini" type="primary" icon="el-icon-caret-right"></el-button>
-                            <el-button v-if="running" title="Run Selected" @click="stopTests" circle size="mini" type="primary" icon="el-icon-close"></el-button>
-                        </el-button-group>
                         <el-button title="Refresh List" @click="refreshList" circle size="mini" type="primary" icon="el-icon-refresh"></el-button>
                     </div>
                 </div>
@@ -92,6 +94,48 @@
                         </el-row>
                     </div>
 
+                    <div class="test-errors">
+                        <el-row :gutter="40">
+                            <el-col :span="12">
+                                <el-alert
+                                    title="Expected Errors"
+                                    type="error"
+                                    show-icon>
+                                </el-alert>
+                                <div class="error-item" v-for="error in currentTest.expectedErrors">{{error}}</div>
+                            </el-col>
+                            <el-col :span="12">
+                                <el-alert
+                                    title="Actual Errors"
+                                    type="error"
+                                    show-icon>
+                                </el-alert>
+                                <div class="error-item"  v-for="error in currentTest.actualErrors">{{error}}</div>
+                            </el-col>
+                        </el-row>
+                    </div>
+
+                    <div class="test-logs">
+                        <el-row :gutter="40">
+                            <el-col :span="12">
+                                <el-alert
+                                    title="Expected Logs"
+                                    type="info"
+                                    show-icon>
+                                </el-alert>
+                                <div class="log-item"  v-for="log in currentTest.expectedLogs">{{log}}</div>
+                            </el-col>
+                            <el-col :span="12">
+                                <el-alert
+                                    title="Actual Logs"
+                                    type="info"
+                                    show-icon>
+                                </el-alert>
+                                <div class="log-item"  v-for="log in currentTest.actualLogs">{{log}}</div>
+                            </el-col>
+                        </el-row>
+                    </div>
+
                     <div class="test-result-nav">
 
                     </div>
@@ -99,6 +143,9 @@
             </el-main>
         </el-container>
     </el-container>
+
+
+
 </div>
 
 <script src="../../../node_modules/socket.io-client/dist/socket.io.js"></script>
@@ -111,6 +158,7 @@
 <script src="client.js"></script>
 
 <link rel="stylesheet" href="client.css">
+<link rel="stylesheet" href="">
 
 </body>
 </html>
\ No newline at end of file

test/runTest/config.js

+module.exports = {
+    port: 8866,
+    origin: 'http://localhost:8866'
+};
\ No newline at end of file

test/runTest/runtime.js

-(function (global) {
-    if (global.autorun) {
-        return;
-    }
-
-    var autorun = {};
-    var inPuppeteer = typeof puppeteerScreenshot !== 'undefined';
-
-    var NativeDate = window.Date;
-
-    var fixedTimestamp = 1566458693300;
-    var actualTimestamp = NativeDate.now();
-
-    function MockDate(params) {
-        if (!params) {
-            var elapsedTime = NativeDate.now() - actualTimestamp;
-            return new NativeDate(fixedTimestamp + elapsedTime);
-        }
-        else {
-            return new NativeDate(params);
-        }
-    }
-    MockDate.prototype = new Date();
-
-    autorun.createScreenshotTest = function (desc, elementQuery, waitTime) {
-
-    }
-
-    /**
-     * Take screenshot immediately.
-     * @param {string} desc
-     * @param {string} [elementQuery] If only screenshot specifed element. Will do full page screenshot if it's not give.
-     */
-    autorun.compareScreenshot = function (desc, elementQuery) {
-        if (!inPuppeteer) {
-            return Promise.resolve();
-        }
-
-        return puppeteerScreenshot(desc, elementQuery);
-    };
-
-    autorun.shouldBe = function (expected, actual) {
-
-    };
-
-    /**
-     * Finish the test.
-     */
-    autorun.finish = function () {
-        if (!inPuppeteer) {
-            return Promise.resolve();
-        }
-        return puppeteerFinishTest();
-    };
-
-
-    if (inPuppeteer) {
-        let myRandom = new Math.seedrandom('echarts-test');
-        // Fixed random generator
-        Math.random = function () {
-            var val = myRandom();
-            return val;
-        };
-
-        // Fixed date
-        window.Date = MockDate;
-    }
-
-    global.autorun = autorun;
-
-})(window);
\ No newline at end of file

test/runTest/runtime/main.js

+import seedrandom from 'seedrandom';
+import lolex from 'lolex';
+
+const NativeDate = window.Date;
+
+const fixedTimestamp = 1566458693300;
+// const actualTimestamp = NativeDate.now();
+// function MockDate(params) {
+//     if (!params) {
+//         const elapsedTime = NativeDate.now() - actualTimestamp;
+//         return new NativeDate(fixedTimestamp + elapsedTime);
+//     }
+//     else {
+//         return new NativeDate(params);
+//     }
+// }
+// MockDate.prototype = new Date();
+
+export function createScreenshotTest (desc, elementQuery, waitTime) {
+
+}
+
+/**
+ * Take screenshot immediately.
+ * @param {string} desc
+ * @param {string} [elementQuery] If only screenshot specifed element. Will do full page screenshot if it's not give.
+ */
+export function compareScreenshot (desc, elementQuery) {
+    return puppeteerScreenshot(desc, elementQuery);
+};
+
+/**
+ * Finish the test.
+ */
+export function finish() {
+    return puppeteerFinishTest();
+};
+
+
+let myRandom = new seedrandom('echarts-test');
+// Fixed random generator
+Math.random = function () {
+    const val = myRandom();
+    return val;
+};
+
+// Fixed date
+// window.Date = MockDate;
+
+lolex.install({
+    shouldAdvanceTime: true,
+    advanceTimeDelta: 50,
+    now: fixedTimestamp
+});

test/runTest/runtime/shim.js

+(function () {
+    if (typeof autorun !== 'undefined') {
+        return;
+    }
+    var autorun = {};
+    autorun.createScreenshotTest = function () {};
+    autorun.compareScreenshot = function () {};
+    autorun.finish = function () {};
+})();
\ No newline at end of file

test/runTest/serve.js

-const handler = require('serve-handler');
-const http = require('http');
-const port = 8866;
-const origin = `http://localhost:${port}`;
-
-function serve() {
-    const server = http.createServer((request, response) => {
-        return handler(request, response, {
-            cleanUrls: false,
-            // Root folder of echarts
-            public: __dirname + '/../../'
-        });
-    });
-
-    server.listen(port, () => {
-        console.log(`Server started. ${origin}`);
-    });
-
-
-    const io = require('socket.io')(server);
-    return {
-        io
-    }
-}
-
-module.exports = {serve, origin};
\ No newline at end of file

test/runTest/server.js

+const handler = require('serve-handler');
+const http = require('http');
+const rollup = require('rollup');
+const resolve = require('rollup-plugin-node-resolve');
+const commonjs = require('rollup-plugin-commonjs');
+const path = require('path');
+const open = require('open');
+const fse = require('fs-extra');
+const {fork} = require('child_process');
+const {port, origin} = require('./config');
+const {getTestsList, prepareTestsList, saveTestsList, mergeTestsResults} = require('./store');
+const {prepareEChartsVersion} = require('./util');
+
+function serve() {
+    const server = http.createServer((request, response) => {
+        return handler(request, response, {
+            cleanUrls: false,
+            // Root folder of echarts
+            public: __dirname + '/../../'
+        });
+    });
+
+    server.listen(port, () => {
+        console.log(`Server started. ${origin}`);
+    });
+
+
+    const io = require('socket.io')(server);
+    return {
+        io
+    };
+};
+
+async function buildRuntimeCode() {
+    const bundle = await rollup.rollup({
+        input: path.join(__dirname, 'runtime/main.js'),
+        plugins: [
+            resolve(),
+            commonjs()
+        ]
+    });
+    const output = await bundle.generate({
+        format: 'iife',
+        name: 'autorun'
+    });
+
+    return output.code;
+}
+
+
+function startTests(testsNameList, socket) {
+    console.log(testsNameList.join(','));
+
+    return new Promise(resolve => {
+        const pendingTests = getTestsList().filter(testOpt => {
+            return testsNameList.includes(testOpt.name);
+        });
+
+        for (let testOpt of pendingTests) {
+            // Reset all tests results
+            testOpt.status = 'pending';
+            testOpt.results = [];
+        }
+
+        socket.emit('update', {tests: getTestsList()});
+
+        let childProcess = fork(path.join(__dirname, 'cli.js'), [
+            pendingTests.map(testOpt => testOpt.fileUrl)
+        ]);
+        // Finished one test
+        childProcess.on('message', testOpt => {
+            mergeTestsResults([testOpt]);
+            // Merge tests.
+            socket.emit('update', {tests: getTestsList(), running: true});
+            saveTestsList();
+        });
+        // Finished all
+        childProcess.on('exit', () => {
+            resolve();
+        });
+    });
+}
+
+async function start() {
+    await prepareEChartsVersion('4.2.1'); // Expected version.
+    await prepareEChartsVersion(); // Version to test
+
+    let runtimeCode = await buildRuntimeCode();
+    // seedrandom use crypto as external module. Set it to null to avoid not defined error.
+    // TODO
+    runtimeCode = 'window.crypto = null\n' + runtimeCode;
+    fse.outputFileSync(path.join(__dirname, 'tmp/testRuntime.js'), runtimeCode, 'utf-8');
+
+    // Start a static server for puppeteer open the html test cases.
+    let {io} = serve();
+
+    io.on('connect', async socket => {
+        await prepareTestsList();
+
+        socket.emit('update', {tests: getTestsList()});
+        // TODO Stop previous?
+        socket.on('run', async testsNameList => {
+            await startTests(testsNameList, socket);
+            socket.emit('finish');
+        });
+    });
+
+    console.log(`Dashboard: ${origin}/test/runTest/client/index.html`);
+    // open(`${origin}/test/runTest/client/index.html`);
+
+}
+
+start();
\ No newline at end of file

test/runTest/store.js

+const path = require('path');
+const fse = require('fs-extra');
+const fs = require('fs');
+const glob = require('glob');
+const {getTestName} = require('./util');
+const util = require('util');
+const blacklist = require('./blacklist');
+
+let _tests = [];
+let _testsMap = {};
+
+function getCacheFilePath() {
+    return path.join(__dirname, 'tmp/__cache__.json');;
+}
+
+module.exports.getTestsList = function () {
+    return _tests;
+};
+
+module.exports.getTestByFileUrl = function (url) {
+    return _testsMap[url];
+};
+
+module.exports.prepareTestsList = async function () {
+    let tmpFolder = path.join(__dirname, 'tmp');
+    fse.ensureDirSync(tmpFolder);
+    _tests = [];
+    _testsMap = {};
+    try {
+        let cachedStr = fs.readFileSync(getCacheFilePath(), 'utf-8');
+        _tests = JSON.parse(cachedStr);
+        _tests.forEach(test => {
+            _testsMap[test.fileUrl] = test;
+        });
+    }
+    catch(e) {
+        _tests = [];
+    }
+    // Find if there is new html file
+    let files = await util.promisify(glob)('**.html', { cwd: path.resolve(__dirname, '../') });
+    files.forEach(fileUrl => {
+        if (blacklist.includes(fileUrl)) {
+            return;
+        }
+        if (_testsMap[fileUrl]) {
+            return;
+        }
+
+        let test = {
+            fileUrl,
+            name: getTestName(fileUrl),
+            status: 'pending',
+            results: []
+        };
+
+        _tests.push(test);
+        _testsMap[fileUrl] = test;
+    });
+    return _tests;
+};
+
+module.exports.saveTestsList = function () {
+    fse.outputFileSync(getCacheFilePath(), JSON.stringify(_tests, null, 2), 'utf-8');
+};
+
+module.exports.mergeTestsResults = function (testsResults) {
+    testsResults.forEach(testResult => {
+        if (_testsMap[testResult.fileUrl]) {
+            Object.assign(_testsMap[testResult.fileUrl], testResult);
+        }
+    });
+};
\ No newline at end of file

test/runTest/util.js

+const path = require('path');
+const fse = require('fs-extra');
+const https = require('https');
+const fs = require('fs');
+
+module.exports.getTestName = function(fileUrl) {
+    return path.basename(fileUrl, '.html');
+};
+
+
+function getVersionDir(version) {
+    version = version || 'developing';
+    return `tmp/__version__/${version}`;
+};
+module.exports.getVersionDir = getVersionDir;
+
+
+module.exports.prepareEChartsVersion = function (version) {
+    let versionFolder = path.join(__dirname, getVersionDir(version));
+    fse.ensureDirSync(versionFolder);
+    if (!version) {
+        // Developing version, make sure it's new build
+        return fse.copy(
+            path.join(__dirname, '../../dist/echarts.js'),
+            `${versionFolder}/echarts.js`
+        );
+    }
+    return new Promise(resolve => {
+        if (!fs.existsSync(`${versionFolder}/echarts.js`)) {
+            const file = fs.createWriteStream(`${versionFolder}/echarts.js`);
+
+            console.log('Downloading echarts4.2.1 from ', `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`);
+            https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => {
+                response.pipe(file);
+
+                file.on('finish', () => {
+                    resolve();
+                });
+            });
+        }
+        else {
+            resolve();
+        }
+    });
+}
\ No newline at end of file