diff --git a/test/runTest/blacklist.js b/test/runTest/blacklist.js index b1a5204c605841994e1ef213b06ef7e5ed2a6ce2..e3d220f097199b1ecc9d8a0a83f2b77979791a1a 100644 --- a/test/runTest/blacklist.js +++ b/test/runTest/blacklist.js @@ -29,5 +29,10 @@ module.exports = [ 'finished-gl.html', 'scatter-gps.html', - 'webkit-dep.html' + 'webkit-dep.html', + + // This case will have timeout + 'visualMap-performance1.html', + 'lines-bus.html', + 'lines-stream-not-large.html' ]; diff --git a/test/runTest/cli.js b/test/runTest/cli.js index 70be90f4342fee74f8c5e0609a915f69fb6f1cf1..62bf18df214b4a9d33bfa5219d19d89a42fb6fa6 100644 --- a/test/runTest/cli.js +++ b/test/runTest/cli.js @@ -32,11 +32,15 @@ const Timeline = require('./Timeline'); program .option('-t, --tests ', 'Tests names list') .option('--no-headless', 'Not headless') - .option('-s, --speed ', 'Playback speed'); + .option('-s, --speed ', 'Playback speed') + .option('--expected ', 'Expected version') + .option('--actual ', 'Actual version'); program.parse(process.argv); program.speed = +program.speed || 1; +program.actual = program.actual || 'local'; +program.expected = program.expected || '4.2.1'; if (!program.tests) { throw new Error('Tests are required'); @@ -144,6 +148,9 @@ async function runTestPage(browser, testOpt, version, runtimeCode) { page.on('pageerror', error => { errors.push(error.toString()); }); + page.on('dialog', async dialog => { + await dialog.dismiss(); + }); try { await page.setViewport({width: 800, height: 600}); @@ -151,19 +158,20 @@ async function runTestPage(browser, testOpt, version, runtimeCode) { waitUntil: 'networkidle2', timeout: 10000 }); + + await waitTime(200); // Wait for animation or something else. Pending + // Final shot. + await page.mouse.move(0, 0); + let desc = 'Full Shot'; + const {screenshotName, screenshotPath} = await takeScreenshot(page, true, fileUrl, desc, version); + screenshots.push({screenshotName, desc, screenshotPath}); + + await runActions(page, testOpt, version, screenshots); } catch(e) { console.error(e); } - await waitTime(200); // Wait for animation or something else. Pending - // Final shot. - let desc = 'Full Shot'; - const {screenshotName, screenshotPath} = await takeScreenshot(page, true, fileUrl, desc, version); - screenshots.push({screenshotName, desc, screenshotPath}); - - await runActions(page, testOpt, version, screenshots); - await page.close(); return { @@ -181,10 +189,10 @@ async function writePNG(diffPNG, diffPath) { }); }; -async function runTest(browser, testOpt, runtimeCode) { +async function runTest(browser, testOpt, runtimeCode, expectedVersion, actualVersion) { testOpt.status === 'running'; - const expectedResult = await runTestPage(browser, testOpt, '4.2.1', runtimeCode); - const actualResult = await runTestPage(browser, testOpt, '', runtimeCode); + const expectedResult = await runTestPage(browser, testOpt, expectedVersion, runtimeCode); + const actualResult = await runTestPage(browser, testOpt, actualVersion, runtimeCode); // sortScreenshots(expectedResult.screenshots); // sortScreenshots(actualResult.screenshots); @@ -218,6 +226,8 @@ async function runTest(browser, testOpt, runtimeCode) { testOpt.expectedLogs = expectedResult.logs; testOpt.actualErrors = actualResult.errors; testOpt.expectedErrors = expectedResult.errors; + testOpt.actualVersion = actualVersion; + testOpt.expectedVersion = expectedVersion; testOpt.lastRun = Date.now(); } @@ -235,11 +245,11 @@ async function runTests(pendingTests) { for (let testOpt of pendingTests) { console.log('Running Test', testOpt.name); try { - await runTest(browser, testOpt, runtimeCode); + await runTest(browser, testOpt, runtimeCode, program.expected, program.actual); } catch (e) { // Restore status - testOpt.status = 'pending'; + testOpt.status = 'unsettled'; console.log(e); } diff --git a/test/runTest/client/client.css b/test/runTest/client/client.css index 01c448ba29a2fa8532020260f0ecd7b2cc001ef8..1dc5484820832668e0ff7318cd086a0a414136e8 100644 --- a/test/runTest/client/client.css +++ b/test/runTest/client/client.css @@ -79,6 +79,11 @@ margin-left: 5px; cursor: pointer; } +.nav-toolbar .el-button { + padding-left: 8px; + padding-right: 8px; +} + .run-config-item { margin: 5px 0; } diff --git a/test/runTest/client/client.js b/test/runTest/client/client.js index 5c67aff627342cf3b7656fc332517632ade326d5..f910178928af5bca2c75006d40f74ab36e566c0a 100644 --- a/test/runTest/client/client.js +++ b/test/runTest/client/client.js @@ -67,9 +67,13 @@ const app = new Vue({ allSelected: false, lastSelectedIndex: -1, + versions: [], + runConfig: { noHeadless: false, replaySpeed: 5, + actualVersion: 'local', + expectedVersion: null, threads: 1 } }, @@ -162,15 +166,20 @@ const app = new Vue({ this.tests[i].selected = selected; } }, - refreshList() { - - }, - runTest(testName) { + runSingleTest(testName) { runTests([testName]); }, - runSelectedTests() { + run(runTarget) { const tests = this.fullTests.filter(test => { - return test.selected; + if (runTarget === 'selected') { + return test.selected; + } + else if (runTarget === 'unfinished') { + return test.status !== 'finished'; + } + else { // Run all + return true; + } }).map(test => { return test.name; }); @@ -184,23 +193,31 @@ const app = new Vue({ }); function runTests(tests) { - if (tests.length > 0) { - app.running = true; - socket.emit('run', { - tests, - threads: app.runConfig.threads, - noHeadless: app.runConfig.noHeadless, - replaySpeed: app.runConfig.noHeadless - ? app.runConfig.replaySpeed - : 5 // Force run at 5x speed + if (!tests.length) { + app.$notify({ + title: 'No test selected.', + position: 'top-right' }); + return; } - else { + if (!app.runConfig.expectedVersion || !app.runConfig.actualVersion) { app.$notify({ - title: 'No test selected.', + title: 'No echarts version selected.', position: 'top-right' }); + return; } + app.running = true; + socket.emit('run', { + tests, + expectedVersion: app.runConfig.expectedVersion, + actualVersion: app.runConfig.actualVersion, + threads: app.runConfig.threads, + noHeadless: app.runConfig.noHeadless, + replaySpeed: app.runConfig.noHeadless + ? app.runConfig.replaySpeed + : 5 // Force run at 5x speed + }); } @@ -240,6 +257,17 @@ socket.on('finish', res => { console.log(`${res.count} test complete, Cost: ${(res.time / 1000).toFixed(1)} s. Threads: ${res.threads}`); app.running = false; }); +socket.on('versions', versions => { + app.versions = versions.filter(version => { + return !version.startsWith('2.') + && !version.match('beta') + && !version.match('rc'); + }).reverse(); + if (!app.runConfig.expectedVersion) { + app.runConfig.expectedVersion = app.versions[0]; + } + app.versions.unshift('local'); +}); function updateTestHash() { app.currentTestName = window.location.hash.slice(1); diff --git a/test/runTest/client/index.html b/test/runTest/client/index.html index b7a47fd1f529bfc946f82219f9cb4715d0e5c478..64c2eba62f290d956fb6b19db1e27dcd5e9020a6 100644 --- a/test/runTest/client/index.html +++ b/test/runTest/client/index.html @@ -40,21 +40,26 @@ under the License.
- + Sort - - - + + Run selected + + Run unfinished + Run all + + + + + Stop + -
- Threads - -
Replay
+
+ Threads + +
+
+ Version + Expected + + + + Actual + + + +
- - -
    @@ -130,7 +148,7 @@ under the License. >

    {{currentTest.name}}

    - + Open Demo @@ -146,7 +164,7 @@ under the License.
    - Expected + Expected - {{currentTest.expectedVersion || ''}}
    @@ -155,7 +173,7 @@ under the License.
    - Actual + Actual - {{currentTest.actualVersion || ''}}
    @@ -164,7 +182,7 @@ under the License.
    - Diff({{result.diffRatio.toFixed(4)}}) + Diff - {{result.diffRatio.toFixed(4)}}
    diff --git a/test/runTest/server.js b/test/runTest/server.js index 3d6805dcff7a92f9d68f0b41271e5036a331ca5a..da2f769d9aed4695038db1a41f14a31071c141f8 100644 --- a/test/runTest/server.js +++ b/test/runTest/server.js @@ -25,7 +25,7 @@ const {fork} = require('child_process'); const semver = require('semver'); const {port, origin} = require('./config'); const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults, updateActionsMeta} = require('./store'); -const {prepareEChartsVersion, getActionsFullPath} = require('./util'); +const {prepareEChartsLib, getActionsFullPath, fetchVersions} = require('./util'); const fse = require('fs-extra'); const fs = require('fs'); @@ -75,12 +75,16 @@ class Thread { this.onUpdate; } - fork(noHeadless, replaySpeed) { + fork(noHeadless, replaySpeed, actualVersion, expectedVersion) { let p = fork(path.join(__dirname, 'cli.js'), [ '--tests', this.tests.map(testOpt => testOpt.name).join(','), '--speed', replaySpeed || 5, + '--actual', + actualVersion, + '--expected', + expectedVersion, ...(noHeadless ? ['--no-headless'] : []) ]); this.p = p; @@ -108,7 +112,9 @@ class Thread { function startTests(testsNameList, socket, { noHeadless, threadsCount, - replaySpeed + replaySpeed, + actualVersion, + expectedVersion }) { console.log('Received: ', testsNameList.join(',')); @@ -147,7 +153,7 @@ function startTests(testsNameList, socket, { for (let i = 0; i < threadsCount; i++) { runningThreads[i].onExit = onExit; runningThreads[i].onUpdate = onUpdate; - runningThreads[i].fork(noHeadless, replaySpeed); + runningThreads[i].fork(noHeadless, replaySpeed, actualVersion, expectedVersion); runningCount++; } // If something bad happens and no proccess are started successfully @@ -174,8 +180,7 @@ async function start() { return; } - await prepareEChartsVersion('4.2.1'); // Expected version. - await prepareEChartsVersion(); // Version to test + let versions = await fetchVersions(); // let runtimeCode = await buildRuntimeCode(); // fse.outputFileSync(path.join(__dirname, 'tmp/testRuntime.js'), runtimeCode, 'utf-8'); @@ -189,7 +194,12 @@ async function start() { socket.emit('update', {tests: getTestsList()}); socket.on('run', async data => { + let startTime = Date.now(); + + await prepareEChartsLib(data.expectedVersion); // Expected version. + await prepareEChartsLib(data.actualVersion); // Version to test + // TODO Should broadcast to all sockets. try { await startTests( @@ -198,7 +208,9 @@ async function start() { { noHeadless: data.noHeadless, threadsCount: data.threads, - replaySpeed: data.replaySpeed + replaySpeed: data.replaySpeed, + actualVersion: data.actualVersion, + expectedVersion: data.expectedVersion } ); } @@ -213,6 +225,8 @@ async function start() { socket.on('stop', () => { stopRunningTests(); }); + + socket.emit('versions', versions); }); io.of('/recorder').on('connect', async socket => { @@ -245,7 +259,9 @@ async function start() { await startTests([data.testName], socket, { noHeadless: true, threadsCount: 1, - replaySpeed: 2 + replaySpeed: 2, + actualVersion: data.actualVersion, + expectedVersion: data.expectedVersion }); } catch (e) { console.error(e); } @@ -264,7 +280,7 @@ async function start() { }); console.log(`Dashboard: ${origin}/test/runTest/client/index.html`); - console.log(`Interaction Recorder: ${origin}/test/runTest/recorder/index.html`); + // console.log(`Interaction Recorder: ${origin}/test/runTest/recorder/index.html`); // open(`${origin}/test/runTest/client/index.html`); } diff --git a/test/runTest/util.js b/test/runTest/util.js index c638f69d34b5cb2456a6a148ba326656bead6339..2e551e3bf6f565c1eb867efa264df2c01ed4a400 100644 --- a/test/runTest/util.js +++ b/test/runTest/util.js @@ -34,7 +34,7 @@ module.exports.fileNameFromTest = function (testName) { }; function getVersionDir(version) { - version = version || 'developing'; + version = version || 'local'; return `tmp/__version__/${version}`; }; module.exports.getVersionDir = getVersionDir; @@ -44,10 +44,10 @@ module.exports.getActionsFullPath = function (testName) { }; -module.exports.prepareEChartsVersion = function (version) { +module.exports.prepareEChartsLib = function (version) { let versionFolder = path.join(__dirname, getVersionDir(version)); fse.ensureDirSync(versionFolder); - if (!version) { + if (!version || version === 'local') { // Developing version, make sure it's new build return fse.copy( path.join(__dirname, '../../dist/echarts.js'), @@ -58,7 +58,7 @@ module.exports.prepareEChartsVersion = function (version) { if (!fs.existsSync(`${versionFolder}/echarts.js`)) { const file = fs.createWriteStream(`${versionFolder}/echarts.js`); - console.log('Downloading echarts4.2.1 from ', `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`); + console.log(`Downloading echarts@${version} from `, `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`); https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => { response.pipe(file); @@ -73,6 +73,29 @@ module.exports.prepareEChartsVersion = function (version) { }); }; +module.exports.fetchVersions = function () { + return new Promise((resolve, reject) => { + https.get(`https://registry.npmjs.org/echarts`, res => { + if (res.statusCode !== 200) { + res.destroy(); + reject('Failed fetch versions from https://registry.npmjs.org/echarts'); + return; + } + var buffers = []; + res.on('data', buffers.push.bind(buffers)); + res.on('end', function () { + try { + var data = Buffer.concat(buffers); + resolve(Object.keys(JSON.parse(data).versions)); + } + catch (e) { + reject(e.toString()); + } + }); + }); + }); +}; + module.exports.buildRuntimeCode = async function () { const bundle = await rollup.rollup({ input: path.join(__dirname, 'runtime/main.js'),