提交 bf3a0ae1 编写于 作者: P pissang

test: add version selection in visual regression testing tool

上级 f06d88f9
...@@ -29,5 +29,10 @@ module.exports = [ ...@@ -29,5 +29,10 @@ module.exports = [
'finished-gl.html', 'finished-gl.html',
'scatter-gps.html', 'scatter-gps.html',
'webkit-dep.html' 'webkit-dep.html',
// This case will have timeout
'visualMap-performance1.html',
'lines-bus.html',
'lines-stream-not-large.html'
]; ];
...@@ -32,11 +32,15 @@ const Timeline = require('./Timeline'); ...@@ -32,11 +32,15 @@ const Timeline = require('./Timeline');
program program
.option('-t, --tests <tests>', 'Tests names list') .option('-t, --tests <tests>', 'Tests names list')
.option('--no-headless', 'Not headless') .option('--no-headless', 'Not headless')
.option('-s, --speed <speed>', 'Playback speed'); .option('-s, --speed <speed>', 'Playback speed')
.option('--expected <expected>', 'Expected version')
.option('--actual <actual>', 'Actual version');
program.parse(process.argv); program.parse(process.argv);
program.speed = +program.speed || 1; program.speed = +program.speed || 1;
program.actual = program.actual || 'local';
program.expected = program.expected || '4.2.1';
if (!program.tests) { if (!program.tests) {
throw new Error('Tests are required'); throw new Error('Tests are required');
...@@ -144,6 +148,9 @@ async function runTestPage(browser, testOpt, version, runtimeCode) { ...@@ -144,6 +148,9 @@ async function runTestPage(browser, testOpt, version, runtimeCode) {
page.on('pageerror', error => { page.on('pageerror', error => {
errors.push(error.toString()); errors.push(error.toString());
}); });
page.on('dialog', async dialog => {
await dialog.dismiss();
});
try { try {
await page.setViewport({width: 800, height: 600}); await page.setViewport({width: 800, height: 600});
...@@ -151,18 +158,19 @@ async function runTestPage(browser, testOpt, version, runtimeCode) { ...@@ -151,18 +158,19 @@ async function runTestPage(browser, testOpt, version, runtimeCode) {
waitUntil: 'networkidle2', waitUntil: 'networkidle2',
timeout: 10000 timeout: 10000
}); });
}
catch(e) {
console.error(e);
}
await waitTime(200); // Wait for animation or something else. Pending await waitTime(200); // Wait for animation or something else. Pending
// Final shot. // Final shot.
await page.mouse.move(0, 0);
let desc = 'Full Shot'; let desc = 'Full Shot';
const {screenshotName, screenshotPath} = await takeScreenshot(page, true, fileUrl, desc, version); const {screenshotName, screenshotPath} = await takeScreenshot(page, true, fileUrl, desc, version);
screenshots.push({screenshotName, desc, screenshotPath}); screenshots.push({screenshotName, desc, screenshotPath});
await runActions(page, testOpt, version, screenshots); await runActions(page, testOpt, version, screenshots);
}
catch(e) {
console.error(e);
}
await page.close(); await page.close();
...@@ -181,10 +189,10 @@ async function writePNG(diffPNG, diffPath) { ...@@ -181,10 +189,10 @@ async function writePNG(diffPNG, diffPath) {
}); });
}; };
async function runTest(browser, testOpt, runtimeCode) { async function runTest(browser, testOpt, runtimeCode, expectedVersion, actualVersion) {
testOpt.status === 'running'; testOpt.status === 'running';
const expectedResult = await runTestPage(browser, testOpt, '4.2.1', runtimeCode); const expectedResult = await runTestPage(browser, testOpt, expectedVersion, runtimeCode);
const actualResult = await runTestPage(browser, testOpt, '', runtimeCode); const actualResult = await runTestPage(browser, testOpt, actualVersion, runtimeCode);
// sortScreenshots(expectedResult.screenshots); // sortScreenshots(expectedResult.screenshots);
// sortScreenshots(actualResult.screenshots); // sortScreenshots(actualResult.screenshots);
...@@ -218,6 +226,8 @@ async function runTest(browser, testOpt, runtimeCode) { ...@@ -218,6 +226,8 @@ async function runTest(browser, testOpt, runtimeCode) {
testOpt.expectedLogs = expectedResult.logs; testOpt.expectedLogs = expectedResult.logs;
testOpt.actualErrors = actualResult.errors; testOpt.actualErrors = actualResult.errors;
testOpt.expectedErrors = expectedResult.errors; testOpt.expectedErrors = expectedResult.errors;
testOpt.actualVersion = actualVersion;
testOpt.expectedVersion = expectedVersion;
testOpt.lastRun = Date.now(); testOpt.lastRun = Date.now();
} }
...@@ -235,11 +245,11 @@ async function runTests(pendingTests) { ...@@ -235,11 +245,11 @@ async function runTests(pendingTests) {
for (let testOpt of pendingTests) { for (let testOpt of pendingTests) {
console.log('Running Test', testOpt.name); console.log('Running Test', testOpt.name);
try { try {
await runTest(browser, testOpt, runtimeCode); await runTest(browser, testOpt, runtimeCode, program.expected, program.actual);
} }
catch (e) { catch (e) {
// Restore status // Restore status
testOpt.status = 'pending'; testOpt.status = 'unsettled';
console.log(e); console.log(e);
} }
......
...@@ -79,6 +79,11 @@ ...@@ -79,6 +79,11 @@
margin-left: 5px; margin-left: 5px;
cursor: pointer; cursor: pointer;
} }
.nav-toolbar .el-button {
padding-left: 8px;
padding-right: 8px;
}
.run-config-item { .run-config-item {
margin: 5px 0; margin: 5px 0;
} }
......
...@@ -67,9 +67,13 @@ const app = new Vue({ ...@@ -67,9 +67,13 @@ const app = new Vue({
allSelected: false, allSelected: false,
lastSelectedIndex: -1, lastSelectedIndex: -1,
versions: [],
runConfig: { runConfig: {
noHeadless: false, noHeadless: false,
replaySpeed: 5, replaySpeed: 5,
actualVersion: 'local',
expectedVersion: null,
threads: 1 threads: 1
} }
}, },
...@@ -162,15 +166,20 @@ const app = new Vue({ ...@@ -162,15 +166,20 @@ const app = new Vue({
this.tests[i].selected = selected; this.tests[i].selected = selected;
} }
}, },
refreshList() { runSingleTest(testName) {
},
runTest(testName) {
runTests([testName]); runTests([testName]);
}, },
runSelectedTests() { run(runTarget) {
const tests = this.fullTests.filter(test => { const tests = this.fullTests.filter(test => {
if (runTarget === 'selected') {
return test.selected; return test.selected;
}
else if (runTarget === 'unfinished') {
return test.status !== 'finished';
}
else { // Run all
return true;
}
}).map(test => { }).map(test => {
return test.name; return test.name;
}); });
...@@ -184,23 +193,31 @@ const app = new Vue({ ...@@ -184,23 +193,31 @@ const app = new Vue({
}); });
function runTests(tests) { function runTests(tests) {
if (tests.length > 0) { if (!tests.length) {
app.$notify({
title: 'No test selected.',
position: 'top-right'
});
return;
}
if (!app.runConfig.expectedVersion || !app.runConfig.actualVersion) {
app.$notify({
title: 'No echarts version selected.',
position: 'top-right'
});
return;
}
app.running = true; app.running = true;
socket.emit('run', { socket.emit('run', {
tests, tests,
expectedVersion: app.runConfig.expectedVersion,
actualVersion: app.runConfig.actualVersion,
threads: app.runConfig.threads, threads: app.runConfig.threads,
noHeadless: app.runConfig.noHeadless, noHeadless: app.runConfig.noHeadless,
replaySpeed: app.runConfig.noHeadless replaySpeed: app.runConfig.noHeadless
? app.runConfig.replaySpeed ? app.runConfig.replaySpeed
: 5 // Force run at 5x speed : 5 // Force run at 5x speed
}); });
}
else {
app.$notify({
title: 'No test selected.',
position: 'top-right'
});
}
} }
...@@ -240,6 +257,17 @@ socket.on('finish', res => { ...@@ -240,6 +257,17 @@ socket.on('finish', res => {
console.log(`${res.count} test complete, Cost: ${(res.time / 1000).toFixed(1)} s. Threads: ${res.threads}`); console.log(`${res.count} test complete, Cost: ${(res.time / 1000).toFixed(1)} s. Threads: ${res.threads}`);
app.running = false; app.running = false;
}); });
socket.on('versions', versions => {
app.versions = versions.filter(version => {
return !version.startsWith('2.')
&& !version.match('beta')
&& !version.match('rc');
}).reverse();
if (!app.runConfig.expectedVersion) {
app.runConfig.expectedVersion = app.versions[0];
}
app.versions.unshift('local');
});
function updateTestHash() { function updateTestHash() {
app.currentTestName = window.location.hash.slice(1); app.currentTestName = window.location.hash.slice(1);
......
...@@ -40,21 +40,26 @@ under the License. ...@@ -40,21 +40,26 @@ under the License.
<el-input v-model="searchString" size="mini" placeholder="Filter Tests"></el-input> <el-input v-model="searchString" size="mini" placeholder="Filter Tests"></el-input>
<div class="controls"> <div class="controls">
<el-checkbox :indeterminate="isSelectAllIndeterminate" v-model="allSelected" @change="handleSelectAllChange"></el-checkbox> <el-checkbox :indeterminate="isSelectAllIndeterminate" v-model="allSelected" @change="handleSelectAllChange"></el-checkbox>
<el-button <el-button title="Sort By Failue Percentage" @click="toggleSort" size="mini" type="primary" icon="el-icon-sort">Sort</el-button>
title="Sort By Failue Percentage" @click="toggleSort" circle size="mini" type="primary" icon="el-icon-sort"
></el-button>
<el-button-group style="margin-left: 10px"> <el-dropdown v-if="!running" split-button type="primary" size="mini" title="Run"
<el-button title="Run Selected" @click="runSelectedTests" :loading="running" circle size="mini" type="primary" icon="el-icon-caret-right"></el-button> @click="run('selected')"
<el-button v-if="running" title="Run Selected" @click="stopTests" circle size="mini" type="primary" icon="el-icon-close"></el-button> @command="run"
>
<i class="el-icon-caret-right"></i> Run selected
<el-dropdown-menu slot="dropdown" >
<el-dropdown-item command="unfinished">Run unfinished</el-dropdown-item>
<el-dropdown-item command="all">Run all</el-dropdown-item>
</el-dropdown-menu>
</el-dropdown>
<el-button-group v-else>
<el-button type="primary" size="mini" :loading="true">Stop</el-button>
<el-button title="Run Selected" @click="stopTests" size="mini" type="primary" icon="el-icon-close" style="padding-left: 3px;padding-right:3px;"></el-button>
</el-button-group> </el-button-group>
<el-popover title="Configuration" class="run-configuration"> <el-popover title="Configuration" class="run-configuration">
<div class="run-config-item">
<span>Threads</span>
<el-slider style="width: 140px;" v-model="runConfig.threads" :step="1" :min="1" :max="8" show-stops></el-slider>
</div>
<div class="run-config-item"> <div class="run-config-item">
<el-checkbox v-model="runConfig.noHeadless">Replay</el-checkbox> <el-checkbox v-model="runConfig.noHeadless">Replay</el-checkbox>
<el-slider <el-slider
...@@ -66,15 +71,28 @@ under the License. ...@@ -66,15 +71,28 @@ under the License.
:disabled="!runConfig.noHeadless" :disabled="!runConfig.noHeadless"
></el-slider> ></el-slider>
</div> </div>
<div class="run-config-item">
<span>Threads</span>
<el-slider style="width: 140px;" v-model="runConfig.threads" :step="1" :min="1" :max="8" show-stops></el-slider>
</div>
<div class="run-config-item">
<span>Version </span>
<span style="font-size: 12px; color:#afafaf">Expected</span>
<el-select size="mini" v-model="runConfig.expectedVersion" placeholder="Select Version"
style="width: 80px;"
>
<el-option v-for="version in versions" :key="version" :label="version" :value="version"></el-option>
</el-select>
<span style="font-size: 12px; color: #afafaf">Actual</span>
<el-select size="mini" v-model="runConfig.actualVersion" placeholder="Select Version"
style="width: 80px;"
>
<el-option v-for="version in versions" :key="version" :label="version" :value="version"></el-option>
</el-select>
</div>
<i slot="reference" class="el-icon-setting"></i> <i slot="reference" class="el-icon-setting"></i>
</el-popover> </el-popover>
<!-- <el-button-group>
<el-button title="Select All" @click="selectAllTests" circle size="mini" type="primary" icon="el-icon-check"></el-button>
<el-button title="Unselect All" @click="unselectAllTests" circle size="mini" type="primary" icon="el-icon-close"></el-button>
</el-button-group> -->
<!-- <el-button title="Refresh List" @click="refreshList" circle size="mini" type="primary" icon="el-icon-refresh"></el-button> -->
</div> </div>
</div> </div>
<ul class="test-list"> <ul class="test-list">
...@@ -130,7 +148,7 @@ under the License. ...@@ -130,7 +148,7 @@ under the License.
></el-progress> ></el-progress>
<h3>{{currentTest.name}}</h3> <h3>{{currentTest.name}}</h3>
<el-button-group style="margin-left: 10px"> <el-button-group style="margin-left: 10px">
<el-button title="Run Selected" @click="runTest(currentTest.name)" :loading="running" circle type="primary" icon="el-icon-caret-right"></el-button> <el-button title="Run Selected" @click="runSingleTest(currentTest.name)" :loading="running" circle type="primary" icon="el-icon-caret-right"></el-button>
<el-button v-if="running" title="Run Selected" @click="stopTests" circle type="primary" icon="el-icon-close"></el-button> <el-button v-if="running" title="Run Selected" @click="stopTests" circle type="primary" icon="el-icon-close"></el-button>
</el-button-group> </el-button-group>
<a target="_blank" :href="currentTestUrl"><i class="el-icon-link"></i>Open Demo</a> <a target="_blank" :href="currentTestUrl"><i class="el-icon-link"></i>Open Demo</a>
...@@ -146,7 +164,7 @@ under the License. ...@@ -146,7 +164,7 @@ under the License.
<el-col :span="8"> <el-col :span="8">
<el-card shadow="hover"> <el-card shadow="hover">
<div slot="header" class="clearfix"> <div slot="header" class="clearfix">
<span>Expected</span> <span>Expected - {{currentTest.expectedVersion || ''}}</span>
</div> </div>
<el-image :src="result.expected"></el-image> <el-image :src="result.expected"></el-image>
</el-card> </el-card>
...@@ -155,7 +173,7 @@ under the License. ...@@ -155,7 +173,7 @@ under the License.
<el-col :span="8"> <el-col :span="8">
<el-card shadow="hover"> <el-card shadow="hover">
<div slot="header" class="clearfix"> <div slot="header" class="clearfix">
<span>Actual</span> <span>Actual - {{currentTest.actualVersion || ''}}</span>
</div> </div>
<el-image :src="result.actual"></el-image> <el-image :src="result.actual"></el-image>
</el-card> </el-card>
...@@ -164,7 +182,7 @@ under the License. ...@@ -164,7 +182,7 @@ under the License.
<el-col :span="8"> <el-col :span="8">
<el-card shadow="hover"> <el-card shadow="hover">
<div slot="header" class="clearfix"> <div slot="header" class="clearfix">
<span>Diff({{result.diffRatio.toFixed(4)}})</span> <span>Diff - {{result.diffRatio.toFixed(4)}}</span>
</div> </div>
<el-image :src="result.diff" :preview-src-list="[result.diff]"></el-image> <el-image :src="result.diff" :preview-src-list="[result.diff]"></el-image>
</el-card> </el-card>
......
...@@ -25,7 +25,7 @@ const {fork} = require('child_process'); ...@@ -25,7 +25,7 @@ const {fork} = require('child_process');
const semver = require('semver'); const semver = require('semver');
const {port, origin} = require('./config'); const {port, origin} = require('./config');
const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults, updateActionsMeta} = require('./store'); const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults, updateActionsMeta} = require('./store');
const {prepareEChartsVersion, getActionsFullPath} = require('./util'); const {prepareEChartsLib, getActionsFullPath, fetchVersions} = require('./util');
const fse = require('fs-extra'); const fse = require('fs-extra');
const fs = require('fs'); const fs = require('fs');
...@@ -75,12 +75,16 @@ class Thread { ...@@ -75,12 +75,16 @@ class Thread {
this.onUpdate; this.onUpdate;
} }
fork(noHeadless, replaySpeed) { fork(noHeadless, replaySpeed, actualVersion, expectedVersion) {
let p = fork(path.join(__dirname, 'cli.js'), [ let p = fork(path.join(__dirname, 'cli.js'), [
'--tests', '--tests',
this.tests.map(testOpt => testOpt.name).join(','), this.tests.map(testOpt => testOpt.name).join(','),
'--speed', '--speed',
replaySpeed || 5, replaySpeed || 5,
'--actual',
actualVersion,
'--expected',
expectedVersion,
...(noHeadless ? ['--no-headless'] : []) ...(noHeadless ? ['--no-headless'] : [])
]); ]);
this.p = p; this.p = p;
...@@ -108,7 +112,9 @@ class Thread { ...@@ -108,7 +112,9 @@ class Thread {
function startTests(testsNameList, socket, { function startTests(testsNameList, socket, {
noHeadless, noHeadless,
threadsCount, threadsCount,
replaySpeed replaySpeed,
actualVersion,
expectedVersion
}) { }) {
console.log('Received: ', testsNameList.join(',')); console.log('Received: ', testsNameList.join(','));
...@@ -147,7 +153,7 @@ function startTests(testsNameList, socket, { ...@@ -147,7 +153,7 @@ function startTests(testsNameList, socket, {
for (let i = 0; i < threadsCount; i++) { for (let i = 0; i < threadsCount; i++) {
runningThreads[i].onExit = onExit; runningThreads[i].onExit = onExit;
runningThreads[i].onUpdate = onUpdate; runningThreads[i].onUpdate = onUpdate;
runningThreads[i].fork(noHeadless, replaySpeed); runningThreads[i].fork(noHeadless, replaySpeed, actualVersion, expectedVersion);
runningCount++; runningCount++;
} }
// If something bad happens and no proccess are started successfully // If something bad happens and no proccess are started successfully
...@@ -174,8 +180,7 @@ async function start() { ...@@ -174,8 +180,7 @@ async function start() {
return; return;
} }
await prepareEChartsVersion('4.2.1'); // Expected version. let versions = await fetchVersions();
await prepareEChartsVersion(); // Version to test
// let runtimeCode = await buildRuntimeCode(); // let runtimeCode = await buildRuntimeCode();
// fse.outputFileSync(path.join(__dirname, 'tmp/testRuntime.js'), runtimeCode, 'utf-8'); // fse.outputFileSync(path.join(__dirname, 'tmp/testRuntime.js'), runtimeCode, 'utf-8');
...@@ -189,7 +194,12 @@ async function start() { ...@@ -189,7 +194,12 @@ async function start() {
socket.emit('update', {tests: getTestsList()}); socket.emit('update', {tests: getTestsList()});
socket.on('run', async data => { socket.on('run', async data => {
let startTime = Date.now(); let startTime = Date.now();
await prepareEChartsLib(data.expectedVersion); // Expected version.
await prepareEChartsLib(data.actualVersion); // Version to test
// TODO Should broadcast to all sockets. // TODO Should broadcast to all sockets.
try { try {
await startTests( await startTests(
...@@ -198,7 +208,9 @@ async function start() { ...@@ -198,7 +208,9 @@ async function start() {
{ {
noHeadless: data.noHeadless, noHeadless: data.noHeadless,
threadsCount: data.threads, threadsCount: data.threads,
replaySpeed: data.replaySpeed replaySpeed: data.replaySpeed,
actualVersion: data.actualVersion,
expectedVersion: data.expectedVersion
} }
); );
} }
...@@ -213,6 +225,8 @@ async function start() { ...@@ -213,6 +225,8 @@ async function start() {
socket.on('stop', () => { socket.on('stop', () => {
stopRunningTests(); stopRunningTests();
}); });
socket.emit('versions', versions);
}); });
io.of('/recorder').on('connect', async socket => { io.of('/recorder').on('connect', async socket => {
...@@ -245,7 +259,9 @@ async function start() { ...@@ -245,7 +259,9 @@ async function start() {
await startTests([data.testName], socket, { await startTests([data.testName], socket, {
noHeadless: true, noHeadless: true,
threadsCount: 1, threadsCount: 1,
replaySpeed: 2 replaySpeed: 2,
actualVersion: data.actualVersion,
expectedVersion: data.expectedVersion
}); });
} }
catch (e) { console.error(e); } catch (e) { console.error(e); }
...@@ -264,7 +280,7 @@ async function start() { ...@@ -264,7 +280,7 @@ async function start() {
}); });
console.log(`Dashboard: ${origin}/test/runTest/client/index.html`); console.log(`Dashboard: ${origin}/test/runTest/client/index.html`);
console.log(`Interaction Recorder: ${origin}/test/runTest/recorder/index.html`); // console.log(`Interaction Recorder: ${origin}/test/runTest/recorder/index.html`);
// open(`${origin}/test/runTest/client/index.html`); // open(`${origin}/test/runTest/client/index.html`);
} }
......
...@@ -34,7 +34,7 @@ module.exports.fileNameFromTest = function (testName) { ...@@ -34,7 +34,7 @@ module.exports.fileNameFromTest = function (testName) {
}; };
function getVersionDir(version) { function getVersionDir(version) {
version = version || 'developing'; version = version || 'local';
return `tmp/__version__/${version}`; return `tmp/__version__/${version}`;
}; };
module.exports.getVersionDir = getVersionDir; module.exports.getVersionDir = getVersionDir;
...@@ -44,10 +44,10 @@ module.exports.getActionsFullPath = function (testName) { ...@@ -44,10 +44,10 @@ module.exports.getActionsFullPath = function (testName) {
}; };
module.exports.prepareEChartsVersion = function (version) { module.exports.prepareEChartsLib = function (version) {
let versionFolder = path.join(__dirname, getVersionDir(version)); let versionFolder = path.join(__dirname, getVersionDir(version));
fse.ensureDirSync(versionFolder); fse.ensureDirSync(versionFolder);
if (!version) { if (!version || version === 'local') {
// Developing version, make sure it's new build // Developing version, make sure it's new build
return fse.copy( return fse.copy(
path.join(__dirname, '../../dist/echarts.js'), path.join(__dirname, '../../dist/echarts.js'),
...@@ -58,7 +58,7 @@ module.exports.prepareEChartsVersion = function (version) { ...@@ -58,7 +58,7 @@ module.exports.prepareEChartsVersion = function (version) {
if (!fs.existsSync(`${versionFolder}/echarts.js`)) { if (!fs.existsSync(`${versionFolder}/echarts.js`)) {
const file = fs.createWriteStream(`${versionFolder}/echarts.js`); const file = fs.createWriteStream(`${versionFolder}/echarts.js`);
console.log('Downloading echarts4.2.1 from ', `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`); console.log(`Downloading echarts@${version} from `, `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`);
https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => { https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => {
response.pipe(file); response.pipe(file);
...@@ -73,6 +73,29 @@ module.exports.prepareEChartsVersion = function (version) { ...@@ -73,6 +73,29 @@ module.exports.prepareEChartsVersion = function (version) {
}); });
}; };
module.exports.fetchVersions = function () {
return new Promise((resolve, reject) => {
https.get(`https://registry.npmjs.org/echarts`, res => {
if (res.statusCode !== 200) {
res.destroy();
reject('Failed fetch versions from https://registry.npmjs.org/echarts');
return;
}
var buffers = [];
res.on('data', buffers.push.bind(buffers));
res.on('end', function () {
try {
var data = Buffer.concat(buffers);
resolve(Object.keys(JSON.parse(data).versions));
}
catch (e) {
reject(e.toString());
}
});
});
});
};
module.exports.buildRuntimeCode = async function () { module.exports.buildRuntimeCode = async function () {
const bundle = await rollup.rollup({ const bundle = await rollup.rollup({
input: path.join(__dirname, 'runtime/main.js'), input: path.join(__dirname, 'runtime/main.js'),
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册