Merge pull request #11229 from apache/test-autorun

Automatic visual regression testing tool: add preview dialog

.github/CONTRIBUTING.md

@@ -107,7 +107,7 @@ Then, open the test cases under `~/workspace/echarts/test` in Web browser. You c
 
 ```bash
 # puppeteer is not in the devDependencies and needs to be installed manually
-npm install puppeteer
+npm install puppeteer --no-save
 npm run test:visual
 ```
 

test/lib/config.js

@@ -25,22 +25,32 @@
     // `true` by default for debugging.
     sourceMap == null && (sourceMap = true);
 
+    var params = {};
+    location.search.slice(1).split('&').forEach(item => {
+        var kv = item.split('=');
+        params[kv[0]] = kv[1];
+    });
     // Set default renderer in dev mode from hash.
-    var matchResult = location.href.match(/[?&]__RENDERER__=(canvas|svg)(&|$)/);
-    if (matchResult) {
-        window.__ECHARTS__DEFAULT__RENDERER__ = matchResult[1];
+    if (params.__RENDERER__) {
+        window.__ECHARTS__DEFAULT__RENDERER__ = params.__RENDERER__;
     }
 
     // Set echarts source code.
-    var matchResult = location.href.match(/[?&]__ECDIST__=(webpack-req-ec|webpack-req-eclibec|webpackold-req-ec|webpackold-req-eclibec)(&|$)/);
-    var ecDistPath = 'dist/echarts';
-    if (matchResult) {
+    var ecDistPath;
+    if (params.__ECDIST__) {
         ecDistPath = ({
             'webpack-req-ec': '../echarts-boilerplate/echarts-webpack/dist/webpack-req-ec',
             'webpack-req-eclibec': '../echarts-boilerplate/echarts-webpack/dist/webpack-req-eclibec',
             'webpackold-req-ec': '../echarts-boilerplate/echarts-webpackold/dist/webpackold-req-ec',
             'webpackold-req-eclibec': '../echarts-boilerplate/echarts-webpackold/dist/webpackold-req-eclibec',
-        })[matchResult[1]];
+        })[params.__ECDIST__];
+        if (!ecDistPath && params.__ECDIST__.match(/[0-9.]/)) {
+            // Version number
+            ecDistPath = 'test/runTest/tmp/__version__/' + params.__ECDIST__ + '/echarts';
+        }
+    }
+    if (!ecDistPath) {
+        ecDistPath = 'dist/echarts';
     }
 
     if (typeof require !== 'undefined') {

test/runTest/blacklist.js

@@ -31,6 +31,9 @@ module.exports.blacklist = [
     'scatter-gps.html',
     'webkit-dep.html',
 
+    // Image size not match
+    'symbol2.html',
+
     // This case will have timeout
     'visualMap-performance1.html',
     'lines-bus.html',

test/runTest/cli.js

@@ -162,7 +162,7 @@ async function runTestPage(browser, testOpt, version, runtimeCode, isExpected) {
             timeout: 10000
         });
 
-        await waitTime(200);  // Wait for animation or something else. Pending
+        await waitTime(500);  // Wait for animation or something else. Pending
         // Final shot.
         await page.mouse.move(0, 0);
         let desc = 'Full Shot';

test/runTest/client/client.css

@@ -170,6 +170,14 @@
     margin-top: 60px;
 }
 
+.test-screenshots .preview {
+    cursor: pointer;
+    color: #409eff;
+}
+.test-screenshots .preview:hover {
+    text-decoration: underline;
+}
+
 .test-screenshots img {
     /* height: 200px; */
     width: 100%;
@@ -196,6 +204,11 @@
     color: #f56c6c
 }
 
+iframe {
+    border: none;
+    overflow: overlay;
+}
+
 
 ::-webkit-scrollbar {
     height: 8px;

test/runTest/client/client.js

@@ -70,6 +70,10 @@ const app = new Vue({
 
         versions: [],
 
+        showIframeDialog: false,
+        previewIframeSrc: '',
+        previewTitle: '',
+
         runConfig: {
             noHeadless: false,
             replaySpeed: 5,
@@ -207,6 +211,25 @@ const app = new Vue({
         stopTests() {
             this.running = false;
             socket.emit('stop');
+        },
+
+        preview(test, version) {
+            let searches = [];
+
+            let ecVersion = test[version + 'Version'];
+            if (ecVersion !== 'local') {
+                searches.push('__ECDIST__=' + ecVersion);
+            }
+            if (test.useSVG) {
+                searches.push('__RENDERER__=svg');
+            }
+            let src = test.fileUrl;
+            if (searches.length) {
+                src = src + '?' + searches.join('&');
+            }
+            this.previewIframeSrc = `../../${src}`;
+            this.previewTitle = src;
+            this.showIframeDialog = true;
         }
     }
 });

test/runTest/client/index.html

@@ -175,8 +175,9 @@ under the License.
                                 <el-card shadow="hover">
                                     <div slot="header" class="clearfix">
                                         <span>Expected - {{currentTest.expectedVersion || ''}}</span>
+                                        <i title="Preview" class="el-icon-view preview" @click="preview(currentTest, 'expected')"></i>
                                     </div>
-                                    <el-image :src="result.expected"></el-image>
+                                    <el-image :src="result.expected" :preview-src-list="[result.expected]"></el-image>
                                 </el-card>
                             </el-col>
 
@@ -184,8 +185,9 @@ under the License.
                                 <el-card shadow="hover">
                                     <div slot="header" class="clearfix">
                                         <span>Actual - {{currentTest.actualVersion || ''}}</span>
+                                        <i title="Preview" class="el-icon-view preview" @click="preview(currentTest, 'actual')"></i>
                                     </div>
-                                    <el-image :src="result.actual"></el-image>
+                                    <el-image :src="result.actual" :preview-src-list="[result.actual]"></el-image>
                                 </el-card>
                             </el-col>
 
@@ -194,7 +196,7 @@ under the License.
                                     <div slot="header" class="clearfix">
                                         <span>Diff - {{result.diffRatio.toFixed(4)}}</span>
                                     </div>
-                                    <el-image :src="result.diff"></el-image>
+                                    <el-image :src="result.diff" :preview-src-list="[result.diff]"></el-image>
                                 </el-card>
                             </el-col>
                         </el-row>
@@ -203,19 +205,11 @@ under the License.
                     <div class="test-errors">
                         <el-row :gutter="40">
                             <el-col :span="12">
-                                <el-alert
-                                    title="Expected Errors"
-                                    type="error"
-                                    show-icon>
-                                </el-alert>
+                                <el-alert title="Expected Errors" type="error" show-icon></el-alert>
                                 <div class="error-item" v-for="error in currentTest.expectedErrors">{{error}}</div>
                             </el-col>
                             <el-col :span="12">
-                                <el-alert
-                                    title="Actual Errors"
-                                    type="error"
-                                    show-icon>
-                                </el-alert>
+                                <el-alert title="Actual Errors" type="error" show-icon> </el-alert>
                                 <div class="error-item"  v-for="error in currentTest.actualErrors">{{error}}</div>
                             </el-col>
                         </el-row>
@@ -224,28 +218,29 @@ under the License.
                     <div class="test-logs">
                         <el-row :gutter="40">
                             <el-col :span="12">
-                                <el-alert
-                                    title="Expected Logs"
-                                    type="info"
-                                    show-icon>
-                                </el-alert>
+                                <el-alert title="Expected Logs" type="info" show-icon> </el-alert>
                                 <div class="log-item"  v-for="log in currentTest.expectedLogs">{{log}}</div>
                             </el-col>
                             <el-col :span="12">
-                                <el-alert
-                                    title="Actual Logs"
-                                    type="info"
-                                    show-icon>
+                                <el-alert title="Actual Logs" type="info" show-icon>
                                 </el-alert>
                                 <div class="log-item"  v-for="log in currentTest.actualLogs">{{log}}</div>
                             </el-col>
                         </el-row>
                     </div>
+                </div>
 
-                    <div class="test-result-nav">
-
+                <el-dialog
+                    :visible.sync="showIframeDialog"
+                    :center="true"
+                    width="850px"
+                >
+                    <div slot="title">
+                        {{previewTitle}}
+                        <a target="_blank" :href="'../../' + previewTitle"><i class="el-icon-link"></i>Open in New Window</a>
                     </div>
-                </div>
+                    <iframe :src="previewIframeSrc" width="800" height="600"></iframe>
+                </el-dialog>
             </el-main>
         </el-container>
     </el-container>

test/runTest/server.js

@@ -184,7 +184,7 @@ function checkPuppeteer() {
 async function start() {
     if (!checkPuppeteer()) {
         // TODO Check version.
-        console.error(`Can't find puppeteer >= 1.19.0, use 'npm install puppeteer' to install or update`);
+        console.error(`Can't find puppeteer >= 1.19.0, use 'npm install puppeteer --no-save' to install or update`);
         return;
     }