This is an automated email from the ASF dual-hosted git repository.
shenyi pushed a commit to branch enhance-visual-regression-test
in repository https://gitbox.apache.org/repos/asf/echarts.git
The following commit(s) were added to refs/heads/enhance-visual-regression-test
by this push:
new c7f9a4b test(visual): add test runs manager
c7f9a4b is described below
commit c7f9a4beed50849f5e9c56f8f1bc53eb2e9ea125
Author: pissang <[email protected]>
AuthorDate: Fri Apr 30 13:49:05 2021 +0800
test(visual): add test runs manager
---
test/runTest/cli.js | 4 ++
test/runTest/client/client.css | 5 ++
test/runTest/client/client.js | 36 +++++++++++++++
test/runTest/client/index.html | 40 +++++++++++++++-
test/runTest/server.js | 36 ++++++++++-----
test/runTest/store.js | 101 +++++++++++++++++++++++++++++++++++++----
6 files changed, 199 insertions(+), 23 deletions(-)
diff --git a/test/runTest/cli.js b/test/runTest/cli.js
index f6f5ed4..59b9bf7 100644
--- a/test/runTest/cli.js
+++ b/test/runTest/cli.js
@@ -327,6 +327,10 @@ async function runTests(pendingTests) {
let runtimeCode = await buildRuntimeCode();
runtimeCode = `window.__TEST_PLAYBACK_SPEED__ = ${program.speed ||
1};\n${runtimeCode}`;
+ process.on('exit', () => {
+ browser.close();
+ });
+
try {
for (let testOpt of pendingTests) {
console.log(`Running test: ${testOpt.name}, renderer:
${program.renderer}`);
diff --git a/test/runTest/client/client.css b/test/runTest/client/client.css
index 22d37f6..f9926ab 100644
--- a/test/runTest/client/client.css
+++ b/test/runTest/client/client.css
@@ -267,6 +267,11 @@ iframe {
overflow: overlay;
}
+#tests-runs-dialog .el-dialog {
+ width: 90%;
+ max-width: 1400px;
+}
+
::-webkit-scrollbar {
height: 8px;
diff --git a/test/runTest/client/client.js b/test/runTest/client/client.js
index 2db45fd..a9dd0e5 100644
--- a/test/runTest/client/client.js
+++ b/test/runTest/client/client.js
@@ -109,6 +109,10 @@ const app = new Vue({
previewIframeSrc: '',
previewTitle: '',
+ // List of all runs.
+ showRunsDialog: false,
+ testsRuns: [],
+
runConfig: Object.assign({
sortBy: 'name',
@@ -336,6 +340,34 @@ const app = new Vue({
}
}
});
+ },
+
+ showAllTestsRuns() {
+ this.showRunsDialog = true;
+ socket.emit('getAllTestsRuns');
+ },
+
+ switchTestsRun(runResult) {
+
+ },
+ genTestsRunReport(runResult) {
+
+ },
+
+ delTestsRun(runResult) {
+ app.$confirm('Are you sure to delete this run?', 'Warn', {
+ confirmButtonText: 'Yes',
+ cancelButtonText: 'No',
+ center: true
+ }).then(value => {
+ const idx = this.testsRuns.indexOf(runResult);
+ if (idx >= 0) {
+ this.testsRuns.splice(idx, 1);
+ }
+ socket.emit('delTestsRun', {
+ id: runResult.id
+ });
+ }).catch(() => {});
}
}
});
@@ -415,6 +447,10 @@ socket.on('abort', res => {
app.running = false;
});
+socket.on('getAllTestsRuns_return', res => {
+ app.testsRuns = res.runs;
+});
+
function updateUrl(notRefresh) {
const searchUrl = assembleParams({
test: app.currentTestName,
diff --git a/test/runTest/client/index.html b/test/runTest/client/index.html
index 72e6536..6205407 100644
--- a/test/runTest/client/index.html
+++ b/test/runTest/client/index.html
@@ -37,6 +37,7 @@ under the License.
</div>
</el-header>
<el-container style="min-height: 0"> <!--
https://juejin.im/post/5c642f2ff265da2de660ecfc -->
+ <!-------- Tests List ----------->
<el-aside width="350px">
<div class="nav-toolbar">
<el-row class="filters" :gutter="10" :align="'middle'">
@@ -119,6 +120,16 @@ under the License.
</el-button-group>
</div>
+ <div class="run-config-item">
+ <el-tooltip content="Show All Tests Runs">
+ <i
+ style="font-size: 20px; cursor: pointer;"
+ class="el-icon-files"
+ @click="showAllTestsRuns"
+ ></i>
+ </el-tooltip>
+ </div>
+
<!-- <div class="run-config-item">
<el-checkbox
v-model="runConfig.noHeadless">Replay</el-checkbox>
@@ -169,7 +180,7 @@ under the License.
</div>
</div>
-
+ <!-------- Single Test Reusult ----------->
<div v-if="currentTest" class="test-result">
<div class="title">
<el-progress
@@ -268,6 +279,33 @@ under the License.
</div>
<iframe :src="previewIframeSrc" width="800"
height="600"></iframe>
</el-dialog>
+
+ <!-------- All Tests Runs ----------->
+ <el-dialog
+ id="tests-runs-dialog"
+ :visible.sync="showRunsDialog"
+ title="All Tests Runs"
+ >
+ <el-table :data="testsRuns">
+ <el-table-column property="expectedVersion"
label="Expected"></el-table-column>
+ <el-table-column property="actualVersion"
label="Actual"></el-table-column>
+ <el-table-column property="renderer" label="Renderer"
width="100"></el-table-column>
+ <el-table-column property="lastRunTime"
label="lastRunTime"></el-table-column>
+ <el-table-column property="total" label="Total Tests"
width="100"></el-table-column>
+ <el-table-column property="finished" label="Finished"
width="100"></el-table-column>
+ <el-table-column property="passed" label="Passed"
width="100"></el-table-column>
+ <el-table-column
+ fixed="right"
+ label=""
+ width="160">
+ <template slot-scope="scope">
+ <el-button type="text"
@click="switchTestsRun(scope.row)" size="small">View</el-button>
+ <el-button type="text"
@click="genTestsRunReport(scope.row)" size="small">Report</el-button>
+ <el-button type="text"
@click="delTestsRun(scope.row)" size="small">Delete</el-button>
+ </template>
+ </el-table-column>
+ </el-table>
+ </el-dialog>
</el-main>
</el-container>
</el-container>
diff --git a/test/runTest/server.js b/test/runTest/server.js
index 4f78187..a73afd3 100644
--- a/test/runTest/server.js
+++ b/test/runTest/server.js
@@ -24,14 +24,22 @@ const path = require('path');
const {fork} = require('child_process');
const semver = require('semver');
const {port, origin} = require('./config');
-const {getTestsList, updateTestsList, saveTestsList, mergeTestsResults,
updateActionsMeta, getResultBaseDir} = require('./store');
+const {
+ getTestsList,
+ updateTestsList,
+ saveTestsList,
+ mergeTestsResults,
+ updateActionsMeta,
+ getResultBaseDir,
+ getRunHash,
+ getAllTestsRuns,
+ delTestsRun
+} = require('./store');
const {prepareEChartsLib, getActionsFullPath} = require('./util');
const fse = require('fs-extra');
const fs = require('fs');
const open = require('open');
-const TEST_HASH_SPLITTER = '__';
-
function serve() {
const server = http.createServer((request, response) => {
return handler(request, response, {
@@ -199,13 +207,6 @@ function checkPuppeteer() {
}
}
-function getTestHash(params) {
- return [
- params.expectedVersion,
- params.actualVersion,
- params.renderer
- ].join(TEST_HASH_SPLITTER);
-}
async function start() {
if (!checkPuppeteer()) {
@@ -234,12 +235,12 @@ async function start() {
}
socket.on('setTestVersions', async (params) => {
- if (_currentTestHash !== getTestHash(params)) {
+ if (_currentTestHash !== getRunHash(params)) {
abortTests();
}
await updateTestsList(
- _currentTestHash = getTestHash(params),
+ _currentTestHash = getRunHash(params),
!running // Set to unsettled if not running
);
@@ -249,6 +250,17 @@ async function start() {
});
});
+ socket.on('getAllTestsRuns', async () => {
+ socket.emit('getAllTestsRuns_return', {
+ runs: await getAllTestsRuns()
+ });
+ });
+
+ socket.on('delTestsRun', async (params) => {
+ delTestsRun(params.id);
+ console.log('Deleted', params.id);
+ });
+
socket.on('run', async data => {
let startTime = Date.now();
diff --git a/test/runTest/store.js b/test/runTest/store.js
index 8da5852..a6135bb 100644
--- a/test/runTest/store.js
+++ b/test/runTest/store.js
@@ -20,14 +20,18 @@
const path = require('path');
const fse = require('fs-extra');
const fs = require('fs');
-const glob = require('glob');
+const globby = require('globby');
const {testNameFromFile} = require('./util');
-const util = require('util');
const {blacklist, SVGBlacklist} = require('./blacklist');
let _tests = [];
let _testsMap = {};
-let _testHash = '';
+let _runHash = '';
+
+const RESULT_FILE_NAME = '__result__.json';
+const RESULTS_ROOT_DIR = path.join(__dirname, 'tmp', 'result');
+
+const TEST_HASH_SPLITTER = '__';
class Test {
constructor(fileUrl) {
@@ -59,14 +63,40 @@ class Test {
}
}
+
+/**
+ * hash of each run is mainly for storing the results.
+ * It depends on two versions and rendering mode.
+ */
+function getRunHash(params) {
+ return [
+ params.expectedVersion,
+ params.actualVersion,
+ params.renderer
+ ].join(TEST_HASH_SPLITTER);
+}
+
+/**
+ * Parse versions and rendering mode from run hash.
+ */
+function parseRunHash(str) {
+ const parts = str.split(TEST_HASH_SPLITTER);
+ return {
+ expectedVersion: parts[0],
+ actualVersion: parts[1],
+ renderer: parts[2]
+ };
+}
+
function getResultBaseDir() {
- return path.join(__dirname, 'tmp', 'result', _testHash);
+ return path.join(RESULTS_ROOT_DIR, _runHash);
}
module.exports.getResultBaseDir = getResultBaseDir;
+module.exports.getRunHash = getRunHash;
-function getCacheFilePath(baseDir) {
- return path.join(getResultBaseDir(), '__result__.json');;
+function getCacheFilePath() {
+ return path.join(getResultBaseDir(), RESULT_FILE_NAME);
}
module.exports.getTestsList = function () {
@@ -78,10 +108,10 @@ module.exports.getTestByFileUrl = function (url) {
};
module.exports.updateTestsList = async function (
- testHash,
+ runHash,
setPendingTestToUnsettled
) {
- _testHash = testHash;
+ _runHash = runHash;
_tests = [];
_testsMap = {};
_testsExists = {};
@@ -105,7 +135,7 @@ module.exports.updateTestsList = async function (
catch(e) {
}
// Find if there is new html file
- const files = await util.promisify(glob)('**.html', { cwd:
path.resolve(__dirname, '../') });
+ const files = await globby('**.html', { cwd: path.resolve(__dirname,
'../') });
files.forEach(fileUrl => {
if (blacklist.includes(fileUrl)) {
return;
@@ -169,4 +199,55 @@ module.exports.updateActionsMeta = function (testName,
actions) {
fs.writeFileSync(metaPath, JSON.stringify(
metaData, Object.keys(metaData).sort((a, b) => a.localeCompare(b)), 2
), 'utf-8');
-};
\ No newline at end of file
+};
+
+/**
+ * Get results of all runs
+ * @return [ { id, expectedVersion, actualVersion, renderer, lastRunTime,
total, finished, passed } ]
+ */
+module.exports.getAllTestsRuns = async function () {
+ const dirs = await globby('*', { cwd: RESULTS_ROOT_DIR, onlyDirectories:
true });
+ return dirs.map((dir) => {
+ const params = parseRunHash(dir);
+ const resultJson = JSON.parse(fs.readFileSync(path.join(
+ RESULTS_ROOT_DIR,
+ dir,
+ RESULT_FILE_NAME
+ ), 'utf-8'));
+
+ const total = resultJson.length;
+ let lastRunTime = 0;
+ let finishedCount = 0;
+ let passedCount = 0;
+
+ resultJson.forEach(test => {
+ lastRunTime = Math.max(test.lastRun, lastRunTime);
+ if (test.status === 'finished') {
+ finishedCount++;
+
+ let passed = true;
+ test.results.forEach(result => {
+ // Threshold?
+ if (result.diffRatio > 0.0001) {
+ passed = false;
+ }
+ });
+ if (passed) {
+ passedCount++;
+ }
+ }
+ });
+
+ params.lastRunTime = lastRunTime > 0 ? new
Date(lastRunTime).toISOString() : 'N/A';
+ params.total = total;
+ params.passed = passedCount;
+ params.finished = finishedCount;
+ params.id = dir;
+
+ return params;
+ });
+}
+
+module.exports.delTestsRun = async function (hash) {
+ fse.removeSync(path.join(RESULTS_ROOT_DIR, hash));
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
