This is an automated email from the ASF dual-hosted git repository. shenyi pushed a commit to branch test-autorun in repository https://gitbox.apache.org/repos/asf/incubator-echarts.git
commit e7eaaa93a231ee0aca9762d7437499495e18da4d Author: pissang <[email protected]> AuthorDate: Thu Aug 29 00:14:50 2019 +0800 test: Run the tests automatically for visual regression test. --- package.json | 12 +- test/runTest/blacklist.js | 4 + test/runTest/cli.js | 312 ++++++++++++++++++++++++++++++++++++++ test/runTest/client/client.css | 26 ++++ test/runTest/client/client.js | 35 +++++ test/runTest/client/index.html | 73 +++++++++ test/runTest/compareScreenshot.js | 41 +++++ test/runTest/runtime.js | 71 +++++++++ test/runTest/serve.js | 29 ++++ 9 files changed, 601 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 18ad5a2..fe0c6dc 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ }, "scripts": { "prepublish": "node build/build.js --prepublish", + "test:visual": "node build/build.js && node test/runTest/cli.js", "test": "node build/build.js" }, "dependencies": { @@ -31,10 +32,17 @@ "escodegen": "1.8.0", "esprima": "2.7.2", "estraverse": "4.1.1", - "fs-extra": "0.26.7", + "fs-extra": "^0.26.7", "glob": "7.0.0", + "open": "^6.4.0", + "pixelmatch": "^5.0.2", + "pngjs": "^3.4.0", + "puppeteer": "^1.19.0", "rollup": "0.50.0", "rollup-plugin-node-resolve": "3.0.0", - "rollup-plugin-uglify": "2.0.1" + "rollup-plugin-uglify": "2.0.1", + "seedrandom": "^3.0.3", + "serve-handler": "^6.1.1", + "slugify": "^1.3.4" } } diff --git a/test/runTest/blacklist.js b/test/runTest/blacklist.js new file mode 100644 index 0000000..e059607 --- /dev/null +++ b/test/runTest/blacklist.js @@ -0,0 +1,4 @@ +module.exports = [ +'-cases.html', +'geo-random-stream.html' +]; \ No newline at end of file diff --git a/test/runTest/cli.js b/test/runTest/cli.js new file mode 100644 index 0000000..b560072 --- /dev/null +++ b/test/runTest/cli.js @@ -0,0 +1,312 @@ +const puppeteer = require('puppeteer'); +const slugify = require('slugify'); +const fse = require('fs-extra'); +const fs = require('fs'); +const https = require('https'); +const path = require('path'); +const open = require('open'); +const util = require('util'); +const glob = require('glob'); +const {serve, origin} = require('./serve'); +const compareScreenshot = require('./compareScreenshot'); +const blacklist = require('./blacklist'); + +const seedrandomCode = fs.readFileSync( + path.join(__dirname, '../../node_modules/seedrandom/seedrandom.js'), + 'utf-8' +); +const runtimeCode = fs.readFileSync(path.join(__dirname, './runtime.js'), 'utf-8'); + +function getVersionDir(version) { + version = version || 'developing'; + return `tmp/__version__/${version}`; +} + +function getScreenshotDir() { + return 'tmp/__screenshot__'; +} + +function getTestName(fileUrl) { + return path.basename(fileUrl, '.html'); +} + +function getCacheFilePath() { + return path.join(__dirname, 'tmp/__cache__.json'); +} + +function sortScreenshots(list) { + return list.sort((a, b) => { + return a.testName.localeCompare(b.testName); + }); +} + +function getClientRelativePath(absPath) { + return path.join('../', path.relative(__dirname, absPath)); +} + +function replaceEChartsVersion(interceptedRequest, version) { + // TODO Extensions and maps + if (interceptedRequest.url().endsWith('dist/echarts.js')) { + interceptedRequest.continue({ + url: `${origin}/test/runTest/${getVersionDir(version)}/echarts.js` + }); + } + else { + interceptedRequest.continue(); + } +} + +function prepareEChartsVersion(version) { + let versionFolder = path.join(__dirname, getVersionDir(version)); + fse.ensureDirSync(versionFolder); + if (!version) { + // Developing version, make sure it's new build + return fse.copy( + path.join(__dirname, '../../dist/echarts.js'), + `${versionFolder}/echarts.js` + ); + } + return new Promise(resolve => { + if (!fs.existsSync(`${versionFolder}/echarts.js`)) { + const file = fs.createWriteStream(`${versionFolder}/echarts.js`); + + console.log('Downloading echarts4.2.1 from ', `https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`); + https.get(`https://cdn.jsdelivr.net/npm/echarts@${version}/dist/echarts.js`, response => { + response.pipe(file); + + file.on('finish', () => { + resolve(); + }) + }); + } + else { + resolve(); + } + }); +} + +function waitPageForFinish(page) { + return new Promise(resolve => { + page.exposeFunction('puppeteerFinishTest', () => { + resolve(); + }); + }); +} + +function createWaitTimeout(maxTime) { + let timeoutHandle; + let resolve; + function keepWait(newMaxTime) { + newMaxTime = newMaxTime == null ? maxTime : newMaxTime; + clearTimeout(timeoutHandle); + createTimeout(newMaxTime); + } + + function createTimeout(maxTime) { + timeoutHandle = setTimeout(() => { + resolve(); + }, maxTime); + } + + function waitTimeout() { + return new Promise(_resolve => { + resolve = _resolve; + createTimeout(maxTime); + }); + } + + return {keepWait, waitTimeout} +} + +async function takeScreenshot(page, elementQuery, fileUrl, desc, version) { + let target = elementQuery ? await page.$(elementQuery) : page; + if (!target) { + console.error(`Can't find element '${elementQuery}'`); + return; + } + let fullPage = !elementQuery; + let testName = getTestName(fileUrl); + if (desc) { + testName += '-' + slugify(desc, { replacement: '-', lower: true }) + } + let screenshotPrefix = version ? 'expected' : 'actual'; + let screenshotPath = path.join(__dirname, `${getScreenshotDir()}/${testName}-${screenshotPrefix}.png`); + await target.screenshot({ + path: screenshotPath, + fullPage + }); + + return {testName, screenshotPath}; +} + +async function runTestPage(browser, fileUrl, version) { + + const {keepWait, waitTimeout} = createWaitTimeout(3200); + const testResults = []; + let screenshotPromises = []; + + const page = await browser.newPage(); + page.setRequestInterception(true); + page.on('request', replaceEChartsVersion); + await page.evaluateOnNewDocument(seedrandomCode); + await page.evaluateOnNewDocument(runtimeCode); + + let descAutoCounter = 0; + + page.exposeFunction('puppeteerScreenshot', (desc, elementQuery) => { + keepWait(); + desc = desc || (descAutoCounter++).toString(); + let promise = takeScreenshot(page, elementQuery, fileUrl, desc, version).then((result) => { + if (!result) { + return; + } + const {testName, screenshotPath} = result; + testResults.push({testName, desc, screenshotPath}); + }); + screenshotPromises.push(promise); + + return promise; + }); + // page.on('console', msg => { + // console.log(msg.text()); + // }); + // page.on('pageerror', error => { + // console.error(error); + // }) + + let pageFinishPromise = waitPageForFinish(page); + + await page.goto(`${origin}/test/${fileUrl}`, { + waitUntil: 'networkidle2', + timeout: 10000 + }); + + // Do auto screenshot for every 1 second. + let count = 1; + let autoSnapshotInterval = setInterval(async () => { + let desc = `autogen-${count++}`; + let promise = takeScreenshot(page, '', fileUrl, desc, version) + .then(({testName, screenshotPath}) => { + testResults.push({testName, desc, screenshotPath}); + }); + screenshotPromises.push(promise); + }, 1000); + + + // Wait for puppeteerFinishTest() is called + // Or compare the whole page if nothing happened after 10 seconds. + await Promise.race([ + pageFinishPromise, + waitTimeout().then(() => { + // console.warn('Test timeout after 3 seconds.'); + }) + ]); + + clearInterval(autoSnapshotInterval); + + // Wait for screenshot finished. + await Promise.all(screenshotPromises); + + await page.close(); + + return testResults; +} + +async function runTest(browser, testOpt) { + const fileUrl = testOpt.fileUrl; + const expectedShots = await runTestPage(browser, fileUrl, '4.2.1'); + const actualShots = await runTestPage(browser, fileUrl); + + sortScreenshots(expectedShots); + sortScreenshots(actualShots); + + const results = []; + expectedShots.forEach(async (shot, idx) => { + let expected = shot; + let actual = actualShots[idx]; + let {diffRatio, diffPNG} = await compareScreenshot( + expected.screenshotPath, + actual.screenshotPath + ); + + + let diffPath = `${path.resolve(__dirname, getScreenshotDir())}/${shot.testName}-diff.png`; + diffPNG.pack().pipe(fs.createWriteStream(diffPath)); + + results.push({ + actual: getClientRelativePath(actual.screenshotPath), + expected: getClientRelativePath(expected.screenshotPath), + diff: getClientRelativePath(diffPath), + name: actual.testName, + diffRatio + }); + }); + + testOpt.results = results; + testOpt.status = 'finished'; +} + +function writeTestsToCache(tests) { + fs.writeFileSync(getCacheFilePath(), JSON.stringify(tests, null, 2), 'utf-8'); +} + +async function getTestsList() { + let tmpFolder = path.join(__dirname, 'tmp'); + fse.ensureDirSync(tmpFolder); + try { + let cachedStr = fs.readFileSync(getCacheFilePath(), 'utf-8'); + let tests = JSON.parse(cachedStr); + return tests; + } + catch(e) { + let files = await util.promisify(glob)('**.html', { cwd: path.resolve(__dirname, '../') }); + let tests = files.filter(fileUrl => { + return blacklist.includes(fileUrl); + }).map(fileUrl => { + return { + fileUrl, + name: getTestName(fileUrl), + status: 'pending', + results: [] + }; + }); + return tests; + } +} + +async function start() { + await prepareEChartsVersion('4.2.1'); // Expected version. + await prepareEChartsVersion(); // Version to test + + fse.ensureDirSync(path.join(__dirname, getScreenshotDir())); + // Start a static server for puppeteer open the html test cases. + let {broadcast, io} = serve(); + + open(`${origin}/test/runTest/client/index.html`); + + const browser = await puppeteer.launch({ /* headless: false */ }); + + const tests = await getTestsList(); + + broadcast({tests}); + + io.on('connect', socket => { + broadcast({tests}); + }); + + for (let testOpt of tests) { + if (testOpt.status === 'finished') { + continue; + } + console.log(`Running test ${testOpt.fileUrl}`) + await runTest(browser, testOpt); + broadcast({tests}); + writeTestsToCache(tests); + } + +} + +start().catch(e => { + console.log('Error during test'); + console.log(e); +}) \ No newline at end of file diff --git a/test/runTest/client/client.css b/test/runTest/client/client.css new file mode 100644 index 0000000..22ce528 --- /dev/null +++ b/test/runTest/client/client.css @@ -0,0 +1,26 @@ +#main { + position: absolute; + left: 0; + top: 0; + right: 0; + bottom: 0; +} + +* { + font-family: "Helvetica Neue",Helvetica,"PingFang SC","Hiragino Sans GB","Microsoft YaHei","微软雅黑",Arial,sans-serif; +} + +.menu-link { + display: block; + text-decoration: none; + font-size: 18px; +} + +.test-screenshots { + margin-top: 50px; +} + +.test-screenshots img { + /* height: 200px; */ + width: 100%; +} \ No newline at end of file diff --git a/test/runTest/client/client.js b/test/runTest/client/client.js new file mode 100644 index 0000000..5bc96b6 --- /dev/null +++ b/test/runTest/client/client.js @@ -0,0 +1,35 @@ +const socket = io(); +socket.on('connect', () => { + console.log('Connected'); + + const app = new Vue({ + el: '#app', + data: { + tests: [], + selectedTestName: '' + }, + computed: { + selectedTest() { + let selectedTest = this.tests.find(item => item.name === this.selectedTestName); + if (!selectedTest) { + selectedTest = this.tests[0]; + } + return selectedTest; + } + } + }); + app.$el.style.display = 'block'; + + socket.on('broadcast', msg => { + app.tests = msg.tests; + }); + + function updateTestHash() { + let testName = window.location.hash.slice(1); + app.selectedTestName = testName; + } + + updateTestHash(); + window.addEventListener('hashchange', updateTestHash); +}); + diff --git a/test/runTest/client/index.html b/test/runTest/client/index.html new file mode 100644 index 0000000..8bfdf3b --- /dev/null +++ b/test/runTest/client/index.html @@ -0,0 +1,73 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta http-equiv="X-UA-Compatible" content="ie=edge"> +</head> +<body> +<div id="app" style="display: none"> + <el-container id="main"> + <el-header> + <h1>Visual Regression Test</h1> + </el-header> + <el-container style="min-height: 0"> <!-- https://juejin.im/post/5c642f2ff265da2de660ecfc --> + <el-aside width="250px"> + <el-menu class="test-list"> + <el-menu-item v-for="test in tests"> + <a :href="'#' + test.name" class="menu-link">{{test.name}}</a> + </el-menu-item> + </el-menu> + </el-aside> + <el-main> + <div v-if="selectedTest"> + <div class="test-screenshots" v-for="result in selectedTest.results"> + <h4 class="md-title">{{result.name}}</h4> + <el-row :gutter="20"> + <el-col :span="8"> + <el-card> + <div slot="header" class="clearfix"> + <span>Expected</span> + </div> + <el-image :src="result.expected" :preview-src-list="[result.expected]"></el-image> + </el-card> + </el-col> + + <el-col :span="8"> + <el-card> + <div slot="header" class="clearfix"> + <span>Actual</span> + </div> + <el-image :src="result.actual" :preview-src-list="[result.actual]"></el-image> + </el-card> + </el-col> + + <el-col :span="8"> + <el-card> + <div slot="header" class="clearfix"> + <span>Diff({{result.diffRatio.toFixed(4)}})</span> + </div> + <el-image :src="result.diff" :preview-src-list="[result.diff]"></el-image> + </el-card> + </el-col> + </el-row> + </div> + </div> + </el-main> + </el-container> + </el-container> +</div> + +<script src="../../../node_modules/socket.io-client/dist/socket.io.js"></script> +<script src="https://unpkg.com/[email protected]/dist/vue.js";></script> + +<!-- Element UI --> +<link rel="stylesheet" href="https://unpkg.com/[email protected]/lib/theme-chalk/index.css";> +<script src="https://unpkg.com/[email protected]/lib/index.js";></script> + +<script src="client.js"></script> + +<link rel="stylesheet" href="client.css"> + +</body> +</html> \ No newline at end of file diff --git a/test/runTest/compareScreenshot.js b/test/runTest/compareScreenshot.js new file mode 100644 index 0000000..339ab56 --- /dev/null +++ b/test/runTest/compareScreenshot.js @@ -0,0 +1,41 @@ +const PNG = require('pngjs').PNG; +const pixelmatch = require('pixelmatch'); +const fs = require('fs'); + +function readPNG(path) { + return new Promise(resolve => { + fs.createReadStream(path) + .pipe(new PNG()) + .on('parsed', function () { + resolve({ + data: this.data, + width: this.width, + height: this.height + }); + }); + }); +} + +module.exports = function (expectedShotPath, actualShotPath, threshold = 0.1) { + return Promise.all([ + readPNG(expectedShotPath), + readPNG(actualShotPath) + ]).then(([expectedImg, actualImg]) => { + let width = expectedImg.width; + let height = expectedImg.height; + if ( + (width !== actualImg.width) + || (height !== actualImg.height) + ) { + throw new Error('Image size not match'); + } + const diffPNG = new PNG({width, height}); + let diffPixelsCount = pixelmatch(expectedImg.data, actualImg.data, diffPNG.data, width, height, {threshold}); + let totalPixelsCount = width * height; + + return { + diffRatio: diffPixelsCount / totalPixelsCount, + diffPNG + }; + }); +}; \ No newline at end of file diff --git a/test/runTest/runtime.js b/test/runTest/runtime.js new file mode 100644 index 0000000..1a17c6a --- /dev/null +++ b/test/runTest/runtime.js @@ -0,0 +1,71 @@ +(function (global) { + if (global.autorun) { + return; + } + + var autorun = {}; + var inPuppeteer = typeof puppeteerScreenshot !== 'undefined'; + + var NativeDate = window.Date; + + var fixedTimestamp = 1566458693300; + var actualTimestamp = NativeDate.now(); + + function MockDate(params) { + if (!params) { + var elapsedTime = NativeDate.now() - actualTimestamp; + return new NativeDate(fixedTimestamp + elapsedTime); + } + else { + return new NativeDate(params); + } + } + MockDate.prototype = new Date(); + + autorun.createScreenshotTest = function (desc, elementQuery, waitTime) { + + } + + /** + * Take screenshot immediately. + * @param {string} desc + * @param {string} [elementQuery] If only screenshot specifed element. Will do full page screenshot if it's not give. + */ + autorun.compareScreenshot = function (desc, elementQuery) { + if (!inPuppeteer) { + return Promise.resolve(); + } + + return puppeteerScreenshot(desc, elementQuery); + }; + + autorun.shouldBe = function (expected, actual) { + + }; + + /** + * Finish the test. + */ + autorun.finish = function () { + if (!inPuppeteer) { + return Promise.resolve(); + } + return puppeteerFinishTest(); + }; + + + if (inPuppeteer) { + let myRandom = new Math.seedrandom('echarts-test'); + // Fixed random generator + Math.random = function () { + var val = myRandom(); + return val; + }; + + // Fixed date + window.Date = MockDate; + } + + global.autorun = autorun; + +})(window); \ No newline at end of file diff --git a/test/runTest/serve.js b/test/runTest/serve.js new file mode 100644 index 0000000..42f3ffa --- /dev/null +++ b/test/runTest/serve.js @@ -0,0 +1,29 @@ +const handler = require('serve-handler'); +const http = require('http'); +const port = 8866; +const origin = `http://localhost:${port}`; + +function serve() { + const server = http.createServer((request, response) => { + return handler(request, response, { + cleanUrls: false, + // Root folder of echarts + public: __dirname + '/../../' + }); + }); + + server.listen(port, () => { + console.log(`Server started. ${origin}`); + }); + + + const io = require('socket.io')(server); + return { + broadcast(data) { + io.emit('broadcast', data); + }, + io + } +} + +module.exports = {serve, origin}; \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
