Log Message
Fix various bugs in the new syncing script https://bugs.webkit.org/show_bug.cgi?id=156393
Reviewed by Darin Adler. * server-tests/resources/common-operations.js: Added. This file was supposed to be added in r199191. (addBuilderForReport): (addSlaveForReport): (connectToDatabaseInEveryTest): (submitReport): * tools/js/buildbot-triggerable.js: (BuildbotTriggerable.prototype._pullBuildbotOnAllSyncers): Don't log every time we pull from buildbot builder as this dramatically increases the amount of log we generate. * tools/js/parse-arguments.js: (parseArguments): Fixed a typo. This should be parseArgument*s*, not parseArgument. * tools/js/remote.js: (RemoteAPI.prototype.url): Fixed a bug that portSuffix wasn't being expanded in the template literal. (RemoteAPI.prototype.configure): Added more validations with nice error messages. (RemoteAPI.prototype.sendHttpRequest): Falling back to port 80 isn't right when scheme is https. Compute the right port in configure instead based on the scheme. * tools/sync-buildbot.js: (syncLoop): Fixed the bug that syncing multiple times fail because Manifest.fetch() create new Platform and Test objects. This results in various references in BuildRequest objects to get outdated. Fixing this properly in Manifest.fetch() because we do need to "forget" about some tests and platforms in some cases. For now, delete all v3 model objects and start over in each syncing cycle. * unit-tests/tools-js-remote-tests.js: Added. Unit tests for the aforementioned changes to RemoteAPI.
Modified Paths
- trunk/Websites/perf.webkit.org/ChangeLog
- trunk/Websites/perf.webkit.org/tools/js/buildbot-triggerable.js
- trunk/Websites/perf.webkit.org/tools/js/parse-arguments.js
- trunk/Websites/perf.webkit.org/tools/js/remote.js
- trunk/Websites/perf.webkit.org/tools/sync-buildbot.js
Added Paths
Diff
Modified: trunk/Websites/perf.webkit.org/ChangeLog (199219 => 199220)
--- trunk/Websites/perf.webkit.org/ChangeLog 2016-04-08 07:13:28 UTC (rev 199219)
+++ trunk/Websites/perf.webkit.org/ChangeLog 2016-04-08 07:15:43 UTC (rev 199220)
@@ -1,3 +1,32 @@
+2016-04-08 Ryosuke Niwa <[email protected]>
+
+ Fix various bugs in the new syncing script
+ https://bugs.webkit.org/show_bug.cgi?id=156393
+
+ Reviewed by Darin Adler.
+
+ * server-tests/resources/common-operations.js: Added. This file was supposed to be added in r199191.
+ (addBuilderForReport):
+ (addSlaveForReport):
+ (connectToDatabaseInEveryTest):
+ (submitReport):
+ * tools/js/buildbot-triggerable.js:
+ (BuildbotTriggerable.prototype._pullBuildbotOnAllSyncers): Don't log every time we pull from buildbot
+ builder as this dramatically increases the amount of log we generate.
+ * tools/js/parse-arguments.js:
+ (parseArguments): Fixed a typo. This should be parseArgument*s*, not parseArgument.
+ * tools/js/remote.js:
+ (RemoteAPI.prototype.url): Fixed a bug that portSuffix wasn't being expanded in the template literal.
+ (RemoteAPI.prototype.configure): Added more validations with nice error messages.
+ (RemoteAPI.prototype.sendHttpRequest): Falling back to port 80 isn't right when scheme is https. Compute
+ the right port in configure instead based on the scheme.
+ * tools/sync-buildbot.js:
+ (syncLoop): Fixed the bug that syncing multiple times fail because Manifest.fetch() create new Platform
+ and Test objects. This results in various references in BuildRequest objects to get outdated. Fixing this
+ properly in Manifest.fetch() because we do need to "forget" about some tests and platforms in some cases.
+ For now, delete all v3 model objects and start over in each syncing cycle.
+ * unit-tests/tools-js-remote-tests.js: Added. Unit tests for the aforementioned changes to RemoteAPI.
+
2016-04-07 Ryosuke Niwa <[email protected]>
sync-buildbot.js doesn't mark disappeared builds as failed
Added: trunk/Websites/perf.webkit.org/server-tests/resources/common-operations.js (0 => 199220)
--- trunk/Websites/perf.webkit.org/server-tests/resources/common-operations.js (rev 0)
+++ trunk/Websites/perf.webkit.org/server-tests/resources/common-operations.js 2016-04-08 07:15:43 UTC (rev 199220)
@@ -0,0 +1,46 @@
+
+const crypto = require('crypto');
+
+function addBuilderForReport(report)
+{
+ return TestServer.database().insert('builders', {
+ name: report.builderName,
+ password_hash: crypto.createHash('sha256').update(report.builderPassword).digest('hex')
+ });
+}
+
+function addSlaveForReport(report)
+{
+ return TestServer.database().insert('build_slaves', {
+ name: report.slaveName,
+ password_hash: crypto.createHash('sha256').update(report.slavePassword).digest('hex')
+ });
+}
+
+function connectToDatabaseInEveryTest()
+{
+ beforeEach(function () {
+ TestServer.database().connect({keepAlive: true});
+ });
+
+ afterEach(function () {
+ TestServer.database().disconnect();
+ });
+}
+
+function submitReport(report)
+{
+ return TestServer.database().insert('builders', {
+ name: report[0].builderName,
+ password_hash: crypto.createHash('sha256').update(report[0].builderPassword).digest('hex')
+ }).then(function () {
+ return TestServer.remoteAPI().postJSON('/api/report/', report);
+ });
+}
+
+if (typeof module != 'undefined') {
+ module.exports.addBuilderForReport = addBuilderForReport;
+ module.exports.addSlaveForReport = addSlaveForReport;
+ module.exports.connectToDatabaseInEveryTest = connectToDatabaseInEveryTest;
+ module.exports.submitReport = submitReport;
+}
Modified: trunk/Websites/perf.webkit.org/tools/js/buildbot-triggerable.js (199219 => 199220)
--- trunk/Websites/perf.webkit.org/tools/js/buildbot-triggerable.js 2016-04-08 07:13:28 UTC (rev 199219)
+++ trunk/Websites/perf.webkit.org/tools/js/buildbot-triggerable.js 2016-04-08 07:15:43 UTC (rev 199220)
@@ -83,7 +83,6 @@
let associatedRequests = new Set;
let self = this;
return Promise.all(this._syncers.map(function (syncer) {
- self._logger.log(`Fetching builds on ${syncer.builderName()}`);
return syncer.pullBuildbot(self._lookbackCount).then(function (entryList) {
for (let entry of entryList) {
let request = BuildRequest.findById(entry.buildRequestId());
Modified: trunk/Websites/perf.webkit.org/tools/js/parse-arguments.js (199219 => 199220)
--- trunk/Websites/perf.webkit.org/tools/js/parse-arguments.js 2016-04-08 07:13:28 UTC (rev 199219)
+++ trunk/Websites/perf.webkit.org/tools/js/parse-arguments.js 2016-04-08 07:15:43 UTC (rev 199220)
@@ -1,5 +1,5 @@
-function parseArgument(argv, acceptedOptions) {
+function parseArguments(argv, acceptedOptions) {
var args = argv.slice(2);
var options = {}
for (var i = 0; i < args.length; i += 2) {
@@ -31,4 +31,4 @@
}
if (typeof module != 'undefined')
- module.exports.parseArgument = parseArgument;
+ module.exports.parseArguments = parseArguments;
Modified: trunk/Websites/perf.webkit.org/tools/js/remote.js (199219 => 199220)
--- trunk/Websites/perf.webkit.org/tools/js/remote.js 2016-04-08 07:13:28 UTC (rev 199219)
+++ trunk/Websites/perf.webkit.org/tools/js/remote.js 2016-04-08 07:15:43 UTC (rev 199220)
@@ -17,19 +17,37 @@
{
let scheme = this._server.scheme;
let port = this._server.port;
- let portSuffix = (scheme == 'http' && port == 80) || (scheme == 'https' && port == 443) ? '' : `:${port}`;
+ let portSuffix = this._server.port == this._server.defaultPort ? '' : `:${port}`;
if (path.charAt(0) != '/')
path = '/' + path;
- return `${scheme}://${this._server.host}portSuffix${path}`;
+ return `${scheme}://${this._server.host}${portSuffix}${path}`;
}
configure(server)
{
assert(server.scheme === 'http' || server.scheme === 'https');
- assert.equal(typeof(server.host), 'string');
- assert(!server.port || typeof(server.port) == 'number');
- assert(!server.auth || typeof(server.auth) == 'object');
- this._server = server;
+ assert.equal(typeof(server.host), 'string', 'host should be a string');
+ assert(!server.port || typeof(server.port) == 'number', 'port should be a number');
+
+ let auth = null;
+ if (server.auth) {
+ assert.equal(typeof(server.auth), 'object', 'auth should be a dictionary with username and password as keys');
+ assert.equal(typeof(server.auth.username), 'string', 'auth should contain a string username');
+ assert.equal(typeof(server.auth.password), 'string', 'auth should contain a string password');
+ auth = {
+ username: server.auth.username,
+ password: server.auth.password,
+ };
+ }
+
+ const defaultPort = server.scheme == 'http' ? 80 : 443;
+ this._server = {
+ scheme: server.scheme,
+ host: server.host,
+ port: server.port || defaultPort,
+ defaultPort: defaultPort,
+ auth: auth,
+ };
}
getJSON(path)
@@ -72,7 +90,7 @@
return new Promise(function (resolve, reject) {
let options = {
hostname: server.host,
- port: server.port || 80,
+ port: server.port,
auth: server.auth ? server.auth.username + ':' + server.auth.password : null,
method: method,
path: path,
Modified: trunk/Websites/perf.webkit.org/tools/sync-buildbot.js (199219 => 199220)
--- trunk/Websites/perf.webkit.org/tools/sync-buildbot.js 2016-04-08 07:13:28 UTC (rev 199219)
+++ trunk/Websites/perf.webkit.org/tools/sync-buildbot.js 2016-04-08 07:15:43 UTC (rev 199220)
@@ -21,6 +21,18 @@
function syncLoop(options)
{
+ // FIXME: Fix Manifest.fetch() to use ensureSingleton to create model objects.
+ global.AnalysisTask._fetchAllPromise = null;
+ global.AnalysisTask.clearStaticMap();
+ global.BuildRequest.clearStaticMap();
+ global.CommitLog.clearStaticMap();
+ global.Metric.clearStaticMap();
+ global.Platform.clearStaticMap();
+ global.Repository.clearStaticMap();
+ global.RootSet.clearStaticMap();
+ global.Test.clearStaticMap();
+ global.TestGroup.clearStaticMap();
+
let serverConfig = JSON.parse(fs.readFileSync(options['--server-config-json'], 'utf8'));
let buildbotConfig = JSON.parse(fs.readFileSync(options['--buildbot-config-json'], 'utf8'));
let buildbotRemote = new RemoteAPI(buildbotConfig.server);
Added: trunk/Websites/perf.webkit.org/unit-tests/tools-js-remote-tests.js (0 => 199220)
--- trunk/Websites/perf.webkit.org/unit-tests/tools-js-remote-tests.js (rev 0)
+++ trunk/Websites/perf.webkit.org/unit-tests/tools-js-remote-tests.js 2016-04-08 07:15:43 UTC (rev 199220)
@@ -0,0 +1,112 @@
+'use strict';
+
+const assert = require('assert');
+
+const RemoteAPI = require('../tools/js/remote.js').RemoteAPI;
+
+describe('RemoteAPI', function () {
+ describe('configure', function () {
+ it('should accept a valid configuration', function () {
+ let remote = new RemoteAPI();
+ assert.doesNotThrow(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80}); });
+ });
+
+ it('should reject a scheme other than http and https', function () {
+ let remote = new RemoteAPI();
+ assert.doesNotThrow(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80}); });
+ assert.doesNotThrow(function () { remote.configure({scheme: 'https', host: 'build.webkit.org', port: 9999}); });
+ assert.throws(function () { remote.configure({scheme: 'ftp', host: 'build.webkit.org', port: 9999}); });
+ });
+
+ it('should accept a configuration without port', function () {
+ let remote = new RemoteAPI();
+ assert.doesNotThrow(function () { remote.configure({scheme: 'http', host: 'build.webkit.org'}); });
+ });
+
+ it('should reject non-string value for scheme', function () {
+ let remote = new RemoteAPI();
+ assert.throws(function () { remote.configure({scheme: 1, host: 'build.webkit.org', port: 80}); });
+ assert.throws(function () { remote.configure({scheme: {}, host: 'build.webkit.org', port: 80}); });
+ assert.throws(function () { remote.configure({scheme: [], host: 'build.webkit.org', port: 80}); });
+ });
+
+ it('should reject non-string value for host', function () {
+ let remote = new RemoteAPI();
+ assert.throws(function () { remote.configure({scheme: 'http', host: 1, port: 80}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: {}, port: 80}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: [], port: 80}); });
+ });
+
+ it('should reject non-object value for auth', function () {
+ let remote = new RemoteAPI();
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: 'a'}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: 1}); });
+ });
+
+ it('should accept a configuration with auth', function () {
+ let remote = new RemoteAPI();
+ assert.doesNotThrow(function () {
+ remote.configure({
+ scheme: 'http',
+ host: 'build.webkit.org',
+ port: 80,
+ auth: {
+ 'username': 'a',
+ 'password': 'b',
+ },
+ });
+ });
+ });
+
+ it('should reject auth without username or password or when either is not a string', function () {
+ let remote = new RemoteAPI();
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: {}}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: {username: 'a'}}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: {password: 'b'}}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: {username: 1, password: 'a'}}); });
+ assert.throws(function () { remote.configure({scheme: 'http', host: 'build.webkit.org', port: 80, auth: {username: 'a', password: 1}}); });
+ });
+ });
+
+ describe('url', function () {
+ it('should be able to serialize http://build.webkit.org', function () {
+ let remote = new RemoteAPI({scheme: 'http', host: 'build.webkit.org', port: 80});
+ assert.equal(remote.url('/json/builders'), 'http://build.webkit.org/json/builders');
+ });
+
+ it('should be able to serialize http://build.webkit.org when port is not specified', function () {
+ let remote = new RemoteAPI({scheme: 'http', host: 'build.webkit.org'});
+ assert.equal(remote.url('/json/builders'), 'http://build.webkit.org/json/builders');
+ });
+
+ it('should be able to serialize https://build.webkit.org when port is 80', function () {
+ let remote = new RemoteAPI({scheme: 'http', host: 'build.webkit.org', port: 80});
+ assert.equal(remote.url('/json/builders'), 'http://build.webkit.org/json/builders');
+ });
+
+ it('should be able to serialize http://build.webkit.org:8080', function () {
+ let remote = new RemoteAPI({scheme: 'http', host: 'build.webkit.org', port: 8080});
+ assert.equal(remote.url('/json/builders'), 'http://build.webkit.org:8080/json/builders');
+ });
+
+ it('should be able to serialize https://build.webkit.org when port is not specified', function () {
+ let remote = new RemoteAPI({scheme: 'https', host: 'build.webkit.org'});
+ assert.equal(remote.url('/json/builders'), 'https://build.webkit.org/json/builders');
+ });
+
+ it('should be able to serialize https://build.webkit.org when port is 443', function () {
+ let remote = new RemoteAPI({scheme: 'https', host: 'build.webkit.org', port: 443});
+ assert.equal(remote.url('/json/builders'), 'https://build.webkit.org/json/builders');
+ });
+
+ it('should be able to serialize https://build.webkit.org:8443', function () {
+ let remote = new RemoteAPI({scheme: 'https', host: 'build.webkit.org', port: 8443});
+ assert.equal(remote.url('/json/builders'), 'https://build.webkit.org:8443/json/builders');
+ });
+
+ it('should automatically prefix the path with /', function () {
+ let remote = new RemoteAPI({scheme: 'http', host: 'build.webkit.org'});
+ assert.equal(remote.url('json/builders'), 'http://build.webkit.org/json/builders');
+ });
+ });
+});
_______________________________________________ webkit-changes mailing list [email protected] https://lists.webkit.org/mailman/listinfo/webkit-changes
