Gehel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/395478 )
Change subject: initial commit
......................................................................
initial commit
Change-Id: Iffebc2da34e0a1c64a02d12a5cf0a0a0f1216c61
---
A .dockerignore
A .gitignore
A .gitreview
A app.js
A config.dev.yaml
A package.json
A server.js
A targets.yaml
8 files changed, 412 insertions(+), 0 deletions(-)
Approvals:
Gehel: Verified; Looks good to me, approved
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..563df11
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+.git
+coverage
+node_modules
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..a0a1c47
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+Dockerfile
+.idea/
+coverage
+config.yaml
+node_modules
+npm-debug.log
+vectors
+variables.yaml
diff --git a/.gitreview b/.gitreview
new file mode 100644
index 0000000..fd59be6
--- /dev/null
+++ b/.gitreview
@@ -0,0 +1,6 @@
+[gerrit]
+host=gerrit.wikimedia.org
+port=29418
+project=maps/tilerator/package.git
+defaultbranch=master
+defaultrebase=0
diff --git a/app.js b/app.js
new file mode 100644
index 0000000..f866804
--- /dev/null
+++ b/app.js
@@ -0,0 +1,203 @@
+'use strict';
+
+
+var http = require('http');
+var BBPromise = require('bluebird');
+var express = require('express');
+var compression = require('compression');
+var bodyParser = require('body-parser');
+var fs = BBPromise.promisifyAll(require('fs'));
+var sUtil = require('./lib/util');
+var packageInfo = require('./package.json');
+var yaml = require('js-yaml');
+
+
+/**
+ * Creates an express app and initialises it
+ * @param {Object} options the options to initialise the app with
+ * @return {bluebird} the promise resolving to the app object
+ */
+function initApp(options) {
+
+ // the main application object
+ var app = express();
+
+ // get the options and make them available in the app
+ app.logger = options.logger; // the logging device
+ app.metrics = options.metrics; // the metrics
+ app.conf = options.config; // this app's config options
+ app.info = packageInfo; // this app's package info
+
+ // ensure some sane defaults
+ if(!app.conf.port) { app.conf.port = 8888; }
+ if(!app.conf.interface) { app.conf.interface = '0.0.0.0'; }
+ if(app.conf.compression_level === undefined) { app.conf.compression_level
= 3; }
+ if(app.conf.cors === undefined) { app.conf.cors = '*'; }
+ if(app.conf.csp === undefined) {
+ app.conf.csp =
+ "default-src 'self'; object-src 'none'; media-src *; img-src *;
style-src *; frame-ancestors 'self'";
+ }
+
+ // set outgoing proxy
+ if(app.conf.proxy) {
+ process.env.HTTP_PROXY = app.conf.proxy;
+ // if there is a list of domains which should
+ // not be proxied, set it
+ if(app.conf.no_proxy_list) {
+ if(Array.isArray(app.conf.no_proxy_list)) {
+ process.env.NO_PROXY = app.conf.no_proxy_list.join(',');
+ } else {
+ process.env.NO_PROXY = app.conf.no_proxy_list;
+ }
+ }
+ }
+
+ // set up header whitelisting for logging
+ if(!app.conf.log_header_whitelist) {
+ app.conf.log_header_whitelist = [
+ 'cache-control', 'content-type', 'content-length', 'if-match',
+ 'user-agent', 'x-request-id'
+ ];
+ }
+ app.conf.log_header_whitelist = new RegExp('^(?:' +
app.conf.log_header_whitelist.map(function(item) {
+ return item.trim();
+ }).join('|') + ')$', 'i');
+
+ // set up the spec
+ if(!app.conf.spec) {
+ app.conf.spec = __dirname + '/spec.yaml';
+ }
+ if(app.conf.spec.constructor !== Object) {
+ try {
+ app.conf.spec = yaml.safeLoad(fs.readFileSync(app.conf.spec));
+ } catch(e) {
+ app.logger.log('warn/spec', 'Could not load the spec: ' + e);
+ app.conf.spec = {};
+ }
+ }
+ if(!app.conf.spec.swagger) {
+ app.conf.spec.swagger = '2.0';
+ }
+ if(!app.conf.spec.info) {
+ app.conf.spec.info = {
+ version: app.info.version,
+ title: app.info.name,
+ description: app.info.description
+ };
+ }
+ app.conf.spec.info.version = app.info.version;
+ if(!app.conf.spec.paths) {
+ app.conf.spec.paths = {};
+ }
+
+ // set the CORS and CSP headers
+ app.all('*', function(req, res, next) {
+ //
+ // Tilerator is an admin app, there is no point to set app.conf.cors
and app.conf.csp
+ //
+ sUtil.initAndLogRequest(req, app);
+ next();
+ });
+
+ // disable the X-Powered-By header
+ app.set('x-powered-by', false);
+ // disable the ETag header - users should provide them!
+ app.set('etag', false);
+ // enable compression
+ app.use(compression({level: app.conf.compression_level}));
+ // use the JSON body parser
+ app.use(bodyParser.json());
+ // use the application/x-www-form-urlencoded parser
+ app.use(bodyParser.urlencoded({extended: true}));
+
+ return BBPromise.resolve(app);
+
+}
+
+
+/**
+ * Loads all routes declared in routes/ into the app
+ * @param {Application} app the application object to load routes into
+ * @returns {bluebird} a promise resolving to the app object
+ */
+function loadRoutes (app) {
+
+ // get the list of files in routes/
+ return fs.readdirAsync(__dirname + '/routes').map(function(fname) {
+ return BBPromise.try(function() {
+ // ... and then load each route
+ // but only if it's a js file
+ if(!/\.js$/.test(fname)) {
+ return undefined;
+ }
+ // import the route file
+ var route = require(__dirname + '/routes/' + fname);
+ return route(app);
+ }).then(function(route) {
+ if(route === undefined) {
+ return undefined;
+ }
+ // check that the route exports the object we need
+ if(route.constructor !== Object || !route.path || !route.router ||
!(route.api_version || route.skip_domain)) {
+ throw new TypeError('routes/' + fname + ' does not export the
correct object!');
+ }
+ // wrap the route handlers with Promise.try() blocks
+ sUtil.wrapRouteHandlers(route.router);
+ // determine the path prefix
+ var prefix = '';
+ if(!route.skip_domain) {
+ prefix = '/:domain/v' + route.api_version;
+ }
+ // all good, use that route
+ app.use(prefix + route.path, route.router);
+ });
+ }).then(function () {
+ // catch errors
+ sUtil.setErrorHandler(app);
+ // route loading is now complete, return the app object
+ return BBPromise.resolve(app);
+ });
+
+}
+
+
+/**
+ * Creates and start the service's web server
+ * @param {Application} app the app object to use in the service
+ * @returns {bluebird} a promise creating the web server
+ */
+function createServer(app) {
+
+ // return a promise which creates an HTTP server,
+ // attaches the app to it, and starts accepting
+ // incoming client requests
+ var server;
+ return new BBPromise(function (resolve) {
+ server = http.createServer(app).listen(
+ app.conf.port,
+ app.conf.interface,
+ resolve
+ );
+ }).then(function () {
+ app.logger.log('info',
+ 'Worker ' + process.pid + ' listening on ' + app.conf.interface +
':' + app.conf.port);
+ return server;
+ });
+
+}
+
+
+/**
+ * The service's entry point. It takes over the configuration
+ * options and the logger- and metrics-reporting objects from
+ * service-runner and starts an HTTP server, attaching the application
+ * object to it.
+ */
+module.exports = function(options) {
+
+ return initApp(options)
+ .then(loadRoutes)
+ .then(createServer);
+
+};
+
diff --git a/config.dev.yaml b/config.dev.yaml
new file mode 100644
index 0000000..4f10557
--- /dev/null
+++ b/config.dev.yaml
@@ -0,0 +1,75 @@
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: 0
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 250
+
+# Logger info
+logging:
+ level: info
+ streams:
+ - type: debug
+# streams:
+# # Use gelf-stream -> logstash
+# - type: gelf
+# host: logstash1003.eqiad.wmnet
+# port: 12201
+
+# Statsd metrics reporter
+metrics:
+ #type: log
+ #host: localhost
+ #port: 8125
+
+services:
+ - name: tilerator
+ # a relative path or the name of an npm package, if different from name
+ module: ./app.js
+ # optionally, a version constraint of the npm package
+ # version: ^0.4.0
+ # per-service config
+ conf:
+ port: 16534
+
+ # restrict to localhost access only
+ interface: localhost
+
+ # more per-service config settings
+ # the location of the spec, defaults to spec.yaml if not specified
+ spec: ./spec.template.yaml
+ # allow cross-domain requests to the API (default '*')
+ cors: '*'
+ # to disable use:
+ # cors: false
+ # to restrict to a particular domain, use:
+ # cors: restricted.domain.org
+ # content for the CSP headers
+ # csp: false # uncomment this line to disable sending them
+ # URL of the outbound proxy to use (complete with protocol)
+ # proxy: http://my.proxy.org:8080
+ # the list of domains for which not to use the proxy defined above
+ # no_proxy_list:
+ # - domain1.com
+ # - domain2.org
+ # the list of incoming request headers that can be logged; if left empty,
+ # the following headers are allowed: cache-control, content-length,
+ # content-type, if-match, user-agent, x-request-id
+ # log_header_whitelist:
+ # - cache-control
+ # - content-length
+ # - content-type
+ # - if-match
+ # - user-agent
+ # - x-request-id
+
+ sources: ../sources.yaml
+
+ # If true, do not enable admin interface
+ daemonOnly: false
+
+ # If true, runs this instance without processing tiles
+ # This could be good for queue management
+ uiOnly: false
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..8ff4c54
--- /dev/null
+++ b/package.json
@@ -0,0 +1,102 @@
+{
+ "name": "tilerator-package",
+ "version": "0.0.1",
+ "description": "Map tiles pre-generation service - packaging for WMF",
+ "main": "./app.js",
+ "scripts": {
+ "start": "service-runner",
+ "test": "mocha",
+ "docker-start": "service-runner docker-start",
+ "docker-test": "service-runner docker-test",
+ "coverage": "istanbul cover _mocha -- -R spec"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://gerrit.wikimedia.org/r/maps/tilerator/package";
+ },
+ "keywords": [
+ "REST",
+ "API",
+ "service template",
+ "MediaWiki"
+ ],
+ "author": "Guillaume Lederrey <ge...@wikimedia.org>",
+ "contributors": [
+ "Paul Norman <pnor...@wikimedia.org>"
+ ],
+ "license": "Apache-2.0",
+ "bugs": "https://github.com/kartotherian/tilerator/issues";,
+ "homepage": "https://github.com/kartotherian/tilerator";,
+ "kartotherian": {
+ "registerSourceLibs": [
+ "tilelive-bridge",
+ "tilelive-vector",
+ "@kartotherian/autogen",
+ "@kartotherian/cassandra",
+ "@kartotherian/demultiplexer",
+ "@kartotherian/layermixer",
+ "@kartotherian/overzoom",
+ "@kartotherian/postgres",
+ "@kartotherian/substantial"
+ ]
+ },
+ "dependencies": {
+ "kartotherian": "git+https://github.com/kartotherian/tilerator.git#master";,
+ "service-runner": "^2.4.6"
+ },
+ "optionalDependencies": {
+ "bunyan-prettystream": "*"
+ },
+ "devDependencies": {
+ "extend": "^3.0.0",
+ "istanbul": "^0.4.3",
+ "mocha": "^3.2.0",
+ "mocha-jshint": "^2.3.1",
+ "mocha-lcov-reporter": "^1.2.0",
+ "swagger-router": "^0.4.2"
+ },
+ "deploy": {
+ "node": "6.9.1",
+ "target": "debian",
+ "install_opts": [
+ "--build-from-source=mapnik",
+ "--fallback-to-build=false"
+ ],
+ "dependencies": {
+ "_all": [
+ "libcairo2-dev",
+ "libgif-dev",
+ "libpango1.0-dev"
+ ],
+ "ubuntu": [
+ "libjpeg62-dev"
+ ],
+ "debian": [
+ "libjpeg62-turbo-dev",
+ "fonts-dejavu",
+ "libboost-filesystem-dev",
+ "libboost-program-options-dev",
+ "libboost-regex-dev",
+ "libboost-system-dev",
+ "libboost-thread-dev",
+ "libgdal-dev",
+ "libicu-dev",
+ "libpq-dev",
+ "libcurl4-gnutls-dev",
+ "libproj-dev",
+ "libtiff-dev",
+ "libwebp5",
+ {
+ "repo_url": "https://apt.wikimedia.org/wikimedia";,
+ "release": "jessie-wikimedia",
+ "pool": "backports",
+ "packages": [
+ "libmapbox-variant-dev",
+ "libmapnik-dev",
+ "mapnik-utils"
+ ]
+ }
+ ]
+ }
+ }
+}
diff --git a/server.js b/server.js
new file mode 100755
index 0000000..da9fad8
--- /dev/null
+++ b/server.js
@@ -0,0 +1,12 @@
+#!/usr/bin/env node
+
+'use strict';
+
+// Service entry point. Try node server --help for commandline options.
+
+// Start the service by running service-runner, which in turn loads the config
+// (config.yaml by default, specify other path with -c). It requires the
+// module(s) specified in the config 'services' section (app.js in this
+// example).
+var ServiceRunner = require('service-runner');
+return new ServiceRunner().run();
diff --git a/targets.yaml b/targets.yaml
new file mode 100644
index 0000000..6a16a18
--- /dev/null
+++ b/targets.yaml
@@ -0,0 +1,3 @@
+debian: 'debian:jessie'
+ubuntu: 'ubuntu:14.04'
+
--
To view, visit https://gerrit.wikimedia.org/r/395478
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iffebc2da34e0a1c64a02d12a5cf0a0a0f1216c61
Gerrit-PatchSet: 2
Gerrit-Project: maps/tilerator/package
Gerrit-Branch: master
Gerrit-Owner: Gehel <guillaume.leder...@wikimedia.org>
Gerrit-Reviewer: Gehel <guillaume.leder...@wikimedia.org>
Gerrit-Reviewer: Pnorman <penor...@mac.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
