Pmiazga has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/390294 )
Change subject: [POC]: An idea to bring QueueManagers into life
......................................................................
[POC]: An idea to bring QueueManagers into life
Change-Id: I77fda9d1b18b4b9f3c07784588f5c2455dd5a409
---
A lib/managers.js
M lib/queue.js
M routes/html2pdf-v1.js
3 files changed, 68 insertions(+), 28 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/chromium-render
refs/changes/94/390294/1
diff --git a/lib/managers.js b/lib/managers.js
new file mode 100644
index 0000000..1418a34
--- /dev/null
+++ b/lib/managers.js
@@ -0,0 +1,48 @@
+'use strict';
+
+const uuid = require('cassandra-uuid');
+const { queueErrors } = require('queue'); // TODO - dependency cycle, move
queueErrors out of queue
+
+class QueueTimeoutManager {
+
+ /**
+ * @param {Number} timeout
+ * @param {Object} logger
+ */
+ constructor(timeout, logger) {
+ this._timeout = timeout;
+ this._logger = logger;
+ }
+
+ onNewJob(queueObject, data, callback) {
+ const logger = this._logger;
+ const timeout = this._timeout;
+
+ // Set a timer before queueing the task and remove the worker
+ // from queue when the time's up if the worker hasn't started
+ // yet. The timer will be aborted when the worker starts. See the
+ // _worker method.
+ data._id = uuid.TimeUuid.now().toString();
+ data._timeoutID = setTimeout(() => {
+ queueObject.remove((worker) => {
+ if (worker.data._id === data._id) {
+ logger.log('trace/warning', {
+ msg: `Queue is still busy after waiting ` +
+ `for ${timeout} secs.`
+ });
+ callback(queueErrors.timeout, null);
+ return true;
+ }
+ return false;
+ });
+ }, this._timeout * 1000);
+ }
+
+ onStart(data) {
+ clearTimeout(data._timeoutID);
+ }
+}
+
+module.exports = {
+ QueueTimeoutManager
+};
diff --git a/lib/queue.js b/lib/queue.js
index a889a88..166d70e 100644
--- a/lib/queue.js
+++ b/lib/queue.js
@@ -2,7 +2,6 @@
const asyncQueue = require('async/queue');
const renderer = require('./renderer');
-const uuid = require('cassandra-uuid');
// common HTTP status codes that are returned in case of various errors
const queueErrors = {
@@ -18,47 +17,29 @@
class Queue {
/**
* @param {number} concurrency number of concurrent render instances
- * @param {number} timeout number of seconds after which the
- * yet-to-start renders are aborted
+ * @param {Array} managers An array of queue managers to handle queue
state
* @param {Object} puppeteerFlags flags used to in starting puppeteer
* @param {Object} pdfOptions pdf options passed to Chromium
* @param {Object} logger app logger
*/
- constructor(concurrency, timeout, puppeteerFlags, pdfOptions, logger) {
+ constructor(concurrency, managers, puppeteerFlags, pdfOptions, logger) {
this._queueObject = asyncQueue(this._worker.bind(this), concurrency);
this._puppeteerFlags = puppeteerFlags;
this._pdfOptions = pdfOptions;
- this._timeout = timeout;
this._logger = logger;
+ this._queueManagers = managers;
}
+
/**
* Push data to the queue
- * If the queue is busy after a predefined time, the task will be aborted.
* @param {Object} data that the worker needs
* @param {Function} callback called when the worker finishes
*/
push(data, callback) {
- const that = this;
-
- // Set a timer before queueing the task and remove the worker
- // from queue when the time's up if the worker hasn't started
- // yet. The timer will be aborted when the worker starts. See the
- // _worker method.
- data._id = uuid.TimeUuid.now().toString();
- data._timeoutID = setTimeout(() => {
- that._queueObject.remove((worker) => {
- if (worker.data._id === data._id) {
- that._logger.log('trace/warning', {
- msg: `Queue is still busy after waiting ` +
- `for ${that._timeout} secs.`
- });
- callback(queueErrors.timeout, null);
- return true;
- }
- return false;
- });
- }, this._timeout * 1000);
+ this._queueManagers.each((strategy) => {
+ strategy.onNewJob(this._queueManagers, data, callback);
+ });
this._queueObject.push(data, callback);
}
@@ -66,7 +47,9 @@
* Worker that renders article and aborts queue timeout
*/
_worker(data, callback) {
- clearTimeout(data._timeoutID);
+ this._queueManagers.each((strategy) => {
+ strategy.onStart(this._queueObject, data, callback);
+ });
renderer
.articleToPdf(data.uri, data.format, this._puppeteerFlags,
diff --git a/routes/html2pdf-v1.js b/routes/html2pdf-v1.js
index c667d52..5ccc2e3 100644
--- a/routes/html2pdf-v1.js
+++ b/routes/html2pdf-v1.js
@@ -2,6 +2,7 @@
const { queueErrors, Queue } = require('../lib/queue');
const sUtil = require('../lib/util');
+const { QueueTimeoutManager } = require('../lib/managers');
/**
* The main router object
@@ -59,9 +60,17 @@
app = appObj;
const conf = app.conf;
+ const managers = [
+ new QueueTimeoutManager(conf.render_queue_timeout, app.logger)
+ // also if we really want
+ // RenderTimeoutManager
+ // MaxQueueSizeManager
+ // MaxRenderingInProgressManager?
+ ];
+
app.queue = new Queue(
conf.render_concurrency,
- conf.render_queue_timeout,
+ managers,
conf.puppeteer_flags,
conf.pdf_options,
app.logger
--
To view, visit https://gerrit.wikimedia.org/r/390294
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I77fda9d1b18b4b9f3c07784588f5c2455dd5a409
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/chromium-render
Gerrit-Branch: master
Gerrit-Owner: Pmiazga <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
