Legoktm has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/316212

Change subject: linter: Implement sampling options
......................................................................

linter: Implement sampling options

When enabling this in production, we want to be very careful not to
overload MediaWiki with a large number of lint errors. To do this, there
are two main linting features:

First, enable selective categories to be reported to MediaWiki. The
"linting" configuration setting can be a list of the names of lint
errors to enable, or true to enable everything.

Then is a per page sampling feature. Since we also log deletions, we
need this to be deterministic, so we use page_id % sampling === 0 to
check whether logs should be sent.

Change-Id: I2f939859a6f49b40aff93f3c07b11c64c0b00fc1
---
M lib/config/ParsoidConfig.js
M lib/logger/linter.js
2 files changed, 27 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/12/316212/1

diff --git a/lib/config/ParsoidConfig.js b/lib/config/ParsoidConfig.js
index dbf54c5..337923c 100644
--- a/lib/config/ParsoidConfig.js
+++ b/lib/config/ParsoidConfig.js
@@ -220,7 +220,8 @@
 ParsoidConfig.prototype.addHTMLTemplateParameters = false;
 
 /**
- * @property {boolean} linting Whether to enable linter Backend.
+ * @property {boolean|Array} linting Whether to enable linter Backend.
+ * Or an array of enabled lint types
  */
 ParsoidConfig.prototype.linting = false;
 
@@ -231,6 +232,14 @@
 ParsoidConfig.prototype.linterAPI = null;
 
 /**
+ * @property {number} linterSampling
+ *
+ * Ratio at which to sample linter errors, per page.
+ * This is deterministic and based on page_id.
+ */
+ParsoidConfig.prototype.linterAPISampling = 1;
+
+/**
  * @property {Function} loggerBackend
  * The logger output function.
  * By default, use stderr to output logs.
diff --git a/lib/logger/linter.js b/lib/logger/linter.js
index 7badefb..7267c19 100644
--- a/lib/logger/linter.js
+++ b/lib/logger/linter.js
@@ -16,6 +16,22 @@
 
 Linter.prototype.logLintOutput = function(logData, cb) {
        var env = this._env;
+       var enabledBuffer;
+       // Everything is enabled
+       if (env.conf.parsoid.linting === true) {
+               enabledBuffer = this.buffer;
+       } else if (Array.isArray(env.conf.parsoid.linting)) {
+               enabledBuffer = this.buffer.filter(function(item) {
+                       return env.conf.parsoid.linting.indexOf(item.type) !== 
-1;
+               });
+       }
+
+       this.buffer = [];
+
+       if (env.page.id % env.conf.parsoid.linterAPISampling !== 0) {
+               return;
+       }
+
        try {
                if (!env.conf.parsoid.linterAPI) {
                        env.log(this.buffer);
@@ -26,7 +42,7 @@
                                request.post(
                                        env.conf.parsoid.linterAPI,
                                        { form: {
-                                               data: 
JSON.stringify(this.buffer),
+                                               data: 
JSON.stringify(enabledBuffer),
                                                page: env.page.name,
                                                revision: 
env.page.meta.revision.revid,
                                                action: 'record-lint',
@@ -39,7 +55,6 @@
                                );
                        }
                }
-               this.buffer = [];
        } catch (e) {
                console.error("Error in logLintOutput: " + e);
        } finally {

-- 
To view, visit https://gerrit.wikimedia.org/r/316212
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2f939859a6f49b40aff93f3c07b11c64c0b00fc1
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipe...@gmail.com>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to