jenkins-bot has submitted this change and it was merged.

Change subject: linter: Implement sampling options
......................................................................


linter: Implement sampling options

When enabling this in production, we want to be very careful not to
overload MediaWiki with a large number of lint errors. To do this, there
are two main linting features:

First, enable selective categories to be reported to MediaWiki. The
"linting" configuration setting can be a list of the names of lint
errors to enable, or true to enable everything.

Then is a per page sampling feature. Since we also log deletions, we
need this to be deterministic, so we use page_id % sampling === 0 to
check whether logs should be sent.

Change-Id: I2f939859a6f49b40aff93f3c07b11c64c0b00fc1
---
M lib/config/ParsoidConfig.js
M lib/logger/linter.js
2 files changed, 27 insertions(+), 3 deletions(-)

Approvals:
  Arlolra: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/lib/config/ParsoidConfig.js b/lib/config/ParsoidConfig.js
index dbf54c5..337923c 100644
--- a/lib/config/ParsoidConfig.js
+++ b/lib/config/ParsoidConfig.js
@@ -220,7 +220,8 @@
 ParsoidConfig.prototype.addHTMLTemplateParameters = false;
 
 /**
- * @property {boolean} linting Whether to enable linter Backend.
+ * @property {boolean|Array} linting Whether to enable linter Backend.
+ * Or an array of enabled lint types
  */
 ParsoidConfig.prototype.linting = false;
 
@@ -231,6 +232,14 @@
 ParsoidConfig.prototype.linterAPI = null;
 
 /**
+ * @property {number} linterSampling
+ *
+ * Ratio at which to sample linter errors, per page.
+ * This is deterministic and based on page_id.
+ */
+ParsoidConfig.prototype.linterAPISampling = 1;
+
+/**
  * @property {Function} loggerBackend
  * The logger output function.
  * By default, use stderr to output logs.
diff --git a/lib/logger/linter.js b/lib/logger/linter.js
index 97b8881..210e463 100644
--- a/lib/logger/linter.js
+++ b/lib/logger/linter.js
@@ -16,6 +16,22 @@
 
 Linter.prototype.logLintOutput = function(logData, cb) {
        var env = this._env;
+       var enabledBuffer;
+       // Everything is enabled
+       if (env.conf.parsoid.linting === true) {
+               enabledBuffer = this.buffer;
+       } else if (Array.isArray(env.conf.parsoid.linting)) {
+               enabledBuffer = this.buffer.filter(function(item) {
+                       return env.conf.parsoid.linting.indexOf(item.type) !== 
-1;
+               });
+       }
+
+       this.buffer = [];
+
+       if (env.page.id % env.conf.parsoid.linterAPISampling !== 0) {
+               return;
+       }
+
        try {
                if (env.conf.parsoid.linterAPI) {
                        // Only send the request if it is
@@ -24,7 +40,7 @@
                                request.post(
                                        env.conf.parsoid.linterAPI,
                                        { form: {
-                                               data: 
JSON.stringify(this.buffer),
+                                               data: 
JSON.stringify(enabledBuffer),
                                                page: env.page.name,
                                                revision: 
env.page.meta.revision.revid,
                                                action: 'record-lint',
@@ -37,7 +53,6 @@
                                );
                        }
                }
-               this.buffer = [];
        } catch (e) {
                console.error("Error in logLintOutput: " + e);
        } finally {

-- 
To view, visit https://gerrit.wikimedia.org/r/316212
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I2f939859a6f49b40aff93f3c07b11c64c0b00fc1
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Legoktm <legoktm.wikipe...@gmail.com>
Gerrit-Reviewer: Arlolra <abrea...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to