This is an automated email from the ASF dual-hosted git repository.
dklco pushed a commit to branch insights-feature
in repository
https://gitbox.apache.org/repos/asf/sling-org-apache-sling-app-cms.git
The following commit(s) were added to refs/heads/insights-feature by this push:
new 514e6b7 Adding working insight tools for HTML Validation and Page
speed
514e6b7 is described below
commit 514e6b753b28ac04278d594281623100da6d40db
Author: Dan Klco <[email protected]>
AuthorDate: Thu Oct 18 13:57:02 2018 -0400
Adding working insight tools for HTML Validation and Page speed
---
core/pom.xml | 8 +
.../providers/HTMLValdiatorInsightProvider.java | 166 +++++++++++++++++++++
.../impl/providers/PageSpeedInsightProvider.java | 152 +++++++++++++++++++
.../providers}/ReadabilityInsightProvider.java | 20 ++-
.../main/resources/OSGI-INF/l10n/bundle.properties | 17 ++-
pom.xml | 14 ++
.../resources/jcr_root/libs/sling-cms/i18n.json | 30 ++++
7 files changed, 400 insertions(+), 7 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 219306d..9f2d39b 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -202,5 +202,13 @@
<groupId>org.apache.felix</groupId>
<artifactId>org.apache.felix.webconsole</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git
a/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/HTMLValdiatorInsightProvider.java
b/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/HTMLValdiatorInsightProvider.java
new file mode 100644
index 0000000..e849772
--- /dev/null
+++
b/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/HTMLValdiatorInsightProvider.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sling.cms.core.insights.impl.providers;
+
+import java.io.StringReader;
+import java.net.URLEncoder;
+
+import javax.json.Json;
+import javax.json.JsonArray;
+import javax.json.JsonObject;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.sling.cms.core.insights.impl.BaseInsightProvider;
+import
org.apache.sling.cms.core.insights.impl.providers.HTMLValdiatorInsightProvider.Config;
+import org.apache.sling.cms.i18n.I18NDictionary;
+import org.apache.sling.cms.i18n.I18NProvider;
+import org.apache.sling.cms.insights.Insight;
+import org.apache.sling.cms.insights.InsightProvider;
+import org.apache.sling.cms.insights.InsightRequest;
+import org.apache.sling.cms.insights.Message;
+import org.apache.sling.cms.insights.PageInsightRequest;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.metatype.annotations.AttributeDefinition;
+import org.osgi.service.metatype.annotations.Designate;
+import org.osgi.service.metatype.annotations.ObjectClassDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+
+@Component(service = InsightProvider.class)
+@Designate(ocd = Config.class)
+public class HTMLValdiatorInsightProvider extends BaseInsightProvider {
+
+ public static final String I18N_KEY_HTMLVALIDATOR_DANGER =
"slingcms.htmlvalidator.danger";
+ public static final String I18N_KEY_HTMLVALIDATOR_WARN =
"slingcms.htmlvalidator.warn";
+ public static final String I18N_KEY_HTMLVALIDATOR_SUCCESS =
"slingcms.htmlvalidator.success";
+
+ @ObjectClassDefinition(name = "%htmlvalidator.config.name", description =
"%htmlvalidator.config.description", localization = "OSGI-INF/l10n/bundle")
+ public @interface Config {
+ @AttributeDefinition(name = "%htmlvalidator.param.enabled.name",
description = "%htmlvalidator.param.enabled.description")
+ boolean enabled() default true;
+ }
+
+ @Reference
+ private I18NProvider i18nProvider;
+
+ private static final Logger log =
LoggerFactory.getLogger(HTMLValdiatorInsightProvider.class);
+
+ private Config config;
+
+ @Activate
+ public void activate(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ protected Insight doEvaluateRequest(InsightRequest request) throws
Exception {
+ Insight insight = new Insight(this, request);
+ insight.setScored(true);
+ PageInsightRequest pageRequest = (PageInsightRequest) request;
+
+ String html = pageRequest.getPageHtml();
+
+ HttpPost httpPost = new
HttpPost("http://validator.w3.org/nu/?out=json&showsource=no&level=all";);
+ httpPost.addHeader("Content-type", "text/html; charset=utf-8");
+ HttpEntity htmlEntity = new ByteArrayEntity(html.getBytes("UTF-8"));
+ httpPost.setEntity(htmlEntity);
+
+ I18NDictionary dictionary =
i18nProvider.getDictionary(request.getResource().getResourceResolver());
+
+ CloseableHttpResponse response = null;
+ try (CloseableHttpClient client = HttpClients.createDefault()) {
+ response = client.execute(httpPost);
+ HttpEntity entity = response.getEntity();
+ JsonObject json = Json.createReader(new
StringReader(EntityUtils.toString(entity))).readObject();
+ log.debug("Loaded response: {}", json.toString(), 2);
+ JsonArray messages = json.getJsonArray("messages");
+ int errors = 0;
+ int warnings = 0;
+ for (int i = 0; i < messages.size(); i++) {
+ JsonObject message = messages.getJsonObject(i);
+ if ("error".equals(message.getString("type"))) {
+ errors++;
+ } else if ("info".equals(message.getString("type")) &&
message.containsKey("subtype")
+ && "warning".equals(message.getString("subtype"))) {
+ warnings++;
+ }
+ }
+ double score = 0.0;
+ if (errors > 5) {
+ insight.setPrimaryMessage(Message
+ .danger(dictionary.get(I18N_KEY_HTMLVALIDATOR_DANGER,
new Object[] { errors, warnings })));
+ score = 0.2;
+ } else if (errors > 0) {
+ insight.setPrimaryMessage(Message
+ .danger(dictionary.get(I18N_KEY_HTMLVALIDATOR_DANGER,
new Object[] { errors, warnings })));
+ score = 0.4;
+ } else if (warnings > 5) {
+ insight.setPrimaryMessage(
+
Message.danger(dictionary.get(I18N_KEY_HTMLVALIDATOR_WARN, new Object[] {
warnings })));
+ score = 0.6;
+ } else if (warnings > 0) {
+ insight.setPrimaryMessage(
+
Message.danger(dictionary.get(I18N_KEY_HTMLVALIDATOR_WARN, new Object[] {
warnings })));
+ score = 0.8;
+ } else {
+
insight.setPrimaryMessage(Message.danger(dictionary.get(I18N_KEY_HTMLVALIDATOR_SUCCESS)));
+ score = 1.0;
+ }
+ insight.setScore(score);
+
insight.addMessage(Message.defaultMsg("https://validator.w3.org/nu/?doc=";
+ +
URLEncoder.encode(pageRequest.getPage().getPublishedUrl(),
Charsets.UTF_8.toString())));
+ }
+
+ return insight;
+ }
+
+ @Override
+ public String getId() {
+ return "htmlvalidator";
+ }
+
+ @Override
+ public String getTitle() {
+ return "HTML Validator";
+ }
+
+ @Override
+ public boolean isEnabled(InsightRequest request) {
+ if (!config.enabled()) {
+ log.debug("HTML Validator is not enabled");
+ return false;
+ }
+ if (request.getType() != InsightRequest.TYPE.PAGE) {
+ log.debug("Request {} is not a page", request);
+ return false;
+ }
+ return true;
+ }
+
+}
diff --git
a/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/PageSpeedInsightProvider.java
b/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/PageSpeedInsightProvider.java
new file mode 100644
index 0000000..6914cbd
--- /dev/null
+++
b/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/PageSpeedInsightProvider.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sling.cms.core.insights.impl.providers;
+
+import java.io.StringReader;
+import java.net.URLEncoder;
+
+import javax.json.Json;
+import javax.json.JsonObject;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.apache.sling.cms.core.insights.impl.BaseInsightProvider;
+import
org.apache.sling.cms.core.insights.impl.providers.PageSpeedInsightProvider.Config;
+import org.apache.sling.cms.i18n.I18NDictionary;
+import org.apache.sling.cms.i18n.I18NProvider;
+import org.apache.sling.cms.insights.Insight;
+import org.apache.sling.cms.insights.InsightProvider;
+import org.apache.sling.cms.insights.InsightRequest;
+import org.apache.sling.cms.insights.Message;
+import org.apache.sling.cms.insights.PageInsightRequest;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.metatype.annotations.AttributeDefinition;
+import org.osgi.service.metatype.annotations.Designate;
+import org.osgi.service.metatype.annotations.ObjectClassDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Component(service = InsightProvider.class)
+@Designate(ocd = Config.class)
+public class PageSpeedInsightProvider extends BaseInsightProvider {
+
+ @ObjectClassDefinition(name = "%pagespeed.config.name", description =
"%pagespeed.config.description", localization = "OSGI-INF/l10n/bundle")
+ public @interface Config {
+ @AttributeDefinition(name = "%pagespeed.param.enabled.name",
description = "%pagespeed.param.enabled.description")
+ boolean enabled();
+
+ @AttributeDefinition(name = "%pagespeed.param.apikey.name",
description = "%pagespeed.param.apikey.description")
+ String apikey();
+ }
+
+ @Reference
+ private I18NProvider i18nProvider;
+
+ public static final String I18N_KEY_READABILITY_RESULT_DANGER =
"slingcms.pagespeed.danger";
+ public static final String I18N_KEY_READABILITY_RESULT_WARN =
"slingcms.pagespeed.warn";
+ public static final String I18N_KEY_READABILITY_RESULT_SUCCESS =
"slingcms.pagespeed.success";
+ private static final String REQUEST_FORMAT =
"https://www.googleapis.com/pagespeedonline/v2/runPagespeed?url=%s&fields=%s&key=%s";;
+ private static final String PARAMETERS =
"id%2CinvalidRules%2CresponseCode%2CruleGroups";
+ private static final String PAGESPEED_FORMAT =
"https://developers.google.com/speed/pagespeed/insights/?url=%s";;
+ private static final Logger log =
LoggerFactory.getLogger(PageSpeedInsightProvider.class);
+
+ private Config config;
+
+ @Activate
+ public void activate(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ protected Insight doEvaluateRequest(InsightRequest request) throws
Exception {
+ Insight insight = new Insight(this, request);
+ PageInsightRequest pageRequest = (PageInsightRequest) request;
+ String publishedUrl = pageRequest.getPage().getPublishedUrl();
+ String checkUrl = String.format(REQUEST_FORMAT,
URLEncoder.encode(publishedUrl, "UTF-8"), PARAMETERS,
+ config.apikey());
+
+ HttpGet httpGet = new HttpGet(checkUrl);
+
+ CloseableHttpResponse response = null;
+ try (CloseableHttpClient client = HttpClients.createDefault()) {
+
+ I18NDictionary dictionary =
i18nProvider.getDictionary(request.getResource().getResourceResolver());
+
+ log.debug("Requesting page speed via: {}", checkUrl);
+ response = client.execute(httpGet);
+ HttpEntity entity = response.getEntity();
+ JsonObject resp = Json.createReader(new
StringReader(EntityUtils.toString(entity))).readObject();
+
+ log.debug("Retrieved response: {}", resp.toString());
+
+ insight.setScored(true);
+ double score =
resp.getJsonObject("ruleGroups").getJsonObject("SPEED").getJsonNumber("score").doubleValue()
+ / 100.0;
+ insight.setScore(score);
+ log.debug("Parsed pagespeed score {}", score);
+
+ if (score < 0.65) {
+
insight.setPrimaryMessage(Message.danger(dictionary.get(I18N_KEY_READABILITY_RESULT_DANGER)));
+ } else if (score < 0.8) {
+
insight.setPrimaryMessage(Message.warn(dictionary.get(I18N_KEY_READABILITY_RESULT_WARN)));
+ } else {
+
insight.setPrimaryMessage(Message.success(dictionary.get(I18N_KEY_READABILITY_RESULT_SUCCESS)));
+ }
+
insight.addMessage(Message.defaultMsg(String.format(PAGESPEED_FORMAT,
URLEncoder.encode(publishedUrl, "UTF-8"))));
+
+ log.debug("Response parsed successfully");
+
+ }
+ return insight;
+ }
+
+ @Override
+ public String getId() {
+ return "pagespeed";
+ }
+
+ @Override
+ public String getTitle() {
+ return "Page Speed";
+ }
+
+ @Override
+ public boolean isEnabled(InsightRequest request) {
+ if (!config.enabled()) {
+ log.debug("Page Speed is not enabled");
+ return false;
+ }
+ if (request.getType() != InsightRequest.TYPE.PAGE) {
+ log.debug("Request {} is not a page", request);
+ return false;
+ }
+ if (!((PageInsightRequest) request).getPage().isPublished()) {
+ log.debug("The page for {} is not published", request);
+ return false;
+ }
+ return true;
+ }
+
+}
diff --git
a/core/src/main/java/org/apache/sling/cms/core/readability/impl/ReadabilityInsightProvider.java
b/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/ReadabilityInsightProvider.java
similarity index 90%
rename from
core/src/main/java/org/apache/sling/cms/core/readability/impl/ReadabilityInsightProvider.java
rename to
core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/ReadabilityInsightProvider.java
index 47d94e2..9a5d0af 100644
---
a/core/src/main/java/org/apache/sling/cms/core/readability/impl/ReadabilityInsightProvider.java
+++
b/core/src/main/java/org/apache/sling/cms/core/insights/impl/providers/ReadabilityInsightProvider.java
@@ -14,10 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.sling.cms.core.readability.impl;
+package org.apache.sling.cms.core.insights.impl.providers;
import java.text.DecimalFormat;
+import org.apache.commons.math.stat.descriptive.moment.StandardDeviation;
import org.apache.sling.api.resource.Resource;
import org.apache.sling.caconfig.resource.ConfigurationResourceResolver;
import org.apache.sling.cms.CMSConstants;
@@ -44,7 +45,6 @@ import org.slf4j.LoggerFactory;
public class ReadabilityInsightProvider extends BaseInsightProvider {
public static final String I18N_KEY_READABILITY_DETAIL =
"slingcms.readability.detail";
-
public static final String I18N_KEY_READABILITY_RESULT_DANGER =
"slingcms.readability.danger";
public static final String I18N_KEY_READABILITY_RESULT_SUCCESS =
"slingcms.readability.success";
public static final String I18N_KEY_READABILITY_RESULT_WARN =
"slingcms.readability.warn";
@@ -109,7 +109,21 @@ public class ReadabilityInsightProvider extends
BaseInsightProvider {
if (score > config.getMaxGradeLevel() || score <
config.getMinGradeLevel()) {
log.debug("Retrieved out of bounds readability {} based on
range {}-{}", score,
config.getMinGradeLevel(), config.getMaxGradeLevel());
- insight.setScore(0.5);
+
+ StandardDeviation sd = new StandardDeviation(false);
+ double stddev = sd.evaluate(new double[] {
config.getMinGradeLevel(), config.getMaxGradeLevel() });
+ double dev = 0.0;
+ if (score > config.getMaxGradeLevel()) {
+ dev = score - config.getMaxGradeLevel();
+ } else {
+ dev = config.getMinGradeLevel() - score;
+ }
+ double calcScore = 1 - (dev / stddev) * .5;
+ if (calcScore > 0) {
+ insight.setScore(calcScore);
+ } else {
+ insight.setScore(0.0);
+ }
insight.setPrimaryMessage(Message.warn(dictionary.get(I18N_KEY_READABILITY_RESULT_WARN,
new Object[] { config.getMinGradeLevel(),
config.getMaxGradeLevel(), scoreStr })));
} else {
diff --git a/core/src/main/resources/OSGI-INF/l10n/bundle.properties
b/core/src/main/resources/OSGI-INF/l10n/bundle.properties
index 168dc90..c1d3250 100644
--- a/core/src/main/resources/OSGI-INF/l10n/bundle.properties
+++ b/core/src/main/resources/OSGI-INF/l10n/bundle.properties
@@ -65,7 +65,7 @@ replacement.char.description=A character with which to
replace any non-allowed \
characters in the name
# User Generated Content
-ugc.name=User Generated Content
+ugc.name=Apache Sling CMS User Generated Content
ugc.description=Service for creating buckets of User Generated Content
ugcRoot.name=UGC Root
@@ -79,7 +79,7 @@ a bucket of bob and a path depth of 1, this would yield a
path like: bob/1/123.
This can be overridden by the path depth in the UGCBucketConfig.
# Path Suggestion Servlet
-pathsuggestionservlet.name=Path Suggestion Servlet
+pathsuggestionservlet.name=Apache Sling CMS Path Suggestion Servlet
pathsuggestionservlet.description=A servlet for providing suggested paths
based \
on a stemmed path
@@ -89,7 +89,7 @@ suggestion servlet. Each filter is provided in the format
[name]=[type1],[type2]
Each child node of the parent resource to the provided path will be checked to
see \
if the node is one of the provided types (or a sub-type) for the filter.
-readability.config.name=Readability Service
+readability.config.name=Apache Sling CMS - Insights Readability Service
readability.config.description=A service for calculating the readability of
text using a number of different readability \
algorithms
@@ -109,4 +109,13 @@ readability.param.complexitymin.name=Minimum Complexity
readability.param.complexitymin.description=The minimum number of syllables
for a word to be considered complex
readability.param.iswordexp.name=Is Word Expression
-readability.param.iswordexp.description=A regular expression for detecting if
a string is a word
\ No newline at end of file
+readability.param.iswordexp.description=A regular expression for detecting if
a string is a word
+
+pagespeed.config.name=Apache Sling CMS - Insights Page Speed
+pagespeed.config.description=Uses Google Page Speed Insights to estimate the
perceived speed of a particular page
+
+pagespeed.param.enabled.name=Enabled
+pagespeed.param.enabled.description=Whether or not the Page Speed Insight
service is enabled
+
+pagespeed.param.apikey.name=API Key
+pagespeed.param.apikey.description=The API key Google Cloud, see:
https://developers.google.com/speed/docs/insights/
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 2003a1a..8b75a63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,6 +189,20 @@
<version>4.2.0</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math</artifactId>
+ <version>2.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.5</version>
+ <scope>provided</scope>
+ </dependency>
+
+
</dependencies>
</dependencyManagement>
diff --git a/ui/src/main/resources/jcr_root/libs/sling-cms/i18n.json
b/ui/src/main/resources/jcr_root/libs/sling-cms/i18n.json
index 48f47f4..b0edf2a 100644
--- a/ui/src/main/resources/jcr_root/libs/sling-cms/i18n.json
+++ b/ui/src/main/resources/jcr_root/libs/sling-cms/i18n.json
@@ -10,6 +10,36 @@
],
"jcr:language": "en",
"sling:resourceType": "sling-cms/components/cms/blank",
+ "slingcms-htmlvalidator-danger":{
+ "jcr:primaryType": "sling:MessageEntry",
+ "sling:message": "There were {0} validation errors and {1}
warnings",
+ "sling:key": "slingcms.htmlvalidator.danger"
+ },
+ "slingcms-htmlvalidator-success":{
+ "jcr:primaryType": "sling:MessageEntry",
+ "sling:message": "HTML Validation successful!",
+ "sling:key": "slingcms.htmlvalidator.success"
+ },
+ "slingcms-htmlvalidator-warn":{
+ "jcr:primaryType": "sling:MessageEntry",
+ "sling:message": "There were {0} validation warnings",
+ "sling:key": "slingcms.htmlvalidator.warn"
+ },
+ "slingcms-pagespeed-danger": {
+ "jcr:primaryType": "sling:MessageEntry",
+ "sling:message": "This website is much slower than average, page
performance must be optimized",
+ "sling:key": "slingcms.pagespeed.danger"
+ },
+ "slingcms-pagespeed-warn": {
+ "jcr:primaryType": "sling:MessageEntry",
+ "sling:message": "This website is slower than average, page
performance should be optimized",
+ "sling:key": "slingcms.pagespeed.warn"
+ },
+ "slingcms-pagespeed-success": {
+ "jcr:primaryType": "sling:MessageEntry",
+ "sling:message": "This website has acceptable performance",
+ "sling:key": "slingcms.pagespeed.success"
+ },
"slingcms-readability-danger": {
"jcr:primaryType": "sling:MessageEntry",
"sling:message": "Failed to calculate readability for {1}",
