Author: thomasm
Date: Fri May 17 09:57:47 2019
New Revision: 1859424
URL: http://svn.apache.org/viewvc?rev=1859424&view=rev
Log:
OAK-8294 Make it possible to mark a (customer) query as bad and skip it
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryValidator.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryValidatorTest.java
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/package-info.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java?rev=1859424&r1=1859423&r2=1859424&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/QueryEngineSettingsMBean.java
Fri May 17 09:57:47 2019
@@ -86,4 +86,32 @@ public interface QueryEngineSettingsMBea
void setFastQuerySize(boolean fastQuerySize);
+ /**
+ * Set or remove a query validator pattern.
+ *
+ * @param key the key
+ * @param pattern the regular expression pattern (empty to remove the
+ * pattern)
+ * @param comment a comment
+ * @param failQuery whether matching queries should fail (true) or just log
+ * a warning (false)
+ */
+ @Description("Set or remove a query validator pattern.")
+ void setQueryValidatorPattern(
+ @Description("the key")
+ @Name("key")
+ String key,
+ @Description("the regular expression pattern (empty to remove the
pattern)")
+ @Name("pattern")
+ String pattern,
+ @Description("a comment")
+ @Name("comment")
+ String comment,
+ @Description("whether matching queries should fail (true) or just
log a warning (false)")
+ @Name("failQuery")
+ boolean failQuery);
+
+ @Description("Get the query validator data as a JSON string.")
+ String getQueryValidatorJson();
+
}
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/package-info.java?rev=1859424&r1=1859423&r2=1859424&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/jmx/package-info.java
Fri May 17 09:57:47 2019
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-@Version("4.7.1")
+@Version("4.8.0")
package org.apache.jackrabbit.oak.api.jmx;
import org.osgi.annotation.versioning.Version;
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java?rev=1859424&r1=1859423&r2=1859424&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java
Fri May 17 09:57:47 2019
@@ -757,6 +757,8 @@ public class Oak {
regs.add(registerMBean(whiteboard, QueryStatsMBean.class,
queryEngineSettings.getQueryStats(), QueryStatsMBean.TYPE,
"Oak Query Statistics (Extended)"));
+ queryEngineSettings.unwrap().getQueryValidator().init(store);
+
// add index hooks later to prevent the OakInitializer to do excessive
indexing
commitHooks.add(new EditorHook(new IndexUpdateProvider(indexEditors,
failOnMissingIndexProvider)));
@@ -931,6 +933,16 @@ public class Oak {
settings.setFastQuerySize(fastQuerySize);
}
+ @Override
+ public void setQueryValidatorPattern(String key, String pattern,
String comment, boolean failQuery) {
+ settings.getQueryValidator().setPattern(key, pattern, comment,
failQuery);
+ }
+
+ @Override
+ public String getQueryValidatorJson() {
+ return settings.getQueryValidator().getJson();
+ }
+
public QueryStatsMBean getQueryStats() {
return settings.getQueryStats();
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java?rev=1859424&r1=1859423&r2=1859424&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineImpl.java
Fri May 17 09:57:47 2019
@@ -157,6 +157,8 @@ public abstract class QueryEngineImpl im
NodeTypeInfoProvider nodeTypes = context.getNodeTypeInfoProvider();
QueryEngineSettings settings = context.getSettings();
+ settings.getQueryValidator().checkStatement(statement);
+
QueryExecutionStats stats =
settings.getQueryStatsReporter().getQueryExecution(statement, language);
SQL2Parser parser = new SQL2Parser(mapper, nodeTypes, settings, stats);
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java?rev=1859424&r1=1859423&r2=1859424&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java
Fri May 17 09:57:47 2019
@@ -83,6 +83,8 @@ public class QueryEngineSettings impleme
*/
private final StatisticsProvider statisticsProvider;
+ private final QueryValidator queryValidator = new QueryValidator();
+
public QueryEngineSettings() {
statisticsProvider = StatisticsProvider.NOOP;
}
@@ -157,6 +159,20 @@ public class QueryEngineSettings impleme
}
@Override
+ public void setQueryValidatorPattern(String key, String pattern, String
comment, boolean failQuery) {
+ queryValidator.setPattern(key, pattern, comment, failQuery);
+ }
+
+ @Override
+ public String getQueryValidatorJson() {
+ return queryValidator.getJson();
+ }
+
+ public QueryValidator getQueryValidator() {
+ return queryValidator;
+ }
+
+ @Override
public String toString() {
return "QueryEngineSettings{" +
"limitInMemory=" + limitInMemory +
@@ -168,4 +184,4 @@ public class QueryEngineSettings impleme
'}';
}
-}
\ No newline at end of file
+}
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryValidator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryValidator.java?rev=1859424&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryValidator.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryValidator.java
Fri May 17 09:57:47 2019
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law
+ * or agreed to in writing, software distributed under the License is
+ * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.apache.jackrabbit.oak.query;
+
+import java.text.ParseException;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.regex.Pattern;
+
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A validator for query. Invalid queries either log a warning, or throw an
+ * exception when trying to execute.
+ */
+public class QueryValidator {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(QueryValidator.class);
+
+ /**
+ * The name of the query validator node.
+ */
+ public static final String QUERY_VALIDATOR = "queryValidator";
+
+ /**
+ * The next time to log a warning for a query, in milliseconds.
+ */
+ private static final int NEXT_LOG_MILLIS = 10 * 1000;
+
+ /**
+ * The map of invalid query patterns.
+ */
+ private final ConcurrentSkipListMap<String, ProblematicQueryPattern> map =
new ConcurrentSkipListMap<>();
+
+ /**
+ * Add a pattern.
+ *
+ * @param key the key
+ * @param pattern the pattern regular expression - if empty, the entry is
removed
+ * @param comment the comment
+ * @param failQuery - if true, trying to run such a query will fail;
+ * otherwise the queries that will work, but will log a warning.
+ * A warning is logged at most once every 10 seconds.
+ */
+ public void setPattern(String key, String pattern, String comment, boolean
failQuery) {
+ LOG.debug("set pattern key={} pattern={} comment={} failQuery={}",
key, pattern, comment, failQuery);
+ if (pattern.isEmpty()) {
+ map.remove(key);
+ } else {
+ ProblematicQueryPattern p = new ProblematicQueryPattern(key,
pattern, comment, failQuery);
+ map.put(key, p);
+ }
+ }
+
+ /**
+ * Get the current set of pattern data.
+ *
+ * @return the json representation
+ */
+ public String getJson() {
+ JsopBuilder b = new JsopBuilder().array();
+ for (ProblematicQueryPattern p : map.values()) {
+ b.newline().encodedValue(p.getJson());
+ }
+ return b.endArray().toString();
+ }
+
+ /**
+ * Check if a query is valid. It is either valid, logs a warning, or
throws a exception if invalid.
+ *
+ * @param statement the query statement
+ * @throws ParseException if it is invalid
+ */
+ public void checkStatement(String statement) throws ParseException {
+ if (map.isEmpty()) {
+ // the normal case: no patterns defined
+ return;
+ }
+ for (ProblematicQueryPattern p : map.values()) {
+ p.checkStatement(statement);
+ }
+ }
+
+ public void init(NodeStore store) {
+ NodeState def =
store.getRoot().getChildNode(IndexConstants.INDEX_DEFINITIONS_NAME).
+ getChildNode(QUERY_VALIDATOR);
+ if (!def.exists()) {
+ return;
+ }
+ for (ChildNodeEntry e : def.getChildNodeEntries()) {
+ String key = e.getName();
+ NodeState n = e.getNodeState();
+ PropertyState p = n.getProperty("pattern");
+ if (p == null) {
+ continue;
+ }
+ String pattern;
+ if (p.isArray()) {
+ int len = p.count();
+ StringBuilder buff = new StringBuilder();
+ for (int i = 0; i < len; i++) {
+ if (buff.length() > 0) {
+ buff.append(".*");
+ }
+ buff.append(Pattern.quote(p.getValue(Type.STRING, i)));
+ }
+ pattern = buff.toString();
+ } else {
+ pattern = p.getValue(Type.STRING);
+ }
+ String comment = n.getProperty("comment").getValue(Type.STRING);
+ boolean failQuery =
n.getProperty("failQuery").getValue(Type.BOOLEAN);
+ if (pattern != null && comment != null) {
+ setPattern(key, pattern, comment, failQuery);
+ }
+ }
+ }
+
+ /**
+ * A query pattern definition.
+ */
+ private static class ProblematicQueryPattern {
+
+ private final String key;
+ private final String pattern;
+ private final String comment;
+ private final Pattern compiledPattern;
+ private final boolean failQuery;
+ private long executedLast;
+ private long executedCount;
+
+ ProblematicQueryPattern(String key, String pattern, String comment,
boolean failQuery) {
+ this.key = key;
+ this.pattern = pattern;
+ this.comment = comment;
+ this.compiledPattern = Pattern.compile(pattern);
+ this.failQuery = failQuery;
+ }
+
+ void checkStatement(String statement) throws ParseException {
+ if (!compiledPattern.matcher(statement).matches()) {
+ return;
+ }
+ executedCount++;
+ long previousExecuted = executedLast;
+ long now = System.currentTimeMillis();
+ executedLast = now;
+ if (failQuery) {
+ String message = "Query is blacklisted: statement=" +
statement + " pattern=" + pattern;
+ ParseException p = new ParseException(message, 0);
+ LOG.warn(message, p);
+ throw p;
+ } else {
+ String message = "Query is questionable, but executed:
statement=" + statement + " pattern=" + pattern;
+ if (previousExecuted + NEXT_LOG_MILLIS < now) {
+ LOG.warn(message, new Exception("QueryValidator"));
+ } else {
+ LOG.debug(message, new Exception("QueryValidator"));
+ }
+ }
+ }
+
+ String getJson() {
+ return new JsopBuilder().object().newline().
+ key("key").value(key).newline().
+ key("pattern").value(pattern).newline().
+ key("comment").value(comment).newline().
+ key("failQuery").value(failQuery).newline().
+ key("executedLast").value(
+ executedLast == 0 ? "" : new
java.sql.Timestamp(executedLast).toString()).newline().
+ key("executedCount").value(executedCount).newline().
+ endObject().toString();
+ }
+ }
+
+}
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryValidatorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryValidatorTest.java?rev=1859424&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryValidatorTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/query/QueryValidatorTest.java
Fri May 17 09:57:47 2019
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.query;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.text.ParseException;
+import java.util.List;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Test;
+import org.slf4j.event.Level;
+
+/**
+ * Tests the query validator.
+ */
+public class QueryValidatorTest {
+
+ @Test
+ public void empty() throws ParseException {
+ QueryValidator v = new QueryValidator();
+ // expected to be very fast
+ v.checkStatement("x");
+ v.setPattern("x", "x.*", "all", false);
+ v.setPattern("x", "", "", false);
+ v.checkStatement("x");
+ }
+
+ @Test
+ public void warning() throws ParseException {
+ QueryValidator v = new QueryValidator();
+ v.setPattern("x", "x.*", "all", false);
+ assertEquals("[\n" +
+ "{\n" +
+ "\"key\":\"x\"\n" +
+ ",\"pattern\":\"x.*\"\n" +
+ ",\"comment\":\"all\"\n" +
+ ",\"failQuery\":false\n" +
+ ",\"executedLast\":\"\"\n" +
+ ",\"executedCount\":0\n" +
+ "}]", v.getJson());
+ LogCustomizer customLogs =
LogCustomizer.forLogger(QueryValidator.class.getName()).enable(Level.WARN).create();
+ try {
+ customLogs.starting();
+ v.checkStatement("x1");
+ v.checkStatement("x2");
+ v.checkStatement("y");
+ List<String> logs = customLogs.getLogs();
+ assertEquals("[Query is questionable, but executed: statement=x1
pattern=x.*]", logs.toString());
+ } finally {
+ customLogs.finished();
+ }
+ }
+
+ @Test
+ public void error() throws ParseException {
+ QueryValidator v = new QueryValidator();
+ v.setPattern("x", "x.*", "all", true);
+ try {
+ v.checkStatement("x1");
+ fail();
+ } catch (ParseException e) {
+ // expected
+ }
+ v.checkStatement("y");
+ assertTrue(v.getJson().startsWith("[\n" +
+ "{\n" +
+ "\"key\":\"x\"\n" +
+ ",\"pattern\":\"x.*\"\n" +
+ ",\"comment\":\"all\"\n" +
+ ",\"failQuery\":true\n"));
+ assertTrue(v.getJson().indexOf("\"executedCount\":1") >= 0);
+ v.checkStatement("y");
+ try {
+ v.checkStatement("x2");
+ fail();
+ } catch (ParseException e) {
+ // expected
+ }
+ assertTrue(v.getJson().indexOf("\"executedCount\":2") >= 0);
+ }
+
+ @Test
+ public void initFromNodeStore() throws CommitFailedException {
+ QueryValidator v = new QueryValidator();
+ MemoryNodeStore ns = new MemoryNodeStore();
+ v.init(ns);
+ assertEquals("[]", v.getJson());
+ NodeBuilder rootBuilder = ns.getRoot().builder();
+ NodeBuilder b;
+ b = rootBuilder.child(IndexConstants.INDEX_DEFINITIONS_NAME).
+ child(QueryValidator.QUERY_VALIDATOR).child("ignored");
+ b = rootBuilder.child(IndexConstants.INDEX_DEFINITIONS_NAME).
+ child(QueryValidator.QUERY_VALIDATOR).child("test");
+ b.setProperty("pattern", "testPattern");
+ b.setProperty("comment", "testComment");
+ b.setProperty("failQuery", "true");
+ ns.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ v.init(ns);
+ assertEquals("[\n" +
+ "{\n" +
+ "\"key\":\"test\"\n" +
+ ",\"pattern\":\"testPattern\"\n" +
+ ",\"comment\":\"testComment\"\n" +
+ ",\"failQuery\":true\n" +
+ ",\"executedLast\":\"\"\n" +
+ ",\"executedCount\":0\n" +
+ "}]",
+ v.getJson());
+ b = rootBuilder.child(IndexConstants.INDEX_DEFINITIONS_NAME).
+ child(QueryValidator.QUERY_VALIDATOR).child("test");
+ b.setProperty("pattern", asList("select", "order by @x"),
Type.STRINGS);
+ b.setProperty("comment", "testComment");
+ b.setProperty("failQuery", "true");
+ ns.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ v.init(ns);
+ assertEquals("[\n" +
+ "{\n" +
+ "\"key\":\"test\"\n" +
+ ",\"pattern\":\"\\\\Qselect\\\\E.*\\\\Qorder by
@x\\\\E\"\n" +
+ ",\"comment\":\"testComment\"\n" +
+ ",\"failQuery\":true\n" +
+ ",\"executedLast\":\"\"\n" +
+ ",\"executedCount\":0\n" +
+ "}]",
+ v.getJson());
+ }
+
+}
