This is an automated email from the ASF dual-hosted git repository. rombert pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/sling-org-apache-sling-testing-clients.git
The following commit(s) were added to refs/heads/master by this push: new 2988bbe SLING-7509 - Add QueryClient 2988bbe is described below commit 2988bbe2456c2cfd6beb62cdb94faa984ac4b8c8 Author: Valentin Olteanu <volte...@adobe.com> AuthorDate: Wed Feb 21 21:37:28 2018 +0100 SLING-7509 - Add QueryClient --- pom.xml | 50 +++++ .../sling/testing/clients/query/QueryClient.java | 218 +++++++++++++++++++++ .../sling/testing/clients/query/package-info.java | 25 +++ .../clients/query/servlet/QueryServlet.java | 168 ++++++++++++++++ .../testing/clients/query/QueryClientTest.java | 167 ++++++++++++++++ 5 files changed, 628 insertions(+) diff --git a/pom.xml b/pom.xml index 17c4562..2eac034 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,11 @@ <artifactId>org.apache.sling.xss</artifactId> <version>1.0.4</version> </dependency> + <dependency> + <groupId>org.ops4j.pax.tinybundles</groupId> + <artifactId>tinybundles</artifactId> + <version>3.0.0</version> + </dependency> <!-- For tests --> <dependency> @@ -158,5 +163,50 @@ <classifier>tests</classifier> <scope>test</scope> </dependency> + + <!-- Used by QueryServlet on server side --> + <dependency> + <groupId>org.apache.sling</groupId> + <artifactId>org.apache.sling.api</artifactId> + <version>2.16.0</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>servlet-api</artifactId> + <version>2.4</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>javax.jcr</groupId> + <artifactId>jcr</artifactId> + <version>2.0</version> + <scope>provided</scope> + </dependency> + <!-- + The dependency below is a duplicate of org.codehaus.jackson, + which was renamed to com.fasterxml.jackson.core. Yet, because of + package name changes, we cannot automatically switch because it would + cause a major change of the API. The new packages are currently needed for + the query servlet, but everything should be updated at some point. + --> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>2.9.4</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.9.4</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.osgi</groupId> + <artifactId>org.osgi.compendium</artifactId> + <version>5.0.0</version> + <scope>provided</scope> + </dependency> </dependencies> </project> diff --git a/src/main/java/org/apache/sling/testing/clients/query/QueryClient.java b/src/main/java/org/apache/sling/testing/clients/query/QueryClient.java new file mode 100644 index 0000000..a6b5d2e --- /dev/null +++ b/src/main/java/org/apache/sling/testing/clients/query/QueryClient.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sling.testing.clients.query; + +import org.apache.http.NameValuePair; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.sling.testing.clients.ClientException; +import org.apache.sling.testing.clients.SlingClient; +import org.apache.sling.testing.clients.SlingClientConfig; +import org.apache.sling.testing.clients.SlingHttpResponse; +import org.apache.sling.testing.clients.osgi.OsgiConsoleClient; +import org.apache.sling.testing.clients.query.servlet.QueryServlet; +import org.apache.sling.testing.clients.util.JsonUtils; +import org.apache.sling.testing.clients.util.URLParameterBuilder; +import org.codehaus.jackson.JsonNode; +import org.ops4j.pax.tinybundles.core.TinyBundles; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.apache.http.HttpStatus.SC_NOT_FOUND; +import static org.apache.http.HttpStatus.SC_OK; + +/** + * <p>Sling client for performing oak queries.</p> + * + * <p>Uses a custom servlet {@link QueryServlet} to execute the query on the server + * and return the results as a json. If the servlet is not yet present, it automatically + * installs it and creates the corresponding nodes</p> + * + * <p>The servlet is exposed under {@value QueryServlet#SERVLET_PATH}.</p> + * + * <p>The servlet is not automatically uninstalled to avoid too much noise on the instance. + * The caller should take care of it, if needed, by calling {@link #uninstallServlet()}</p> + */ +public class QueryClient extends SlingClient { + + /** + * Query types, as defined in {@code org.apache.jackrabbit.oak.query.QueryEngineImpl} + */ + public enum QueryType { + SQL2("JCR-SQL2"), + SQL("sql"), + XPATH("xpath"), + JQOM("JCR-JQOM"); + + private final String name; + + QueryType(String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } + } + + private static final Logger LOG = LoggerFactory.getLogger(QueryClient.class); + + private static final String BUNDLE_BSN = "org.apache.sling.testing.clients.query"; + private static final String BUNDLE_NAME = "Sling Testing Clients Query Servlet"; + private static final String BUNDLE_VERSION = "1.0.0"; + + private static final long BUNDLE_START_TIMEOUT = TimeUnit.SECONDS.toMillis(10); + + /** + * Constructor used by adaptTo + * + * @param http underlying HttpClient + * @param config config state + * @throws ClientException if the client cannot be created + */ + public QueryClient(CloseableHttpClient http, SlingClientConfig config) throws ClientException { + super(http, config); + } + + /** + * Convenience constructor + * + * @param url host url + * @param user username + * @param password password + * @throws ClientException if the client cannot be constructed + */ + public QueryClient(URI url, String user, String password) throws ClientException { + super(url, user, password); + } + + /** + * Executes a query on the server and returns the results as a json + * + * @param query query to be executed + * @param type type of the query + * @return the results in json as exported by {@link QueryServlet} + * @throws ClientException if the request failed to execute + */ + public JsonNode doQuery(final String query, final QueryType type) throws ClientException, InterruptedException { + return doQuery(query, type, true, false); + } + + /** + * Executes a query on the server and returns only the number of rows in the result + * + * @param query query to be executed + * @param type type of the query + * @return total results returned by the query + * @throws ClientException if the request failed to execute + */ + public long doCount(final String query, final QueryType type) throws ClientException, InterruptedException { + return doQuery(query, type, false, false).get("total").getLongValue(); + } + + /** + * Retrieves the plan of the query. Useful for determining which index is used + * + * @param query query to be executed + * @param type type of the query + * @return total results returned by the query + * @throws ClientException if the request failed to execute + */ + public String getPlan(final String query, final QueryType type) throws ClientException, InterruptedException { + return doQuery(query, type, false, true).get("plan").toString(); + } + + protected JsonNode doQuery(final String query, final QueryType type, final boolean showResults, final boolean explain) + throws ClientException, InterruptedException { + + List<NameValuePair> params = URLParameterBuilder.create() + .add("query", query) + .add("type", type.toString()) + .add("showresults", Boolean.toString(showResults)) + .add("explain", Boolean.toString(explain)) + .getList(); + + try { + // try optimistically to execute the query + SlingHttpResponse response = this.doGet(QueryServlet.SERVLET_PATH, params, SC_OK); + return JsonUtils.getJsonNodeFromString(response.getContent()); + } catch (ClientException e) { + if (e.getHttpStatusCode() == SC_NOT_FOUND) { + LOG.info("Could not find query servlet, will try to install it"); + installServlet(); + LOG.info("Retrying the query"); + SlingHttpResponse response = this.doGet(QueryServlet.SERVLET_PATH, params, SC_OK); + return JsonUtils.getJsonNodeFromString(response.getContent()); + } else { + throw e; + } + } + } + + /** + * <p>Installs the servlet to be able to perform queries.</p> + * + * <p>By default, methods of this client automatically install the servlet if needed, + * so there is no need to explicitly call from outside</p> + * + * @throws ClientException if the installation fails + */ + public QueryClient installServlet() throws ClientException, InterruptedException { + InputStream bundleStream = TinyBundles.bundle() + .set("Bundle-SymbolicName", BUNDLE_BSN) + .set("Bundle-Version", BUNDLE_VERSION) + .set("Bundle-Name", BUNDLE_NAME) + .add(QueryServlet.class) + .build(TinyBundles.withBnd()); + + try { + File bundleFile = File.createTempFile(BUNDLE_BSN + "-" + BUNDLE_VERSION, ".jar"); + Files.copy(bundleStream, bundleFile.toPath(), REPLACE_EXISTING); + + adaptTo(OsgiConsoleClient.class).installBundle(bundleFile, true); + adaptTo(OsgiConsoleClient.class).waitBundleStarted(BUNDLE_BSN, BUNDLE_START_TIMEOUT, 100); + + LOG.info("query servlet installed at {}", getUrl(QueryServlet.SERVLET_PATH)); + } catch (IOException e) { + throw new ClientException("Failed to create the query servlet bundle", e); + } catch (TimeoutException e) { + throw new ClientException("The query servlet bundle did not successfully start", e); + } + + return this; + } + + /** + * Deletes all the resources created by {@link #installServlet()} + * + * @throws ClientException if any of the resources fails to uninstall + */ + public QueryClient uninstallServlet() throws ClientException { + adaptTo(OsgiConsoleClient.class).uninstallBundle(BUNDLE_BSN); + return this; + } +} diff --git a/src/main/java/org/apache/sling/testing/clients/query/package-info.java b/src/main/java/org/apache/sling/testing/clients/query/package-info.java new file mode 100644 index 0000000..cf0283c --- /dev/null +++ b/src/main/java/org/apache/sling/testing/clients/query/package-info.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Query tools leveraging javax.jcr.query + */ +@Version("0.1.0") +package org.apache.sling.testing.clients.query; + +import org.osgi.annotation.versioning.Version; diff --git a/src/main/java/org/apache/sling/testing/clients/query/servlet/QueryServlet.java b/src/main/java/org/apache/sling/testing/clients/query/servlet/QueryServlet.java new file mode 100644 index 0000000..54cdc1f --- /dev/null +++ b/src/main/java/org/apache/sling/testing/clients/query/servlet/QueryServlet.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sling.testing.clients.query.servlet; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.sling.api.SlingHttpServletRequest; +import org.apache.sling.api.SlingHttpServletResponse; +import org.apache.sling.api.servlets.SlingSafeMethodsServlet; +import org.osgi.service.component.annotations.Component; + +import javax.jcr.Session; +import javax.jcr.query.*; +import javax.servlet.Servlet; +import javax.servlet.ServletException; +import java.io.IOException; +import java.util.Date; + +import static org.apache.sling.api.servlets.ServletResolverConstants.SLING_SERVLET_METHODS; +import static org.apache.sling.api.servlets.ServletResolverConstants.SLING_SERVLET_PATHS; + +@Component( + name = QueryServlet.SERVLET_NAME, + service = {Servlet.class}, + property = { + SLING_SERVLET_PATHS + "=" + QueryServlet.SERVLET_PATH, + SLING_SERVLET_METHODS + "=GET" + } +) +public class QueryServlet extends SlingSafeMethodsServlet { + private static final long serialVersionUID = 1L; + + public static final String SERVLET_PATH = "/system/testing/query"; + public static final String SERVLET_NAME = "Sling Testing Clients Query Servlet"; + + @Override + protected void doGet(SlingHttpServletRequest request, SlingHttpServletResponse response) + throws ServletException, IOException { + + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + + try { + final QueryManager qm = request.getResourceResolver().adaptTo(Session.class) + .getWorkspace().getQueryManager(); + + long before = 0; + long after = 0; + long total = 0; + + String query = request.getParameter("query"); + String type = request.getParameter("type"); + + // default for showResults is true, unless parameter is matching exactly "false" + boolean showResults = !("false".equalsIgnoreCase(request.getParameter("showresults"))); + // default for explainQuery is false, unless parameter is present and is not matching "false" + String explainParam = request.getParameter("explain"); + boolean explainQuery = (explainParam != null) && !("false".equalsIgnoreCase(explainParam)); + + boolean tidy = false; + for (String selector : request.getRequestPathInfo().getSelectors()) { + if ("tidy".equals(selector)) { + tidy = true; + } + } + + if ((query == null) || query.equals("") || (type == null) || type.equals("")) { + response.sendError(400, "Parameters query and type are required"); // invalid request + return; + } + + // prepare + if (explainQuery) { + query = "explain " + query; + } + + Query q = qm.createQuery(query, type); + + // execute + before = new Date().getTime(); + QueryResult result = q.execute(); + after = new Date().getTime(); + + // collect results + String firstSelector = null; + if (result.getSelectorNames().length > 1) { + firstSelector = result.getSelectorNames()[0]; + try { + String[] columnNames = result.getColumnNames(); + if (columnNames.length > 0) { + String firstColumnName = columnNames[0]; + int firstDot = firstColumnName.indexOf('.'); + if (firstDot > 0) { + firstSelector = firstColumnName.substring(0, firstDot); + } + } + } catch (Exception ignored) { + } + } + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode responseJson = mapper.createObjectNode(); + + if (explainQuery) { + responseJson.put("plan", result.getRows().nextRow().getValue("plan").getString()); + } else if (showResults) { + ArrayNode results = mapper.createArrayNode(); + + RowIterator rows = result.getRows(); + while (rows.hasNext()) { + Row row = rows.nextRow(); + String rowPath = (firstSelector != null) ? row.getPath(firstSelector) : row.getPath(); + String rowType = (firstSelector != null) + ? row.getNode(firstSelector).getPrimaryNodeType().getName() + : row.getNode().getPrimaryNodeType().getName(); + + ObjectNode rowJson = mapper.createObjectNode(); + rowJson.put("path", rowPath); + rowJson.put("type", rowType); + results.add(rowJson); + + total++; + } + + responseJson.set("results", results); + } else { + // only count results + RowIterator rows = result.getRows(); + while (rows.hasNext()) { + rows.nextRow(); + total++; + } + } + + responseJson.put("total", total); + responseJson.put("time", after - before); + + if (tidy) { + response.getWriter().write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(responseJson)); + } else { + response.getWriter().write(responseJson.toString()); + } + + } catch (InvalidQueryException e) { + // Consider InvalidQueryException as an invalid request instead of sending 500 server error + response.sendError(400, e.getMessage()); + e.printStackTrace(response.getWriter()); + } catch (final Exception e) { + response.sendError(500, e.getMessage()); + e.printStackTrace(response.getWriter()); + } + } +} diff --git a/src/test/java/org/apache/sling/testing/clients/query/QueryClientTest.java b/src/test/java/org/apache/sling/testing/clients/query/QueryClientTest.java new file mode 100644 index 0000000..8e8d6f5 --- /dev/null +++ b/src/test/java/org/apache/sling/testing/clients/query/QueryClientTest.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.sling.testing.clients.query; + +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHttpEntityEnclosingRequest; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpRequestHandler; +import org.apache.sling.testing.clients.ClientException; +import org.apache.sling.testing.clients.HttpServerRule; +import org.codehaus.jackson.JsonNode; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +public class QueryClientTest { + private static final Logger LOG = LoggerFactory.getLogger(QueryClientTest.class); + + private static final String QUERY_PATH = "/system/testing/query"; // same as in QueryServlet + private static final String BUNDLE_PATH = "/system/console/bundles/org.apache.sling.testing.clients.query"; + private static final String QUERY_RESPONSE = "{\"total\": 1234,\"time\": 1}"; + private static final String EXPLAIN_RESPONSE = "{\"plan\": \"some plan\",\"time\": 1}"; + private static final String JSON_BUNDLE = "{\n" + + " \"status\": \"Bundle information: 546 bundles in total, 537 bundles active, 8 bundles active fragments, 1 bundle resolved.\",\n" + + " \"s\": [\n" + + " 546,\n" + + " 537,\n" + + " 8,\n" + + " 1,\n" + + " 0\n" + + " ],\n" + + " \"data\": [\n" + + " {\n" + + " \"id\": 560,\n" + + " \"name\": \"Query servlet for testing\",\n" + + " \"fragment\": false,\n" + + " \"stateRaw\": 32,\n" + + " \"state\": \"Active\",\n" + + " \"version\": \"1.0.0\",\n" + + " \"symbolicName\": \"org.apache.sling.testing.clients.query\",\n" + + " \"category\": \"\"\n" + + " }\n" + + " ]\n" + + "}"; + + @ClassRule + public static HttpServerRule httpServer = new HttpServerRule() { + @Override + protected void registerHandlers() throws IOException { + + // Normal query request + serverBootstrap.registerHandler(QUERY_PATH, new HttpRequestHandler() { + @Override + public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { + List<NameValuePair> parameters = URLEncodedUtils.parse( + request.getRequestLine().getUri(), Charset.defaultCharset()); + + for (NameValuePair parameter : parameters) { + if (parameter.getName().equals("explain") && !parameter.getValue().equals("false")) { + response.setEntity(new StringEntity(EXPLAIN_RESPONSE)); + return; + } + } + + response.setEntity(new StringEntity(QUERY_RESPONSE)); + } + }); + + // Install servlet + serverBootstrap.registerHandler("/system/console/bundles", new HttpRequestHandler() { + @Override + public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { + // is install (post) or checking status (get) + if (request instanceof BasicHttpEntityEnclosingRequest) { + response.setStatusCode(302); + } else { + response.setStatusCode(200); + } + } + }); + + // Check bundle status + serverBootstrap.registerHandler(BUNDLE_PATH + ".json", new HttpRequestHandler() { + @Override + public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { + response.setEntity(new StringEntity(JSON_BUNDLE)); + } + }); + + // Uninstall bundle + serverBootstrap.registerHandler(BUNDLE_PATH, new HttpRequestHandler() { + @Override + public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { + response.setStatusCode(200); + } + }); + } + }; + + private static QueryClient client; + + public QueryClientTest() throws ClientException { + client = new QueryClient(httpServer.getURI(), "admin", "admin"); + // for testing an already running instance + // client = new QueryClient(java.net.URI.create("http://localhost:8080"), "admin", "admin"); + } + + @Test + public void testInstallServlet() throws ClientException, InterruptedException { + client.installServlet(); + } + + @Test + public void testDoQuery() throws ClientException, InterruptedException { + JsonNode response = client.doQuery("SELECT * FROM [nt:file] WHERE ISDESCENDANTNODE([/etc/])", +// JsonNode response = client.doQuery("SELECT * FROM [cq:Tag] WHERE ISDESCENDANTNODE([/etc/])", + QueryClient.QueryType.SQL2); + LOG.info(response.toString()); + Assert.assertNotEquals(0, response.get("total").getLongValue()); + } + + @Test + public void testDoCount() throws ClientException, InterruptedException { + long results = client.doCount("SELECT * FROM [nt:file] WHERE ISDESCENDANTNODE([/etc/])", + QueryClient.QueryType.SQL2); + LOG.info("results={}", results); + Assert.assertNotEquals(0, results); + } + + @Test + public void testGetPlan() throws ClientException, InterruptedException { + String plan = client.getPlan("SELECT * FROM [nt:file] WHERE ISDESCENDANTNODE([/etc/])", + QueryClient.QueryType.SQL2); + LOG.info("plan={}", plan); + Assert.assertNotEquals("", plan); + } + + @Test + public void testUninstallServlet() throws ClientException { + client.uninstallServlet(); + } +} -- To stop receiving notification emails like this one, please contact romb...@apache.org.