This is an automated email from the ASF dual-hosted git repository. arina pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7509dc46d08d7c936aada557e6237fe5fae3624b Author: Anton Gozhiy <[email protected]> AuthorDate: Mon Feb 17 17:24:45 2020 +0200 DRILL-7582: Moved Drillbits REST API communication to the back end layer closes #1999 --- exec/java-exec/pom.xml | 11 +++ .../java/org/apache/drill/exec/ExecConstants.java | 1 + .../apache/drill/exec/server/rest/DrillRoot.java | 15 +++ .../drill/exec/server/rest/StatusResources.java | 12 +++ .../apache/drill/exec/server/rest/WebUtils.java | 106 +++++++++++++++++++++ .../java-exec/src/main/resources/drill-module.conf | 3 + exec/java-exec/src/main/resources/rest/index.ftl | 90 +++++++---------- 7 files changed, 181 insertions(+), 57 deletions(-) diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml index 21fc9ff..96e0cec 100644 --- a/exec/java-exec/pom.xml +++ b/exec/java-exec/pom.xml @@ -37,6 +37,17 @@ <artifactId>hamcrest-core</artifactId> </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpasyncclient</artifactId> + <version>4.1.4</version> + <exclusions> + <exclusion> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.apache.kerby</groupId> <artifactId>kerb-client</artifactId> <version>${kerby.version}</version> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 1276384..7532471 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -212,6 +212,7 @@ public final class ExecConstants { public static final String HTTP_JETTY_SERVER_SELECTORS = "drill.exec.http.jetty.server.selectors"; public static final String HTTP_JETTY_SERVER_HANDLERS = "drill.exec.http.jetty.server.handlers"; public static final String HTTP_ENABLE_SSL = "drill.exec.http.ssl_enabled"; + public static final String HTTP_CLIENT_TIMEOUT = "drill.exec.http.client.timeout"; public static final String HTTP_CORS_ENABLED = "drill.exec.http.cors.enabled"; public static final String HTTP_CORS_ALLOWED_ORIGINS = "drill.exec.http.cors.allowedOrigins"; public static final String HTTP_CORS_ALLOWED_METHODS = "drill.exec.http.cors.allowedMethods"; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java index e3dad27..1910737 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRoot.java @@ -17,15 +17,18 @@ */ package org.apache.drill.exec.server.rest; +import java.net.URL; import java.util.Collection; import java.util.HashMap; import java.util.Map; import javax.annotation.security.PermitAll; import javax.annotation.security.RolesAllowed; import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -53,6 +56,7 @@ import org.apache.drill.exec.work.foreman.rm.DynamicResourceManager; import org.apache.drill.exec.work.foreman.rm.QueryQueue; import org.apache.drill.exec.work.foreman.rm.ResourceManager; import org.apache.drill.exec.work.foreman.rm.ThrottledResourceManager; +import org.apache.http.client.methods.HttpPost; import org.glassfish.jersey.server.mvc.Viewable; import com.fasterxml.jackson.annotation.JsonCreator; @@ -72,6 +76,8 @@ public class DrillRoot { SecurityContext sc; @Inject Drillbit drillbit; + @Inject + HttpServletRequest request; @GET @Produces(MediaType.TEXT_HTML) @@ -133,6 +139,15 @@ public class DrillRoot { } @POST + @Path("/gracefulShutdown/{hostname}") + @Produces(MediaType.APPLICATION_JSON) + @RolesAllowed(ADMIN_ROLE) + public String shutdownDrillbitByName(@PathParam("hostname") String hostname) throws Exception { + URL shutdownURL = WebUtils.getDrillbitURL(work, request, hostname, "/gracefulShutdown"); + return WebUtils.doHTTPRequest(new HttpPost(shutdownURL.toURI()), work.getContext().getConfig()); + } + + @POST @Path("/shutdown") @Produces(MediaType.APPLICATION_JSON) @RolesAllowed(ADMIN_ROLE) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java index a27d9c1..00397ce 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.rest; +import java.net.URL; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -32,6 +33,7 @@ import javax.ws.rs.FormParam; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; @@ -51,6 +53,7 @@ import org.apache.drill.exec.server.options.SystemOptionManager; import org.apache.drill.exec.server.rest.DrillRestServer.UserAuthEnabled; import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal; import org.apache.drill.exec.work.WorkManager; +import org.apache.http.client.methods.HttpGet; import org.glassfish.jersey.server.mvc.Viewable; import com.fasterxml.jackson.annotation.JsonCreator; @@ -64,6 +67,7 @@ public class StatusResources { public static final String REST_API_SUFFIX = ".json"; public static final String PATH_STATUS_JSON = "/status" + REST_API_SUFFIX; public static final String PATH_STATUS = "/status"; + public static final String PATH_METRICS = PATH_STATUS + "/metrics"; public static final String PATH_OPTIONS_JSON = "/options" + REST_API_SUFFIX; public static final String PATH_INTERNAL_OPTIONS_JSON = "/internal_options" + REST_API_SUFFIX; public static final String PATH_OPTIONS = "/options"; @@ -97,6 +101,14 @@ public class StatusResources { return ViewableWithPermissions.create(authEnabled.get(), "/rest/status.ftl", sc, getStatusJSON()); } + @GET + @Path(StatusResources.PATH_METRICS + "/{hostname}") + @Produces(MediaType.APPLICATION_JSON) + public String getMetrics(@PathParam("hostname") String hostname) throws Exception { + URL metricsURL = WebUtils.getDrillbitURL(work, request, hostname, StatusResources.PATH_METRICS); + return WebUtils.doHTTPRequest(new HttpGet(metricsURL.toURI()), work.getContext().getConfig()); + } + private List<OptionWrapper> getSystemOptionsJSONHelper(boolean internal) { List<OptionWrapper> options = new LinkedList<>(); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUtils.java index 4e1b99a..e34d2fd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUtils.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUtils.java @@ -17,13 +17,34 @@ */ package org.apache.drill.exec.server.rest; +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; +import org.apache.drill.exec.work.WorkManager; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.impl.nio.client.HttpAsyncClients; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.ssl.SSLContexts; + +import javax.net.ssl.SSLContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpSession; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URL; import java.security.SecureRandom; import java.util.Base64; +import java.util.stream.Collectors; public class WebUtils { + private static CloseableHttpAsyncClient httpClient; + /** * Retrieves the CSRF protection token from the HTTP request. * @@ -48,4 +69,89 @@ public class WebUtils { new SecureRandom().nextBytes(buffer); return Base64.getUrlEncoder().withoutPadding().encodeToString(buffer); } + + /** + * Build an URL of a remote Drillbit endpoint. + * + * @param work {@link WorkManager} instance needed to retrieve the Drillbit address. + * @param request {@link HttpServletRequest} instance needed to set the URL schema. + * @param hostname hostname of the Drillbit. + * @param path relative path to the endpoint. + * @return remote Drillbit endpoint URL. + * @throws RuntimeException if there is no Drillbit at the given hostname. + */ + static URL getDrillbitURL(WorkManager work, HttpServletRequest request, String hostname, String path) { + int drillbitPort = work.getContext().getAvailableBits().stream() + .filter(db -> db.getAddress().equals(hostname)) + .findAny() + .map(DrillbitEndpoint::getHttpPort) + .orElseThrow(() -> new RuntimeException("No such drillbit: " + hostname)); + try { + return new URL(request.getScheme(), hostname, drillbitPort, path); + } catch (MalformedURLException e) { + throw new IllegalStateException("Should never occur", e); + } + } + + /** + * Send an HTTP request and return response body as String. + * + * @param httpRequest {@link org.apache.http.client.methods.HttpGet} or {@link org.apache.http.client.methods.HttpPost} instance representing an HTTP request. + * @param drillConfig {@link DrillConfig} instance needed to setup HTTP Client. + * @return String response body. + * @throws Exception if unable to create HTTP client or in case of HTTP timeout. + */ + static String doHTTPRequest(HttpRequestBase httpRequest, DrillConfig drillConfig) throws Exception { + CloseableHttpAsyncClient httpClient = getHttpClient(drillConfig); + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + httpClient.execute(httpRequest, null) + .get() + .getEntity() + .getContent()))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + + /** + * Get singleton instance of an HTTP Client. + * + * @param drillConfig {@link DrillConfig} instance needed to retrieve Drill SSL parameters. + * @return HTTP Client instance. + * @throws Exception if unable to create an HTTP Client. + */ + private static CloseableHttpAsyncClient getHttpClient(DrillConfig drillConfig) throws Exception { + CloseableHttpAsyncClient localHttpClient = httpClient; + if (localHttpClient == null) { + synchronized (WebUtils.class) { + localHttpClient = httpClient; + if (httpClient == null) { + localHttpClient = createHttpClient(drillConfig); + localHttpClient.start(); + httpClient = localHttpClient; + } + } + } + return localHttpClient; + } + + private static CloseableHttpAsyncClient createHttpClient(DrillConfig drillConfig) throws Exception { + HttpAsyncClientBuilder clientBuilder = HttpAsyncClients.custom(); + if (drillConfig.getBoolean(ExecConstants.HTTP_ENABLE_SSL)) { + SSLContext sslContext = SSLContexts.custom() + .loadTrustMaterial(new TrustSelfSignedStrategy()) + .build(); + SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy( + sslContext, + new String[]{drillConfig.getString(ExecConstants.SSL_PROTOCOL)}, + null, + SSLIOSessionStrategy.getDefaultHostnameVerifier() + ); + clientBuilder.setSSLStrategy(sessionStrategy); + } + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(drillConfig.getInt(ExecConstants.HTTP_CLIENT_TIMEOUT)) + .build(); + clientBuilder.setDefaultRequestConfig(requestConfig); + return clientBuilder.build(); + } } diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index b365047..471f0f1 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -131,6 +131,9 @@ drill.exec: { }, http: { enabled: true, + client: { + timeout: 5000 + }, ssl_enabled: false, porthunt: false, port: 8047, diff --git a/exec/java-exec/src/main/resources/rest/index.ftl b/exec/java-exec/src/main/resources/rest/index.ftl index 10385c2..d8e6853 100644 --- a/exec/java-exec/src/main/resources/rest/index.ftl +++ b/exec/java-exec/src/main/resources/rest/index.ftl @@ -51,6 +51,8 @@ </div> </#if> + <#include "*/confirmationModals.ftl"> + <div class="row"> <div class="col-md-12"> <h3><span id="sizeLabel">Drillbits <span class="label label-primary" id="size">${model.getDrillbits()?size}</span> @@ -223,18 +225,6 @@ var size = $("#size").html(); reloadMetrics(); setInterval(reloadMetrics, refreshTime); - var hideShutdownBtns = setShutdownCtrl(); - - //Hide Shutdown buttons for remote HTTPS entries - function setShutdownCtrl() { - for (i = 1; i <= size; i++) { - let currentRow = $("#row-"+i); - if ( location.protocol == "https:" && (currentRow.find("#current").html() != "Current") ) { - //Hide Shutdown Button for remote nodes with HTTPS enabled - currentRow.find(".shutdownCtrl").css('display','none'); - } - } - } //Gets a refresh time for graceful shutdown function getRefreshTime() { @@ -350,17 +340,17 @@ currentRow.find("#shutdown").prop('disabled',true).css('opacity',0.5).css('cursor','not-allowed'); currentRow.find("#queriesCount").text(""); } else { - if (status_map[key] == "ONLINE") { + if (status_map[key] === "ONLINE") { currentRow.find("#status").text(status_map[key]).css('font-style','').prop('title',''); - //EnableShutdown IFF => !isAuthEnabled-&&-!HTTPS OR isAuthEnabled-&&-current - if ( ( !${model.isAuthEnabled()?c} && location.protocol != "https:" ) || ( ${model.shouldShowAdminInfo()?c} && currentRow.find("#current").html() == "Current" ) ) { + //EnableShutdown IFF => !isAuthEnabled OR isAuthEnabled-&&-current + if ( ( !${model.isAuthEnabled()?c} ) || ( ${model.shouldShowAdminInfo()?c} && currentRow.find("#current").html() === "Current" ) ) { currentRow.find("#shutdown").prop('disabled',false).css('opacity',1.0).css('cursor','pointer').attr('title',''); } } else { - if (currentRow.find("#current").html() == "Current") { + if (currentRow.find("#current").html() === "Current") { fillQueryCount(i); } - currentRow.find("#status").text(status_map[key]).css('font-style','').prop('title','');; + currentRow.find("#status").text(status_map[key]).css('font-style', '').prop('title', ''); } //Removing accounted key delete bitMap[key]; @@ -433,15 +423,9 @@ function shutdown(shutdownBtn) { let rowElem = $(shutdownBtn).parent().parent(); let hostAddr = $(rowElem).find('#address').contents().get(0).nodeValue.trim(); - let hostPort = $(rowElem).find('#httpPort').html(); - // Always use the host address from the url for the current Drillbit. For details refer DRILL-6663 - if ((rowElem.find("#current").html() == "Current")) { - hostAddr = location.hostname; - } - let host = hostAddr+":"+hostPort + let url = rowElem.find("#current").html() === "Current" ? "/gracefulShutdown" : "/gracefulShutdown/" + hostAddr; - if (confirm("Are you sure you want to shutdown Drillbit running on " + host + " node?")) { - let url = location.protocol + "//" + host + "/gracefulShutdown"; + showConfirmationDialog("Are you sure you want to shutdown Drillbit running on " + hostAddr + " node?", function() { let result = $.ajax({ type: 'POST', url: url, @@ -454,7 +438,7 @@ shutdownBtn.prop('disabled',true).css('opacity',0.5); } }); - } + }); } </#if> @@ -475,52 +459,44 @@ //Iterates through all the nodes for update function reloadMetrics() { - for (i = 1; i <= size; i++) { - //Skip metrics update for remote bits in HTTPS mode - if (i > 1 && location.protocol == "https:") { - break; - } - let currentRow = $("#row-"+i); - let address = ""; - //For 'current' bit, address is referred to by location.hostname instead of FQDN () - if (i == 1) { - address = location.hostname; - } else { - address = currentRow.find("#address").contents().get(0).nodeValue.trim(); - } - let httpPort = currentRow.find("#httpPort").contents().get(0).nodeValue.trim(); - updateMetricsHtml(address, httpPort, i); + for (i = 1; i <= size; i++) { + let currentRow = $("#row-" + i); + let address = ""; + let isCurrent = false; + //For 'current' bit, address is referred to by location.hostname instead of FQDN () + if (i === 1) { + isCurrent = true; + } else { + address = currentRow.find("#address").contents().get(0).nodeValue.trim(); } + updateMetricsHtml(address, isCurrent, i); + } } //Update memory - function updateMetricsHtml(drillbit,webport,idx) { - /* NOTE: For remote drillbits: - If Authentication or SSL is enabled; we'll assume we don't have valid certificates - */ - let remoteHost = location.protocol+"//"+drillbit+":"+webport; - // + function updateMetricsHtml(drillbit, isCurrent, idx) { + let drillbitURL = isCurrent ? "/status/metrics" : "/status/" + drillbit + "/metrics"; let result = $.ajax({ type: 'GET', - url: location.protocol+"//"+drillbit+":"+webport+"/status/metrics", + url: drillbitURL, dataType: "json", - error: function(data) { + error: function (data) { resetMetricsHtml(idx); }, - complete: function(data) { + complete: function (data) { if (typeof data.responseJSON == 'undefined') { - resetMetricsHtml(idx); - return; + resetMetricsHtml(idx); + return; } let metrics = data.responseJSON['gauges']; //Memory let usedHeap = metrics['heap.used'].value; - let maxHeap = metrics['heap.max'].value; + let maxHeap = metrics['heap.max'].value; let usedDirect = metrics['drill.allocator.root.used'].value; let peakDirect = metrics['drill.allocator.root.peak'].value; - let heapUsage = computeMemUsage(usedHeap,maxHeap); - let directUsage = computeMemUsage(usedDirect,peakDirect); - let rowElem = document.getElementById("row-"+idx); + let heapUsage = computeMemUsage(usedHeap, maxHeap); + let directUsage = computeMemUsage(usedDirect, peakDirect); + let rowElem = document.getElementById("row-" + idx); let heapElem = rowElem.getElementsByClassName("heap")[0]; heapElem.innerHTML = heapUsage; let directElem = rowElem.getElementsByClassName("direct")[0]; @@ -529,7 +505,7 @@ let dbitLoad = metrics['drillbit.load.avg'].value; let dbitLoadElem = rowElem.getElementsByClassName("bitload")[0]; if (dbitLoad >= 0) { - dbitLoadElem.innerHTML = parseFloat(Math.round(dbitLoad * 10000)/100).toFixed(2) + "%"; + dbitLoadElem.innerHTML = parseFloat(Math.round(dbitLoad * 10000) / 100).toFixed(2) + "%"; } else { dbitLoadElem.innerHTML = nAText; }
