This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 70ecda4952 Adding endpoint to download local log files for each
component (#9259)
70ecda4952 is described below
commit 70ecda495281a6922b725c182f27cb010530b5f2
Author: Xiang Fu <[email protected]>
AuthorDate: Fri Aug 26 19:08:09 2022 -0700
Adding endpoint to download local log files for each component (#9259)
---
.../broker/api/resources/PinotBrokerLogger.java | 35 ++++++
.../broker/broker/BrokerAdminApiApplication.java | 6 +
.../pinot/common/utils/LoggerFileServer.java | 76 ++++++++++++
.../pinot/common/utils/config/InstanceUtils.java | 21 ++++
.../apache/pinot/common/utils/http/HttpClient.java | 2 +-
.../pinot/common/utils/LoggerFileServerTest.java | 79 ++++++++++++
.../pinot/controller/BaseControllerStarter.java | 5 +
.../api/resources/PinotControllerLogger.java | 138 +++++++++++++++++++++
.../pinot/minion/MinionAdminApiApplication.java | 5 +
.../minion/api/resources/PinotMinionLogger.java | 35 ++++++
.../server/api/resources/PinotServerLogger.java | 35 ++++++
.../server/starter/helix/AdminApiApplication.java | 5 +
.../apache/pinot/spi/utils/CommonConstants.java | 5 +
13 files changed, 446 insertions(+), 1 deletion(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerLogger.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerLogger.java
index acdaf3cbbb..849807f42d 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerLogger.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/api/resources/PinotBrokerLogger.java
@@ -25,8 +25,11 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
@@ -37,6 +40,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.common.utils.LoggerUtils;
import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_KEY;
@@ -51,6 +55,9 @@ import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_K
@Path("/")
public class PinotBrokerLogger {
+ @Inject
+ private LoggerFileServer _loggerFileServer;
+
@GET
@Path("/loggers")
@Produces(MediaType.APPLICATION_JSON)
@@ -80,4 +87,32 @@ public class PinotBrokerLogger {
@ApiParam(value = "Logger level") @QueryParam("level") String level) {
return LoggerUtils.setLoggerLevel(loggerName, level);
}
+
+ @GET
+ @Path("/loggers/files")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get all local log files")
+ public Set<String> getLocalLogFiles() {
+ try {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory doesn't exist",
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.getAllPaths();
+ } catch (IOException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/loggers/download")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ @ApiOperation(value = "Download a log file")
+ public Response downloadLogFile(
+ @ApiParam(value = "Log file path", required = true)
@QueryParam("filePath") String filePath) {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory is not configured",
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.downloadLogFile(filePath);
+ }
}
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
index 3978b65891..1a43f7cf25 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/broker/BrokerAdminApiApplication.java
@@ -32,6 +32,7 @@ import
org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.pinot.broker.requesthandler.BrokerRequestHandler;
import org.apache.pinot.broker.routing.BrokerRoutingManager;
import org.apache.pinot.common.metrics.BrokerMetrics;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.core.api.ServiceAutoDiscoveryFeature;
import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
import org.apache.pinot.core.transport.ListenerConfig;
@@ -51,6 +52,7 @@ public class BrokerAdminApiApplication extends ResourceConfig
{
private static final String RESOURCE_PACKAGE =
"org.apache.pinot.broker.api.resources";
public static final String PINOT_CONFIGURATION = "pinotConfiguration";
public static final String BROKER_INSTANCE_ID = "brokerInstanceId";
+
private final boolean _useHttps;
private HttpServer _httpServer;
@@ -78,6 +80,10 @@ public class BrokerAdminApiApplication extends
ResourceConfig {
bind(routingManager).to(BrokerRoutingManager.class);
bind(brokerRequestHandler).to(BrokerRequestHandler.class);
bind(brokerMetrics).to(BrokerMetrics.class);
+ String loggerRootDir =
brokerConf.getProperty(CommonConstants.Broker.CONFIG_OF_LOGGER_ROOT_DIR);
+ if (loggerRootDir != null) {
+ bind(new LoggerFileServer(loggerRootDir)).to(LoggerFileServer.class);
+ }
bind(brokerConf.getProperty(CommonConstants.Broker.CONFIG_OF_BROKER_ID)).named(BROKER_INSTANCE_ID);
}
});
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/LoggerFileServer.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/LoggerFileServer.java
new file mode 100644
index 0000000000..b0eac3f946
--- /dev/null
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/LoggerFileServer.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.utils;
+
+import com.google.common.base.Preconditions;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.TreeSet;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+
+
+/**
+ * Logger file server.
+ */
+public class LoggerFileServer {
+ private final File _loggerRootDir;
+ private final Path _loggerRootDirPath;
+
+ public LoggerFileServer(String loggerRootDir) {
+ Preconditions.checkNotNull(loggerRootDir, "Logger root directory is null");
+ _loggerRootDir = new File(loggerRootDir);
+ Preconditions.checkState(_loggerRootDir.exists(), "Logger directory
doesn't exists");
+ _loggerRootDirPath = Paths.get(_loggerRootDir.getAbsolutePath());
+ }
+
+ public Set<String> getAllPaths()
+ throws IOException {
+ Set<String> allFiles = new TreeSet<>();
+ Files.walk(_loggerRootDirPath).filter(Files::isRegularFile).forEach(
+ f ->
allFiles.add(f.toAbsolutePath().toString().replace(_loggerRootDirPath.toAbsolutePath()
+ "/", "")));
+ return allFiles;
+ }
+
+ public Response downloadLogFile(String filePath) {
+ try {
+ if (!getAllPaths().contains(filePath)) {
+ throw new WebApplicationException("Invalid file path: " + filePath,
Response.Status.FORBIDDEN);
+ }
+ File logFile = new File(_loggerRootDir, filePath);
+ if (!logFile.exists()) {
+ throw new WebApplicationException("File: " + filePath + " doesn't
exists", Response.Status.NOT_FOUND);
+ }
+ Response.ResponseBuilder builder = Response.ok();
+ builder.entity(logFile);
+ builder.entity((StreamingOutput) output -> Files.copy(logFile.toPath(),
output));
+ builder.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename="
+ logFile.getName());
+ builder.header(HttpHeaders.CONTENT_LENGTH, logFile.length());
+ return builder.build();
+ } catch (IOException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/InstanceUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/InstanceUtils.java
index c840674390..07187fbd01 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/InstanceUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/InstanceUtils.java
@@ -25,6 +25,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.zookeeper.datamodel.ZNRecord;
+import org.apache.pinot.common.helix.ExtraInstanceConfig;
import org.apache.pinot.spi.config.instance.Instance;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Helix;
@@ -157,4 +158,24 @@ public class InstanceUtils {
mapFields.remove(POOL_KEY);
}
}
+
+ public static String getInstanceBaseUri(InstanceConfig instanceConfig) {
+ Map<String, String> fieldMap =
instanceConfig.getRecord().getSimpleFields();
+ String hostName = instanceConfig.getHostName();
+ String adminPort;
+ String scheme;
+ if
(fieldMap.containsKey(CommonConstants.Helix.Instance.ADMIN_HTTPS_PORT_KEY)) {
+ // For Pinot Server admin https port
+ adminPort =
fieldMap.get(CommonConstants.Helix.Instance.ADMIN_HTTPS_PORT_KEY);
+ scheme = "https";
+ } else if
(fieldMap.containsKey(ExtraInstanceConfig.PinotInstanceConfigProperty.PINOT_TLS_PORT.toString()))
{
+ // For Pinot Controller/Broker TLS port
+ adminPort =
fieldMap.get(ExtraInstanceConfig.PinotInstanceConfigProperty.PINOT_TLS_PORT.toString());
+ scheme = "https";
+ } else {
+ adminPort =
fieldMap.getOrDefault(CommonConstants.Helix.Instance.ADMIN_PORT_KEY,
instanceConfig.getPort());
+ scheme = "http";
+ }
+ return String.format("%s://%s:%s", scheme, hostName, adminPort);
+ }
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/http/HttpClient.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/http/HttpClient.java
index d0179c3950..4bd7117ab8 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/http/HttpClient.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/http/HttpClient.java
@@ -288,7 +288,7 @@ public class HttpClient implements AutoCloseable {
}
}
- protected CloseableHttpResponse execute(HttpUriRequest request)
+ public CloseableHttpResponse execute(HttpUriRequest request)
throws IOException {
return _httpClient.execute(request);
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/LoggerFileServerTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/LoggerFileServerTest.java
new file mode 100644
index 0000000000..83ddc49dac
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/LoggerFileServerTest.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Response;
+import org.apache.commons.io.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+
+public class LoggerFileServerTest {
+
+ @Test
+ public void testLoggerFileServer()
+ throws IOException {
+ File logRootDir = new File(FileUtils.getTempDirectory(),
"testGetAllLoggers-" + System.currentTimeMillis());
+ try {
+ logRootDir.mkdirs();
+ LoggerFileServer loggerFileServer = new
LoggerFileServer(logRootDir.getAbsolutePath());
+
+ // Empty root log directory
+ assertEquals(loggerFileServer.getAllPaths().size(), 0);
+ try {
+ loggerFileServer.downloadLogFile("log1");
+ Assert.fail("Shouldn't reach here");
+ } catch (WebApplicationException e1) {
+ assertEquals(e1.getResponse().getStatus(),
Response.Status.FORBIDDEN.getStatusCode());
+ }
+
+ // 1 file: [ log1 ] in root log directory
+ FileUtils.writeStringToFile(new File(logRootDir, "log1"), "mylog1",
Charset.defaultCharset());
+ assertEquals(loggerFileServer.getAllPaths().size(), 1);
+ assertNotNull(loggerFileServer.downloadLogFile("log1"));
+ try {
+ loggerFileServer.downloadLogFile("log2");
+ Assert.fail("Shouldn't reach here");
+ } catch (WebApplicationException e1) {
+ assertEquals(e1.getResponse().getStatus(),
Response.Status.FORBIDDEN.getStatusCode());
+ }
+
+ // 2 files: [ log1, log2 ] in root log directory
+ FileUtils.writeStringToFile(new File(logRootDir, "log2"), "mylog2",
Charset.defaultCharset());
+ assertEquals(loggerFileServer.getAllPaths().size(), 2);
+ assertNotNull(loggerFileServer.downloadLogFile("log1"));
+ assertNotNull(loggerFileServer.downloadLogFile("log2"));
+ try {
+ loggerFileServer.downloadLogFile("log3");
+ Assert.fail("Shouldn't reach here");
+ } catch (WebApplicationException e1) {
+ assertEquals(e1.getResponse().getStatus(),
Response.Status.FORBIDDEN.getStatusCode());
+ }
+ } finally {
+ FileUtils.deleteQuietly(logRootDir);
+ }
+ }
+}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index 9f4a35eb3c..9ba042bbcb 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -59,6 +59,7 @@ import org.apache.pinot.common.metrics.ValidationMetrics;
import org.apache.pinot.common.minion.InMemoryTaskManagerStatusCache;
import org.apache.pinot.common.minion.TaskGeneratorMostRecentRunInfo;
import org.apache.pinot.common.minion.TaskManagerStatusCache;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.common.utils.ServiceStartableUtils;
import org.apache.pinot.common.utils.ServiceStatus;
import org.apache.pinot.common.utils.TlsUtils;
@@ -472,6 +473,10 @@ public abstract class BaseControllerStarter implements
ServiceStartable {
bind(_periodicTaskScheduler).to(PeriodicTaskScheduler.class);
bind(_sqlQueryExecutor).to(SqlQueryExecutor.class);
bind(_pinotLLCRealtimeSegmentManager).to(PinotLLCRealtimeSegmentManager.class);
+ String loggerRootDir =
_config.getProperty(CommonConstants.Controller.CONFIG_OF_LOGGER_ROOT_DIR);
+ if (loggerRootDir != null) {
+ bind(new LoggerFileServer(loggerRootDir)).to(LoggerFileServer.class);
+ }
}
});
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java
index 5000ba4fe5..445d3a1283 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java
@@ -25,8 +25,16 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
@@ -34,10 +42,24 @@ import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.pinot.common.utils.FileUploadDownloadClient;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.common.utils.LoggerUtils;
+import org.apache.pinot.common.utils.SimpleHttpResponse;
+import org.apache.pinot.common.utils.config.InstanceUtils;
+import org.apache.pinot.controller.api.access.AccessType;
+import org.apache.pinot.controller.api.access.Authenticate;
+import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_KEY;
@@ -51,6 +73,14 @@ import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_K
@Path("/")
public class PinotControllerLogger {
+ private final FileUploadDownloadClient _fileUploadDownloadClient = new
FileUploadDownloadClient();
+
+ @Inject
+ private LoggerFileServer _loggerFileServer;
+
+ @Inject
+ PinotHelixResourceManager _pinotHelixResourceManager;
+
@GET
@Path("/loggers")
@Produces(MediaType.APPLICATION_JSON)
@@ -80,4 +110,112 @@ public class PinotControllerLogger {
@ApiParam(value = "Logger level") @QueryParam("level") String level) {
return LoggerUtils.setLoggerLevel(loggerName, level);
}
+
+ @GET
+ @Path("/loggers/files")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get all local log files")
+ public Set<String> getLocalLogFiles() {
+ try {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory doesn't exist",
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.getAllPaths();
+ } catch (IOException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/loggers/download")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ @Authenticate(AccessType.DELETE)
+ @ApiOperation(value = "Download a log file")
+ public Response downloadLogFile(
+ @ApiParam(value = "Log file path", required = true)
@QueryParam("filePath") String filePath) {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory is not configured",
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.downloadLogFile(filePath);
+ }
+
+ @GET
+ @Path("/loggers/instances")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Collect log files from all the instances")
+ public Map<String, Set<String>> getLogFilesFromAllInstances() {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root directory doesn't exist",
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ Map<String, Set<String>> instancesToLogFilesMap = new HashMap<>();
+ List<String> onlineInstanceList =
_pinotHelixResourceManager.getOnlineInstanceList();
+ onlineInstanceList.forEach(
+ instance -> {
+ try {
+ instancesToLogFilesMap.put(instance,
getLogFilesFromInstance(instance));
+ } catch (Exception e) {
+ // Skip the instance for any exception.
+ }
+ });
+ return instancesToLogFilesMap;
+ }
+
+ @GET
+ @Path("/loggers/instances/{instanceName}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Collect log files from a given instance")
+ public Set<String> getLogFilesFromInstance(
+ @ApiParam(value = "Instance Name", required = true)
@PathParam("instanceName") String instanceName) {
+ try {
+ URI uri = new URI(getInstanceBaseUri(instanceName) + "/loggers/files");
+ SimpleHttpResponse simpleHttpResponse =
_fileUploadDownloadClient.getHttpClient().sendGetRequest(uri);
+ if (simpleHttpResponse.getStatusCode() >= 400) {
+ throw new WebApplicationException("Failed to fetch logs from instance
name: " + instanceName,
+
Response.Status.fromStatusCode(simpleHttpResponse.getStatusCode()));
+ }
+ String responseString = simpleHttpResponse.getResponse();
+ responseString = responseString.substring(1, responseString.length() -
1).replace("\"", "");
+ return new HashSet<>(Arrays.asList(responseString.split(",")));
+ } catch (IOException | URISyntaxException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/loggers/instances/{instanceName}/download")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ @Authenticate(AccessType.DELETE)
+ @ApiOperation(value = "Download a log file from a given instance")
+ public Response downloadLogFileFromInstance(
+ @ApiParam(value = "Instance Name", required = true)
@PathParam("instanceName") String instanceName,
+ @ApiParam(value = "Log file path", required = true)
@QueryParam("filePath") String filePath,
+ @Context Map<String, String> headers) {
+ try {
+ URI uri =
UriBuilder.fromUri(getInstanceBaseUri(instanceName)).path("/loggers/download")
+ .queryParam("filePath", filePath).build();
+ RequestBuilder requestBuilder =
RequestBuilder.get(uri).setVersion(HttpVersion.HTTP_1_1);
+ if (MapUtils.isNotEmpty(headers)) {
+ for (Map.Entry<String, String> header : headers.entrySet()) {
+ requestBuilder.addHeader(header.getKey(), header.getValue());
+ }
+ }
+ CloseableHttpResponse httpResponse =
_fileUploadDownloadClient.getHttpClient().execute(requestBuilder.build());
+ if (httpResponse.getStatusLine().getStatusCode() >= 400) {
+ throw new
WebApplicationException(IOUtils.toString(httpResponse.getEntity().getContent(),
"UTF-8"),
+
Response.Status.fromStatusCode(httpResponse.getStatusLine().getStatusCode()));
+ }
+ Response.ResponseBuilder builder = Response.ok();
+ builder.entity(httpResponse.getEntity().getContent());
+ builder.contentLocation(uri);
+ builder.header(HttpHeaders.CONTENT_LENGTH,
httpResponse.getEntity().getContentLength());
+ return builder.build();
+ } catch (IOException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ private String getInstanceBaseUri(String instanceName) {
+ return
InstanceUtils.getInstanceBaseUri(_pinotHelixResourceManager.getHelixInstanceConfig(instanceName));
+ }
}
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
index 97eae3ff7d..dfcef6d6ca 100644
---
a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
+++
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionAdminApiApplication.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.List;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
import org.apache.pinot.spi.env.PinotConfiguration;
@@ -59,6 +60,10 @@ public class MinionAdminApiApplication extends
ResourceConfig {
protected void configure() {
// TODO: Add bindings as needed in future.
bind(instanceId).named(MINION_INSTANCE_ID);
+ String loggerRootDir =
minionConf.getProperty(CommonConstants.Minion.CONFIG_OF_LOGGER_ROOT_DIR);
+ if (loggerRootDir != null) {
+ bind(new LoggerFileServer(loggerRootDir)).to(LoggerFileServer.class);
+ }
}
});
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionLogger.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionLogger.java
index 7572cd9cc5..fb0d282fba 100644
---
a/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionLogger.java
+++
b/pinot-minion/src/main/java/org/apache/pinot/minion/api/resources/PinotMinionLogger.java
@@ -25,8 +25,11 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
@@ -37,6 +40,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.common.utils.LoggerUtils;
import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_KEY;
@@ -51,6 +55,9 @@ import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_K
@Path("/")
public class PinotMinionLogger {
+ @Inject
+ private LoggerFileServer _loggerFileServer;
+
@GET
@Path("/loggers")
@Produces(MediaType.APPLICATION_JSON)
@@ -80,4 +87,32 @@ public class PinotMinionLogger {
@ApiParam(value = "Logger level") @QueryParam("level") String level) {
return LoggerUtils.setLoggerLevel(loggerName, level);
}
+
+ @GET
+ @Path("/loggers/files")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get all local log files")
+ public Set<String> getLocalLogFiles() {
+ try {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory doesn't exist",
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.getAllPaths();
+ } catch (IOException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/loggers/download")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ @ApiOperation(value = "Download a log file")
+ public Response downloadLogFile(
+ @ApiParam(value = "Log file path", required = true)
@QueryParam("filePath") String filePath) {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory is not configured",
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.downloadLogFile(filePath);
+ }
}
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/PinotServerLogger.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/PinotServerLogger.java
index 0e69d89539..14b7d71d61 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/PinotServerLogger.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/PinotServerLogger.java
@@ -25,8 +25,11 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
@@ -37,6 +40,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.common.utils.LoggerUtils;
import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_KEY;
@@ -51,6 +55,9 @@ import static
org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_K
@Path("/")
public class PinotServerLogger {
+ @Inject
+ private LoggerFileServer _loggerFileServer;
+
@GET
@Path("/loggers")
@Produces(MediaType.APPLICATION_JSON)
@@ -80,4 +87,32 @@ public class PinotServerLogger {
@ApiParam(value = "Logger level") @QueryParam("level") String level) {
return LoggerUtils.setLoggerLevel(loggerName, level);
}
+
+ @GET
+ @Path("/loggers/files")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get all local log files")
+ public Set<String> getLocalLogFiles() {
+ try {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory doesn't exist",
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.getAllPaths();
+ } catch (IOException e) {
+ throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ @GET
+ @Path("/loggers/download")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ @ApiOperation(value = "Download a log file")
+ public Response downloadLogFile(
+ @ApiParam(value = "Log file path", required = true)
@QueryParam("filePath") String filePath) {
+ if (_loggerFileServer == null) {
+ throw new WebApplicationException("Root log directory is not configured",
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ return _loggerFileServer.downloadLogFile(filePath);
+ }
}
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/AdminApiApplication.java
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/AdminApiApplication.java
index 48c07aeea1..c0dc370cf9 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/AdminApiApplication.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/AdminApiApplication.java
@@ -29,6 +29,7 @@ import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
import org.apache.pinot.common.metrics.ServerMetrics;
+import org.apache.pinot.common.utils.LoggerFileServer;
import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
import org.apache.pinot.server.access.AccessControlFactory;
@@ -70,6 +71,10 @@ public class AdminApiApplication extends ResourceConfig {
bind(_serverInstance.getServerMetrics()).to(ServerMetrics.class);
bind(accessControlFactory).to(AccessControlFactory.class);
bind(serverConf.getProperty(CommonConstants.Server.CONFIG_OF_INSTANCE_ID)).named(SERVER_INSTANCE_ID);
+ String loggerRootDir =
serverConf.getProperty(CommonConstants.Server.CONFIG_OF_LOGGER_ROOT_DIR);
+ if (loggerRootDir != null) {
+ bind(new LoggerFileServer(loggerRootDir)).to(LoggerFileServer.class);
+ }
}
});
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index d437b76ea7..738800c6cb 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -262,6 +262,8 @@ public class CommonConstants {
// TODO: Support populating clientIp for GrpcRequestIdentity.
public static final boolean DEFAULT_BROKER_REQUEST_CLIENT_IP_LOGGING =
false;
+ public static final String CONFIG_OF_LOGGER_ROOT_DIR =
"pinot.broker.logger.root.dir";
+
public static class Request {
public static final String SQL = "sql";
public static final String TRACE = "trace";
@@ -453,6 +455,7 @@ public class CommonConstants {
// The complete config key is pinot.server.instance.segment.store.uri
public static final String CONFIG_OF_SEGMENT_STORE_URI =
"segment.store.uri";
+ public static final String CONFIG_OF_LOGGER_ROOT_DIR =
"pinot.server.logger.root.dir";
public static class SegmentCompletionProtocol {
public static final String PREFIX_OF_CONFIG_OF_SEGMENT_UPLOADER =
"pinot.server.segment.uploader";
@@ -532,6 +535,7 @@ public class CommonConstants {
"pinot.controller.query.rewriter.class.names";
//Set to true to load all services tagged and compiled with
hk2-metadata-generator. Default to False
public static final String CONTROLLER_SERVICE_AUTO_DISCOVERY =
"pinot.controller.service.auto.discovery";
+ public static final String CONFIG_OF_LOGGER_ROOT_DIR =
"pinot.controller.logger.root.dir";
}
public static class Minion {
@@ -573,6 +577,7 @@ public class CommonConstants {
public static final String CONFIG_TASK_AUTH_NAMESPACE = "task.auth";
public static final String MINION_TLS_PREFIX = "pinot.minion.tls";
public static final String CONFIG_OF_MINION_QUERY_REWRITER_CLASS_NAMES =
"pinot.minion.query.rewriter.class.names";
+ public static final String CONFIG_OF_LOGGER_ROOT_DIR =
"pinot.minion.logger.root.dir";
}
public static class ControllerJob {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
