This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push: new a739360 AMBARI-24833. Cloud base path can be URI as well a739360 is described below commit a739360576790a52f8f67969a3ce97f08874d13e Author: Oliver Szabo <oleew...@gmail.com> AuthorDate: Fri Dec 7 01:08:32 2018 +0100 AMBARI-24833. Cloud base path can be URI as well --- .../ambari/logfeeder/conf/LogFeederProps.java | 6 +- .../output/cloud/CloudStorageUploader.java | 22 +- .../output/cloud/upload/HDFSUploadClient.java | 9 + .../ambari/logfeeder/util/LogFeederHDFSUtil.java | 4 +- .../src/main/resources/core-site.xml | 2 +- .../output/cloud/CloudStorageUploaderTest.java | 82 +++ docs/api-docs/logsearch-swagger.yaml | 694 ++++++++++----------- docs/logfeeder_properties.md | 2 +- 8 files changed, 465 insertions(+), 356 deletions(-) diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java index bafb321..41445cd 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java @@ -311,12 +311,12 @@ public class LogFeederProps implements LogFeederProperties { @LogSearchPropertyDescription( name = LogFeederConstants.CLOUD_STORAGE_BASE_PATH, - description = "Base path prefix for storing logs (cloud storage / hdfs)", - examples = {"/user/logsearch/mypath"}, + description = "Base path prefix for storing logs (cloud storage / hdfs), could be an absolute path or URI. (if URI used, that will override the default.FS with HDFS client)", + examples = {"/user/logsearch/mypath", "s3a:///user/logsearch"}, defaultValue = "/apps/logsearch", sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE} ) - @Value("${" + LogFeederConstants.CLOUD_STORAGE_BASE_PATH + ":}") + @Value("${" + LogFeederConstants.CLOUD_STORAGE_BASE_PATH + ":/apps/logsearch}") private String cloudBasePath; @LogSearchPropertyDescription( diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java index cb4cac3..6a80cab 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java @@ -18,10 +18,12 @@ */ package org.apache.ambari.logfeeder.output.cloud; +import com.google.common.annotations.VisibleForTesting; import org.apache.ambari.logfeeder.conf.LogFeederProps; import org.apache.ambari.logfeeder.output.cloud.upload.UploadClient; import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -90,9 +92,7 @@ public class CloudStorageUploader extends Thread { logger.debug("Not found any files to upload."); } else { for (File file : filesToUpload) { - String basePath = logFeederProps.getCloudBasePath(); - String outputPath = String.format("%s/%s/%s/%s/%s", basePath, clusterName, hostName, file.getParentFile().getName(), file.getName()) - .replaceAll("//", "/"); + final String outputPath = generateOutputPath(logFeederProps.getCloudBasePath(), clusterName, hostName, file); logger.info("Upload will start: input: {}, output: {}", file.getAbsolutePath(), outputPath); Future<?> future = executorService.submit(() -> { try { @@ -114,4 +114,20 @@ public class CloudStorageUploader extends Thread { } } + @VisibleForTesting + String generateOutputPath(String basePath, String clusterName, String hostName, File localFile) { + final String outputWithoutBasePath = Paths.get(clusterName, hostName, localFile.getParentFile().getName(), localFile.getName()).toString(); + final String outputPath; + if (StringUtils.isNotEmpty(basePath)) { + if (!basePath.endsWith("/")){ + outputPath = basePath + "/" + outputWithoutBasePath; + } else { + outputPath = basePath + outputWithoutBasePath; + } + } else { + outputPath = outputWithoutBasePath; + } + return outputPath; + } + } diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java index 3d5ec8f..e072277 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java @@ -64,7 +64,12 @@ public class HDFSUploadClient implements UploadClient { configuration = new Configuration(); logger.info("Initialize HDFS client (cloud mode), using core-site.xml from the classpath."); } + if (hasScheme(logFeederProps.getCloudBasePath())) { + logger.info("Use cloud base path ({}) as fs.defaultFS", logFeederProps.getCloudBasePath()); + configuration.set(FS_DEFAULT_FS, logFeederProps.getCloudBasePath()); + } if (StringUtils.isNotBlank(logFeederProps.getCustomFs())) { + logger.info("Override fs.defaultFS with {}", logFeederProps.getCustomFs()); configuration.set(FS_DEFAULT_FS, logFeederProps.getCustomFs()); } if (hdfsOutputConfig.isHdfsKerberos()) { @@ -90,6 +95,10 @@ public class HDFSUploadClient implements UploadClient { LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, configurationRef.get()); } + private boolean hasScheme(String path) { + return StringUtils.isNotBlank(path) && path.split(":/").length > 1; + } + @Override public void upload(String source, String target) throws Exception { final FileSystem fs = LogFeederHDFSUtil.buildFileSystem(configurationRef.get()); diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java index b741229..c489fbb 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java @@ -19,6 +19,7 @@ package org.apache.ambari.logfeeder.util; import java.io.IOException; +import java.nio.file.Paths; import java.util.Map; import java.util.Properties; @@ -39,9 +40,10 @@ public class LogFeederHDFSUtil { public static void copyFromLocal(String sourceFilepath, String destFilePath, FileSystem fileSystem, boolean overwrite, boolean delSrc, FsPermission fsPermission) throws Exception { + String fsUri = fileSystem.getUri().toString(); Path src = new Path(sourceFilepath); Path dst = new Path(destFilePath); - logger.info("copying localfile := " + sourceFilepath + " to hdfsPath := " + destFilePath); + logger.info("Copying localfile '{}' to hdfsPath (FS base URI: {}) '{}'", sourceFilepath, fsUri, destFilePath); fileSystem.copyFromLocalFile(delSrc, overwrite, src, dst); if (fsPermission != null) { fileSystem.setPermission(dst, fsPermission); diff --git a/ambari-logsearch-logfeeder/src/main/resources/core-site.xml b/ambari-logsearch-logfeeder/src/main/resources/core-site.xml index 9ec1d04..cc1e4a6 100644 --- a/ambari-logsearch-logfeeder/src/main/resources/core-site.xml +++ b/ambari-logsearch-logfeeder/src/main/resources/core-site.xml @@ -17,7 +17,7 @@ <configuration> <property> <name>fs.defaultFS</name> - <value>s3a://logfeeder</value> + <value>s3a://logfeeder/apps</value> </property> <property> <name>fs.s3a.endpoint</name> diff --git a/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploaderTest.java b/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploaderTest.java new file mode 100644 index 0000000..f4a2767 --- /dev/null +++ b/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploaderTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logfeeder.output.cloud; + +import org.apache.ambari.logfeeder.conf.CloudStorageDestination; +import org.apache.ambari.logfeeder.conf.LogFeederProps; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; + +public class CloudStorageUploaderTest { + + private static final String CLUSTER_NAME = "cl"; + private static final String HOSTNAME = "hostname"; + + private CloudStorageUploader underTest; + + @Before + public void setUp() { + LogFeederProps logFeederProps = new LogFeederProps(); + logFeederProps.setCloudStorageDestination(CloudStorageDestination.DEFAULT_FS); + underTest = new CloudStorageUploader("name", null, logFeederProps); + } + + @Test + public void testGenerateOutputPath() { + // GIVEN + String basePath = "example"; + // WHEN + String output = underTest.generateOutputPath(basePath, CLUSTER_NAME, HOSTNAME, new File("/my/path")); + // THEN + Assert.assertEquals("example/cl/hostname/my/path", output); + } + + @Test + public void testGenerateOutputPathWithEmptyBasePath() { + // GIVEN + String basePath = ""; + // WHEN + String output = underTest.generateOutputPath(basePath, CLUSTER_NAME, HOSTNAME, new File("/my/path")); + // THEN + Assert.assertEquals("cl/hostname/my/path", output); + } + + @Test + public void testGenerateOutputPathWithSlashEndAndStart() { + // GIVEN + String basePath = "example/"; + // WHEN + String output = underTest.generateOutputPath(basePath, CLUSTER_NAME, HOSTNAME, new File("/my/path")); + // THEN + Assert.assertEquals("example/cl/hostname/my/path", output); + } + + @Test + public void testGenerateOutputPathWithScheme() { + // GIVEN + String basePath = "s3a://bucket/example"; + // WHEN + String output = underTest.generateOutputPath(basePath, CLUSTER_NAME, HOSTNAME, new File("/my/path")); + // THEN + Assert.assertEquals("s3a://bucket/example/cl/hostname/my/path", output); + } +} diff --git a/docs/api-docs/logsearch-swagger.yaml b/docs/api-docs/logsearch-swagger.yaml index 4ae364a..d80182b 100644 --- a/docs/api-docs/logsearch-swagger.yaml +++ b/docs/api-docs/logsearch-swagger.yaml @@ -19,172 +19,6 @@ schemes: - "http" - "https" paths: - /audit/logs/serviceload: - get: - tags: - - "auditlogs" - summary: "The graph for showing the top users accessing the services" - description: "" - operationId: "getServiceLoadGet" - produces: - - "application/json" - parameters: - - name: "startIndex" - in: "query" - description: "Start index of the queried result" - required: false - type: "string" - - name: "page" - in: "query" - description: "Number of pages for the results" - required: false - type: "string" - default: "0" - - name: "pageSize" - in: "query" - description: "Page size of the results" - required: false - type: "string" - default: "1000" - - name: "sortBy" - in: "query" - description: "Sorting the results based on this field" - required: false - type: "string" - - name: "sortType" - in: "query" - description: "Type of sorting (osc, desc)" - required: false - type: "string" - - name: "start_time" - in: "query" - description: "Date range param which is suportted from browser url" - required: false - type: "string" - - name: "end_time" - in: "query" - description: "Date range param which is supported from browser url" - required: false - type: "string" - - name: "clusters" - in: "query" - description: "filter for clusters (comma separated list)" - required: false - type: "string" - - name: "includeMessage" - in: "query" - description: "Include query which will query against message column" - required: false - type: "string" - - name: "excludeMessage" - in: "query" - description: "Exclude query which will query against message column" - required: false - type: "string" - - name: "mustBe" - in: "query" - description: "Include the components, comma separated values" - required: false - type: "string" - - name: "mustNot" - in: "query" - description: "Exclude the components, comma separated values" - required: false - type: "string" - - name: "includeQuery" - in: "query" - description: "Include the values in query result e.g.: [{message:*exception*}]" - required: false - type: "string" - - name: "excludeQuery" - in: "query" - description: "Exclude the values in query result e.g.: [{message:*timeout*}]" - required: false - type: "string" - - name: "from" - in: "query" - description: "Date range param, start date" - required: false - type: "string" - - name: "to" - in: "query" - description: "Date range param, end date" - required: false - type: "string" - - name: "userList" - in: "query" - description: "Filter for users (comma separated list)" - required: false - type: "string" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/BarGraphDataListResponse" - security: - - basicAuth: [] - post: - tags: - - "auditlogs" - summary: "The graph for showing the top users accessing the services" - description: "" - operationId: "getServiceLoadPost" - consumes: - - "application/json" - produces: - - "application/json" - parameters: - - in: "body" - name: "body" - required: false - schema: - $ref: "#/definitions/AuditServiceLoadBodyRequest" - responses: - 200: - description: "successful operation" - schema: - $ref: "#/definitions/BarGraphDataListResponse" - security: - - basicAuth: [] - /audit/logs/clusters: - get: - tags: - - "auditlogs" - summary: "Get all of the clusters for audit logs" - description: "" - operationId: "getClustersForAuditLogGet" - produces: - - "application/json" - parameters: [] - responses: - 200: - description: "successful operation" - schema: - type: "array" - items: - type: "string" - security: - - basicAuth: [] - post: - tags: - - "auditlogs" - summary: "Get all of the clusters for audit logs" - description: "" - operationId: "getClustersForAuditLogPost" - consumes: - - "application/json" - produces: - - "application/json" - parameters: [] - responses: - 200: - description: "successful operation" - schema: - type: "array" - items: - type: "string" - security: - - basicAuth: [] /audit/logs/schema/fields: get: tags: @@ -826,6 +660,172 @@ paths: description: "successful operation" security: - basicAuth: [] + /audit/logs/serviceload: + get: + tags: + - "auditlogs" + summary: "The graph for showing the top users accessing the services" + description: "" + operationId: "getServiceLoadGet" + produces: + - "application/json" + parameters: + - name: "startIndex" + in: "query" + description: "Start index of the queried result" + required: false + type: "string" + - name: "page" + in: "query" + description: "Number of pages for the results" + required: false + type: "string" + default: "0" + - name: "pageSize" + in: "query" + description: "Page size of the results" + required: false + type: "string" + default: "1000" + - name: "sortBy" + in: "query" + description: "Sorting the results based on this field" + required: false + type: "string" + - name: "sortType" + in: "query" + description: "Type of sorting (osc, desc)" + required: false + type: "string" + - name: "start_time" + in: "query" + description: "Date range param which is suportted from browser url" + required: false + type: "string" + - name: "end_time" + in: "query" + description: "Date range param which is supported from browser url" + required: false + type: "string" + - name: "clusters" + in: "query" + description: "filter for clusters (comma separated list)" + required: false + type: "string" + - name: "includeMessage" + in: "query" + description: "Include query which will query against message column" + required: false + type: "string" + - name: "excludeMessage" + in: "query" + description: "Exclude query which will query against message column" + required: false + type: "string" + - name: "mustBe" + in: "query" + description: "Include the components, comma separated values" + required: false + type: "string" + - name: "mustNot" + in: "query" + description: "Exclude the components, comma separated values" + required: false + type: "string" + - name: "includeQuery" + in: "query" + description: "Include the values in query result e.g.: [{message:*exception*}]" + required: false + type: "string" + - name: "excludeQuery" + in: "query" + description: "Exclude the values in query result e.g.: [{message:*timeout*}]" + required: false + type: "string" + - name: "from" + in: "query" + description: "Date range param, start date" + required: false + type: "string" + - name: "to" + in: "query" + description: "Date range param, end date" + required: false + type: "string" + - name: "userList" + in: "query" + description: "Filter for users (comma separated list)" + required: false + type: "string" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/BarGraphDataListResponse" + security: + - basicAuth: [] + post: + tags: + - "auditlogs" + summary: "The graph for showing the top users accessing the services" + description: "" + operationId: "getServiceLoadPost" + consumes: + - "application/json" + produces: + - "application/json" + parameters: + - in: "body" + name: "body" + required: false + schema: + $ref: "#/definitions/AuditServiceLoadBodyRequest" + responses: + 200: + description: "successful operation" + schema: + $ref: "#/definitions/BarGraphDataListResponse" + security: + - basicAuth: [] + /audit/logs/clusters: + get: + tags: + - "auditlogs" + summary: "Get all of the clusters for audit logs" + description: "" + operationId: "getClustersForAuditLogGet" + produces: + - "application/json" + parameters: [] + responses: + 200: + description: "successful operation" + schema: + type: "array" + items: + type: "string" + security: + - basicAuth: [] + post: + tags: + - "auditlogs" + summary: "Get all of the clusters for audit logs" + description: "" + operationId: "getClustersForAuditLogPost" + consumes: + - "application/json" + produces: + - "application/json" + parameters: [] + responses: + 200: + description: "successful operation" + schema: + type: "array" + items: + type: "string" + security: + - basicAuth: [] /info/features: get: tags: @@ -877,13 +877,13 @@ paths: type: "object" additionalProperties: type: "boolean" - /metadata: + /metadata/list: get: tags: - "metadata" - summary: "Get metadata" + summary: "Get metadata list" description: "" - operationId: "getMetadata" + operationId: "getMetadataList" produces: - "application/json" parameters: @@ -911,15 +911,17 @@ paths: 200: description: "successful operation" schema: - $ref: "#/definitions/LogsearchMetaData" + type: "array" + items: + $ref: "#/definitions/LogsearchMetaData" security: - basicAuth: [] post: tags: - "metadata" - summary: "Save metadata" + summary: "Save metadata list" description: "" - operationId: "saveMetadata" + operationId: "saveMetadataList" produces: - "application/json" parameters: @@ -927,7 +929,9 @@ paths: name: "body" required: false schema: - $ref: "#/definitions/LogsearchMetaData" + type: "array" + items: + $ref: "#/definitions/LogsearchMetaData" responses: 200: description: "successful operation" @@ -938,27 +942,29 @@ paths: delete: tags: - "metadata" - summary: "Delete metadata" + summary: "Delete metadata list" description: "" - operationId: "deleteMetadata" + operationId: "deleteMetadataList" parameters: - in: "body" name: "body" required: false schema: - $ref: "#/definitions/LogsearchMetaData" + type: "array" + items: + $ref: "#/definitions/LogsearchMetaData" responses: default: description: "successful operation" security: - basicAuth: [] - /metadata/list: + /metadata: get: tags: - "metadata" - summary: "Get metadata list" + summary: "Get metadata" description: "" - operationId: "getMetadataList" + operationId: "getMetadata" produces: - "application/json" parameters: @@ -986,17 +992,15 @@ paths: 200: description: "successful operation" schema: - type: "array" - items: - $ref: "#/definitions/LogsearchMetaData" + $ref: "#/definitions/LogsearchMetaData" security: - basicAuth: [] post: tags: - "metadata" - summary: "Save metadata list" + summary: "Save metadata" description: "" - operationId: "saveMetadataList" + operationId: "saveMetadata" produces: - "application/json" parameters: @@ -1004,9 +1008,7 @@ paths: name: "body" required: false schema: - type: "array" - items: - $ref: "#/definitions/LogsearchMetaData" + $ref: "#/definitions/LogsearchMetaData" responses: 200: description: "successful operation" @@ -1017,17 +1019,15 @@ paths: delete: tags: - "metadata" - summary: "Delete metadata list" + summary: "Delete metadata" description: "" - operationId: "deleteMetadataList" + operationId: "deleteMetadata" parameters: - in: "body" name: "body" required: false schema: - type: "array" - items: - $ref: "#/definitions/LogsearchMetaData" + $ref: "#/definitions/LogsearchMetaData" responses: default: description: "successful operation" @@ -3157,13 +3157,13 @@ paths: type: "string" security: - basicAuth: [] - /shipper/input/{clusterName}/services/{serviceName}: + /shipper/input/{clusterName}/services: get: tags: - "shipper" - summary: "Get shipper config" + summary: "Get service names" description: "" - operationId: "getShipperConfig" + operationId: "getServices" produces: - "application/json" parameters: @@ -3171,50 +3171,42 @@ paths: in: "path" required: true type: "string" - - name: "serviceName" - in: "path" - required: true - type: "string" responses: 200: description: "successful operation" schema: - $ref: "#/definitions/LSServerInputConfig" + type: "array" + items: + type: "string" security: - basicAuth: [] - post: + /shipper/filters/{clusterName}/level: + get: tags: - "shipper" - summary: "Set shipper config" + summary: "Get log level filter" description: "" - operationId: "createShipperConfig" + operationId: "getLogLevelFilters" produces: - "application/json" parameters: - - in: "body" - name: "body" - required: false - schema: - $ref: "#/definitions/LSServerInputConfig" - name: "clusterName" in: "path" required: true type: "string" - - name: "serviceName" - in: "path" - required: true - type: "string" responses: - default: + 200: description: "successful operation" + schema: + $ref: "#/definitions/LSServerLogLevelFilterMap" security: - basicAuth: [] put: tags: - "shipper" - summary: "Set shipper config" + summary: "Update log level filter" description: "" - operationId: "setShipperConfig" + operationId: "setLogLevelFilter" produces: - "application/json" parameters: @@ -3222,27 +3214,23 @@ paths: name: "body" required: false schema: - $ref: "#/definitions/LSServerInputConfig" + $ref: "#/definitions/LSServerLogLevelFilterMap" - name: "clusterName" in: "path" required: true type: "string" - - name: "serviceName" - in: "path" - required: true - type: "string" responses: default: description: "successful operation" security: - basicAuth: [] - /shipper/input/{clusterName}/services: + /shipper/input/{clusterName}/services/{serviceName}: get: tags: - "shipper" - summary: "Get service names" + summary: "Get shipper config" description: "" - operationId: "getServices" + operationId: "getShipperConfig" produces: - "application/json" parameters: @@ -3250,42 +3238,50 @@ paths: in: "path" required: true type: "string" + - name: "serviceName" + in: "path" + required: true + type: "string" responses: 200: description: "successful operation" schema: - type: "array" - items: - type: "string" + $ref: "#/definitions/LSServerInputConfig" security: - basicAuth: [] - /shipper/filters/{clusterName}/level: - get: + post: tags: - "shipper" - summary: "Get log level filter" + summary: "Set shipper config" description: "" - operationId: "getLogLevelFilters" + operationId: "createShipperConfig" produces: - "application/json" parameters: + - in: "body" + name: "body" + required: false + schema: + $ref: "#/definitions/LSServerInputConfig" - name: "clusterName" in: "path" required: true type: "string" + - name: "serviceName" + in: "path" + required: true + type: "string" responses: - 200: + default: description: "successful operation" - schema: - $ref: "#/definitions/LSServerLogLevelFilterMap" security: - basicAuth: [] put: tags: - "shipper" - summary: "Update log level filter" + summary: "Set shipper config" description: "" - operationId: "setLogLevelFilter" + operationId: "setShipperConfig" produces: - "application/json" parameters: @@ -3293,11 +3289,15 @@ paths: name: "body" required: false schema: - $ref: "#/definitions/LSServerLogLevelFilterMap" + $ref: "#/definitions/LSServerInputConfig" - name: "clusterName" in: "path" required: true type: "string" + - name: "serviceName" + in: "path" + required: true + type: "string" responses: default: description: "successful operation" @@ -3415,66 +3415,6 @@ securityDefinitions: basicAuth: type: "basic" definitions: - BarGraphData: - type: "object" - properties: - dataCount: - type: "array" - items: - $ref: "#/definitions/NameValueData" - name: - type: "string" - BarGraphDataListResponse: - type: "object" - properties: - graphData: - type: "array" - items: - $ref: "#/definitions/BarGraphData" - NameValueData: - type: "object" - properties: - name: - type: "string" - value: - type: "string" - AuditServiceLoadBodyRequest: - type: "object" - properties: - startIndex: - type: "string" - page: - type: "string" - pageSize: - type: "string" - sortBy: - type: "string" - sortType: - type: "string" - start_time: - type: "string" - end_time: - type: "string" - clusters: - type: "string" - includeMessage: - type: "string" - excludeMessage: - type: "string" - mustBe: - type: "string" - mustNot: - type: "string" - includeQuery: - type: "string" - excludeQuery: - type: "string" - from: - type: "string" - to: - type: "string" - userList: - type: "string" AuditFieldMetadataResponse: type: "object" properties: @@ -3509,27 +3449,23 @@ definitions: format: "int32" policy: type: "string" - reqContext: - type: "string" - action: - type: "string" reason: type: "string" text: type: "string" + sess: + type: "string" access: type: "string" - proxyUsers: - type: "array" - items: - type: "string" - logType: + reqContext: type: "string" tags: type: "array" items: type: "string" - sess: + logType: + type: "string" + action: type: "string" agent: type: "string" @@ -3557,6 +3493,10 @@ definitions: type: "string" tags_str: type: "string" + proxyUsers: + type: "array" + items: + type: "string" id: type: "string" type: @@ -3566,12 +3506,12 @@ definitions: _version_: type: "integer" format: "int64" - log_message: - type: "string" bundle_id: type: "string" case_id: type: "string" + log_message: + type: "string" logfile_line_number: type: "integer" format: "int32" @@ -3682,6 +3622,29 @@ definitions: properties: clusters: type: "string" + BarGraphData: + type: "object" + properties: + dataCount: + type: "array" + items: + $ref: "#/definitions/NameValueData" + name: + type: "string" + BarGraphDataListResponse: + type: "object" + properties: + graphData: + type: "array" + items: + $ref: "#/definitions/BarGraphData" + NameValueData: + type: "object" + properties: + name: + type: "string" + value: + type: "string" AuditBarGraphBodyRequest: type: "object" properties: @@ -3801,6 +3764,43 @@ definitions: type: "string" format: type: "string" + AuditServiceLoadBodyRequest: + type: "object" + properties: + startIndex: + type: "string" + page: + type: "string" + pageSize: + type: "string" + sortBy: + type: "string" + sortType: + type: "string" + start_time: + type: "string" + end_time: + type: "string" + clusters: + type: "string" + includeMessage: + type: "string" + excludeMessage: + type: "string" + mustBe: + type: "string" + mustNot: + type: "string" + includeQuery: + type: "string" + excludeQuery: + type: "string" + from: + type: "string" + to: + type: "string" + userList: + type: "string" LogsearchMetaData: type: "object" properties: @@ -3854,11 +3854,11 @@ definitions: type: "string" group: type: "string" + ip: + type: "string" logtime: type: "string" format: "date-time" - ip: - type: "string" id: type: "string" file: @@ -3866,12 +3866,12 @@ definitions: _version_: type: "integer" format: "int64" - log_message: - type: "string" bundle_id: type: "string" case_id: type: "string" + log_message: + type: "string" logfile_line_number: type: "integer" format: "int32" @@ -4684,6 +4684,39 @@ definitions: type: "string" clusters: type: "string" + LSServerLogLevelFilter: + type: "object" + required: + - "defaultLevels" + - "hosts" + - "label" + properties: + label: + type: "string" + hosts: + type: "array" + items: + type: "string" + defaultLevels: + type: "array" + items: + type: "string" + overrideLevels: + type: "array" + items: + type: "string" + expiryTime: + type: "string" + format: "date-time" + LSServerLogLevelFilterMap: + type: "object" + required: + - "filter" + properties: + filter: + type: "object" + additionalProperties: + $ref: "#/definitions/LSServerLogLevelFilter" LSServerConditions: type: "object" required: @@ -4795,45 +4828,12 @@ definitions: $ref: "#/definitions/LSServerFilter" LSServerPostMapValuesList: type: "object" - LSServerLogLevelFilter: - type: "object" - required: - - "defaultLevels" - - "hosts" - - "label" - properties: - label: - type: "string" - hosts: - type: "array" - items: - type: "string" - defaultLevels: - type: "array" - items: - type: "string" - overrideLevels: - type: "array" - items: - type: "string" - expiryTime: - type: "string" - format: "date-time" - LSServerLogLevelFilterMap: - type: "object" - required: - - "filter" - properties: - filter: - type: "object" - additionalProperties: - $ref: "#/definitions/LSServerLogLevelFilter" SolrCollectionState: type: "object" properties: - znodeReady: - type: "boolean" solrCollectionReady: type: "boolean" + znodeReady: + type: "boolean" configurationUploaded: type: "boolean" diff --git a/docs/logfeeder_properties.md b/docs/logfeeder_properties.md index 8eec4cd..9f8f9eb 100644 --- a/docs/logfeeder_properties.md +++ b/docs/logfeeder_properties.md @@ -36,7 +36,7 @@ limitations under the License. |`logfeeder.cloud.rollover.threshold.size`|Rollover cloud log files after the log file size reach this limit|80|<ul><li>`1024`</li></ul>| |`logfeeder.cloud.rollover.threshold.size.unit`|Rollover cloud log file size unit (e.g: KB, MB etc.)|MB|<ul><li>`KB`</li></ul>| |`logfeeder.cloud.rollover.use.gzip`|Use GZip on archived logs.|true|<ul><li>`false`</li></ul>| -|`logfeeder.cloud.storage.base.path`|Base path prefix for storing logs (cloud storage / hdfs)|/apps/logsearch|<ul><li>`/user/logsearch/mypath`</li></ul>| +|`logfeeder.cloud.storage.base.path`|Base path prefix for storing logs (cloud storage / hdfs), could be an absolute path or URI. (if URI used, that will override the default.FS with HDFS client)|/apps/logsearch|<ul><li>`/user/logsearch/mypath`</li><li>`s3a:///user/logsearch`</li></ul>| |`logfeeder.cloud.storage.bucket`|Amazon S3 bucket.|logfeeder|<ul><li>`logs`</li></ul>| |`logfeeder.cloud.storage.bucket.bootstrap`|Create bucket on startup.|true|<ul><li>`false`</li></ul>| |`logfeeder.cloud.storage.custom.fs`|If it is not empty, override fs.defaultFS for HDFS client. Can be useful to write data to a different bucket (from other services) if the bucket address is read from core-site.xml|`EMPTY`|<ul><li>`s3a://anotherbucket`</li></ul>|