[ambari] branch trunk updated: AMBARI-24917. Implement complex Add Service request using default configs (#2631)
This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 94d1860 AMBARI-24917. Implement complex Add Service request using
default configs (#2631)
94d1860 is described below
commit 94d18601e8adef6bd43d2ce1e70d3c1e4f14ae3c
Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com>
AuthorDate: Tue Nov 20 08:14:45 2018 +0100
AMBARI-24917. Implement complex Add Service request using default configs
(#2631)
---
.../ambari/server/api/handlers/CreateHandler.java | 2 +-
.../internal/HostComponentResourceProvider.java| 2 +-
.../internal/ServiceResourceProvider.java | 8 +-
.../ambari/server/controller/internal/Stack.java | 24 +++
.../server/controller/internal/UnitUpdater.java| 23 ++-
.../server/topology/addservice/AddServiceInfo.java | 46 +-
.../addservice/AddServiceOrchestrator.java | 61 ++-
.../addservice/ResourceProviderAdapter.java| 183 +++--
8 files changed, 312 insertions(+), 37 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/CreateHandler.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/CreateHandler.java
index 484547a..b5dddf8 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/CreateHandler.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/CreateHandler.java
@@ -77,7 +77,7 @@ public class CreateHandler extends BaseManagementHandler {
} catch (ResourceAlreadyExistsException e) {
result = new ResultImpl(new ResultStatus(ResultStatus.STATUS.CONFLICT,
e.getMessage()));
} catch(IllegalArgumentException e) {
- LOG.error("Bad request received: " + e.getMessage());
+ LOG.error("Bad request received: " + e.getMessage(), e);
result = new ResultImpl(new
ResultStatus(ResultStatus.STATUS.BAD_REQUEST, e.getMessage()));
} catch (RuntimeException e) {
if (LOG.isErrorEnabled()) {
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
index d6f52a4..79441d5 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
@@ -799,7 +799,7 @@ public class HostComponentResourceProvider extends
AbstractControllerResourcePro
* @throws NoSuchResourceExceptionthe query didn't match any
resources
* @throws NoSuchParentResourceException a specified parent resource
doesn't exist
*/
- private RequestStageContainer doUpdateResources(final RequestStageContainer
stages, final Request request,
+ public RequestStageContainer doUpdateResources(final RequestStageContainer
stages, final Request request,
Predicate predicate, boolean
performQueryEvaluation,
boolean useGeneratedConfigs,
boolean useClusterHostInfo)
throws
UnsupportedPropertyException,
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
index a4aaf0b..eaa8cb1 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java
@@ -254,9 +254,9 @@ public class ServiceResourceProvider extends
AbstractControllerResourceProvider
if (request.getProperties().size() == 1) {
Map requestProperties =
request.getProperties().iterator().next();
if (isAddServiceRequest(requestProperties)) {
-processAddServiceRequest(requestProperties,
request.getRequestInfoProperties());
+RequestStatusResponse response =
processAddServiceRequest(requestProperties, request.getRequestInfoProperties());
notifyCreate(Resource.Type.Service, request);
-return getRequestStatus(null);
+return getRequestStatus(response);
}
}
@@ -1228,11 +1228,11 @@ public class ServiceResourceProvider extends
AbstractControllerResourceProvider
return
OperationType.ADD_SERVICE.name().equals(properties.get(OPERATION_TYPE));
}
- private void processAddServiceRequest(Map requestProperties,
Map requestInfoProperties) throws NoSuchParentResourceException
{
+ private RequestStatusResponse processAddServiceRequest(Map
requestProperties, Map
[GitHub] oleewere closed pull request #29: AMBARI-24833. Use clustername and hostname in cloud log archive dir
oleewere closed pull request #29: AMBARI-24833. Use clustername and hostname in
cloud log archive dir
URL: https://github.com/apache/ambari-logsearch/pull/29
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
index 0cfdbcc1e2..ea422b9c31 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
@@ -21,6 +21,7 @@
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.plugin.input.Input;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
@@ -56,10 +57,11 @@ public static Logger createLogger(Input input,
LoggerContext loggerContext, LogF
String type = input.getLogType().replace(LogFeederConstants.CLOUD_PREFIX,
"");
String uniqueThreadName = input.getThread().getName();
Configuration config = loggerContext.getConfiguration();
-String destination = logFeederProps.getCloudStorageDestination().getText();
String baseDir =
logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir();
-String activeLogDir = Paths.get(baseDir, destination, ACTIVE_FOLDER,
type).toFile().getAbsolutePath();
-String archiveLogDir = Paths.get(baseDir, destination, ARCHIVED_FOLDER,
type).toFile().getAbsolutePath();
+String destination = logFeederProps.getCloudStorageDestination().getText();
+String clusterHostnameBaseDir = Paths.get(baseDir, destination,
logFeederProps.getClusterName(),
LogFeederUtil.hostName).toFile().getAbsolutePath();
+String activeLogDir = Paths.get(clusterHostnameBaseDir, ACTIVE_FOLDER,
type).toFile().getAbsolutePath();
+String archiveLogDir = Paths.get(clusterHostnameBaseDir, ARCHIVED_FOLDER,
type).toFile().getAbsolutePath();
boolean useGzip = logFeederProps.getRolloverConfig().isUseGzip();
final String archiveFilePattern;
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
index af9326aed4..22c7fc1c61 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
@@ -26,6 +26,7 @@
import org.apache.logging.log4j.Logger;
import java.io.File;
+import java.nio.file.Paths;
import java.util.Collection;
/**
@@ -74,10 +75,11 @@ public void run() {
*/
void doUpload() {
try {
- final String archiveLogDir = String.join(File.separator,
logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir(), uploaderType,
"archived");
- if (new File(archiveLogDir).exists()) {
+ final File archiveLogDir =
Paths.get(logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir(),
+uploaderType, clusterName, hostName, "archived").toFile();
+ if (archiveLogDir.exists()) {
String[] extensions = {"log", "json", "gz"};
-Collection filesToUpload = FileUtils.listFiles(new
File(archiveLogDir), extensions, true);
+Collection filesToUpload = FileUtils.listFiles(archiveLogDir,
extensions, true);
if (filesToUpload.isEmpty()) {
logger.debug("Not found any files to upload.");
} else {
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[ambari-logsearch] branch master updated: AMBARI-24833. Let logfeeder.properties override core-site configs. (#30)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new defc854 AMBARI-24833. Let logfeeder.properties override core-site
configs. (#30)
defc854 is described below
commit defc854b8d1948745a07a1a6e07322670ffef340
Author: Olivér Szabó
AuthorDate: Mon Nov 19 21:49:56 2018 +0100
AMBARI-24833. Let logfeeder.properties override core-site configs. (#30)
---
.../output/cloud/upload/HDFSS3UploadClient.java | 1 +
.../output/cloud/upload/HDFSUploadClient.java | 1 +
.../ambari/logfeeder/util/LogFeederHDFSUtil.java | 19 +++
.../src/main/resources/logfeeder.properties | 2 ++
4 files changed, 23 insertions(+)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
index 5405a10..df1b8a5 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
@@ -57,6 +57,7 @@ public class HDFSS3UploadClient extends AbstractS3CloudClient
implements UploadC
conf.set("fs.s3a.endpoint", s3OutputConfig.getEndpoint());
conf.set("fs.s3a.path.style.access",
String.valueOf(s3OutputConfig.isPathStyleAccess()));
conf.set("fs.s3a.multiobjectdelete.enable",
String.valueOf(s3OutputConfig.isMultiobjectDeleteEnable()));
+LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, conf);
this.fs = LogFeederHDFSUtil.buildFileSystem(conf);
}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
index 421c4c5..7e1b471 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
@@ -84,6 +84,7 @@ public class HDFSUploadClient implements UploadClient {
}
}
logger.info("HDFS client - will use '{}' permission for uploaded files",
hdfsOutputConfig.getHdfsFilePermissions());
+LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, configuration);
this.fs = LogFeederHDFSUtil.buildFileSystem(configuration);
}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
index 3549e04..b741229 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
@@ -19,7 +19,10 @@
package org.apache.ambari.logfeeder.util;
import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -95,4 +98,20 @@ public class LogFeederHDFSUtil {
}
}
}
+
+ /**
+ * Override Hadoop configuration object based on logfeeder.properties
configurations (with keys that starts with "fs." or "hadoop.*")
+ * @param logFeederProps global property holder
+ * @param configuration hadoop configuration holder
+ */
+ public static void overrideFileSystemConfigs(LogFeederProps logFeederProps,
Configuration configuration) {
+Properties properties = logFeederProps.getProperties();
+for (Map.Entry prop : properties.entrySet()) {
+ String propertyName = prop.getKey().toString();
+ if (propertyName.startsWith("fs.")) {
+logger.info("Override {} configuration (by logfeeder.properties)",
propertyName);
+configuration.set(propertyName, prop.getValue().toString());
+ }
+}
+ }
}
\ No newline at end of file
diff --git a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 45c05f3..2537913 100644
--- a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -68,3 +68,5 @@ logfeeder.s3.access.key=MyAccessKey
logfeeder.s3.object.acl=public-read
logfeeder.s3.path.style.access=true
logfeeder.s3.multiobject.delete.enable=false
+
+fs.s3a.acl.default=PublicReadWrite
[ambari-logsearch] branch master updated: AMBARI-24833. Use clustername and hostname in cloud log archive dir (#29)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new 1af5b0b AMBARI-24833. Use clustername and hostname in cloud log
archive dir (#29)
1af5b0b is described below
commit 1af5b0b50442df0bb292151554a169e64372289a
Author: Olivér Szabó
AuthorDate: Mon Nov 19 21:49:42 2018 +0100
AMBARI-24833. Use clustername and hostname in cloud log archive dir (#29)
* AMBARI-24833. Use clustername and hostname in cloud log archive dir
* AMBARI-24833. Check the right folders on upload and use Paths.get
---
.../ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java | 8 +---
.../ambari/logfeeder/output/cloud/CloudStorageUploader.java | 8 +---
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
index 0cfdbcc..ea422b9 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
@@ -21,6 +21,7 @@ package org.apache.ambari.logfeeder.output.cloud;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.plugin.input.Input;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
@@ -56,10 +57,11 @@ public class CloudStorageLoggerFactory {
String type = input.getLogType().replace(LogFeederConstants.CLOUD_PREFIX,
"");
String uniqueThreadName = input.getThread().getName();
Configuration config = loggerContext.getConfiguration();
-String destination = logFeederProps.getCloudStorageDestination().getText();
String baseDir =
logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir();
-String activeLogDir = Paths.get(baseDir, destination, ACTIVE_FOLDER,
type).toFile().getAbsolutePath();
-String archiveLogDir = Paths.get(baseDir, destination, ARCHIVED_FOLDER,
type).toFile().getAbsolutePath();
+String destination = logFeederProps.getCloudStorageDestination().getText();
+String clusterHostnameBaseDir = Paths.get(baseDir, destination,
logFeederProps.getClusterName(),
LogFeederUtil.hostName).toFile().getAbsolutePath();
+String activeLogDir = Paths.get(clusterHostnameBaseDir, ACTIVE_FOLDER,
type).toFile().getAbsolutePath();
+String archiveLogDir = Paths.get(clusterHostnameBaseDir, ARCHIVED_FOLDER,
type).toFile().getAbsolutePath();
boolean useGzip = logFeederProps.getRolloverConfig().isUseGzip();
final String archiveFilePattern;
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
index af9326a..22c7fc1 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
@@ -26,6 +26,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.File;
+import java.nio.file.Paths;
import java.util.Collection;
/**
@@ -74,10 +75,11 @@ public class CloudStorageUploader extends Thread {
*/
void doUpload() {
try {
- final String archiveLogDir = String.join(File.separator,
logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir(), uploaderType,
"archived");
- if (new File(archiveLogDir).exists()) {
+ final File archiveLogDir =
Paths.get(logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir(),
+uploaderType, clusterName, hostName, "archived").toFile();
+ if (archiveLogDir.exists()) {
String[] extensions = {"log", "json", "gz"};
-Collection filesToUpload = FileUtils.listFiles(new
File(archiveLogDir), extensions, true);
+Collection filesToUpload = FileUtils.listFiles(archiveLogDir,
extensions, true);
if (filesToUpload.isEmpty()) {
logger.debug("Not found any files to upload.");
} else {
[GitHub] oleewere closed pull request #30: AMBARI-24833. Let logfeeder.properties override core-site configs.
oleewere closed pull request #30: AMBARI-24833. Let logfeeder.properties
override core-site configs.
URL: https://github.com/apache/ambari-logsearch/pull/30
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
index 5405a1051c..df1b8a55b2 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
@@ -57,6 +57,7 @@ public void init(LogFeederProps logFeederProps) {
conf.set("fs.s3a.endpoint", s3OutputConfig.getEndpoint());
conf.set("fs.s3a.path.style.access",
String.valueOf(s3OutputConfig.isPathStyleAccess()));
conf.set("fs.s3a.multiobjectdelete.enable",
String.valueOf(s3OutputConfig.isMultiobjectDeleteEnable()));
+LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, conf);
this.fs = LogFeederHDFSUtil.buildFileSystem(conf);
}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
index 421c4c5cb7..7e1b471de0 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
@@ -84,6 +84,7 @@ public void init(LogFeederProps logFeederProps) {
}
}
logger.info("HDFS client - will use '{}' permission for uploaded files",
hdfsOutputConfig.getHdfsFilePermissions());
+LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, configuration);
this.fs = LogFeederHDFSUtil.buildFileSystem(configuration);
}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
index 3549e04af6..b741229a13 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
@@ -19,7 +19,10 @@
package org.apache.ambari.logfeeder.util;
import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -95,4 +98,20 @@ public static void closeFileSystem(FileSystem fileSystem) {
}
}
}
+
+ /**
+ * Override Hadoop configuration object based on logfeeder.properties
configurations (with keys that starts with "fs." or "hadoop.*")
+ * @param logFeederProps global property holder
+ * @param configuration hadoop configuration holder
+ */
+ public static void overrideFileSystemConfigs(LogFeederProps logFeederProps,
Configuration configuration) {
+Properties properties = logFeederProps.getProperties();
+for (Map.Entry prop : properties.entrySet()) {
+ String propertyName = prop.getKey().toString();
+ if (propertyName.startsWith("fs.")) {
+logger.info("Override {} configuration (by logfeeder.properties)",
propertyName);
+configuration.set(propertyName, prop.getValue().toString());
+ }
+}
+ }
}
\ No newline at end of file
diff --git a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 45c05f39cf..2537913bbe 100644
--- a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -68,3 +68,5 @@ logfeeder.s3.access.key=MyAccessKey
logfeeder.s3.object.acl=public-read
logfeeder.s3.path.style.access=true
logfeeder.s3.multiobject.delete.enable=false
+
+fs.s3a.acl.default=PublicReadWrite
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] asfgit commented on issue #30: AMBARI-24833. Let logfeeder.properties override core-site configs.
asfgit commented on issue #30: AMBARI-24833. Let logfeeder.properties override core-site configs. URL: https://github.com/apache/ambari-logsearch/pull/30#issuecomment-440036535 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/Ambari-LogSearch-Github-PR-Builder/61/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] oleewere opened a new pull request #30: AMBARI-24833. Let logfeeder.properties override core-site configs.
oleewere opened a new pull request #30: AMBARI-24833. Let logfeeder.properties override core-site configs. URL: https://github.com/apache/ambari-logsearch/pull/30 # What changes were proposed in this pull request? core site configs can be override by logfeeder.properties. It can be useful if no core-site.xml available on a host ## How was this patch tested? FT: manually This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[ambari-logsearch] 11/28: AMBARI-24802 - Logsearch: upgrade swagger-ui to 3.19.0 (#10)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 3053cffa1841943372a4bf3d38a8cebb5ea8012f
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Thu Oct 18 15:20:30 2018 +0200
AMBARI-24802 - Logsearch: upgrade swagger-ui to 3.19.0 (#10)
---
ambari-logsearch-server/pom.xml| 2 +-
.../ambari/logsearch/common/ApiDocStorage.java | 34 +++--
.../ambari/logsearch/rest/AuditLogsResource.java | 28 ++--
.../logsearch/rest/EventHistoryResource.java | 17 ++-
.../ambari/logsearch/rest/ServiceLogsResource.java | 35 -
.../logsearch/rest/ShipperConfigResource.java | 3 +-
.../ambari/logsearch/rest/StatusResource.java | 25 ++--
.../src/main/resources/swagger/swagger.html| 161 +++--
8 files changed, 143 insertions(+), 162 deletions(-)
diff --git a/ambari-logsearch-server/pom.xml b/ambari-logsearch-server/pom.xml
index d18a47c..8c955ae 100755
--- a/ambari-logsearch-server/pom.xml
+++ b/ambari-logsearch-server/pom.xml
@@ -572,7 +572,7 @@
org.webjars
swagger-ui
- 2.2.2
+ 3.19.0
org.springframework.data
diff --git
a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ApiDocStorage.java
b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ApiDocStorage.java
index e31a3b4..f2edcce 100644
---
a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ApiDocStorage.java
+++
b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ApiDocStorage.java
@@ -18,17 +18,20 @@
*/
package org.apache.ambari.logsearch.common;
-import io.swagger.jaxrs.config.BeanConfig;
-import io.swagger.models.Swagger;
-import io.swagger.util.Yaml;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Named;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.swagger.jaxrs.config.BeanConfig;
+import io.swagger.models.Swagger;
+import io.swagger.models.auth.BasicAuthDefinition;
+import io.swagger.util.Yaml;
@Named
public class ApiDocStorage {
@@ -47,20 +50,19 @@ public class ApiDocStorage {
public void run() {
logger.info("Start thread to scan REST API doc from endpoints.");
Swagger swagger = beanConfig.getSwagger();
+swagger.addSecurityDefinition("basicAuth", new BasicAuthDefinition());
beanConfig.configure(swagger);
beanConfig.scanAndRead();
setSwagger(swagger);
try {
- if (swagger != null) {
-String yaml = Yaml.mapper().writeValueAsString(swagger);
-StringBuilder b = new StringBuilder();
-String[] parts = yaml.split("\n");
-for (String part : parts) {
- b.append(part);
- b.append("\n");
-}
-setSwaggerYaml(b.toString());
+ String yaml = Yaml.mapper().writeValueAsString(swagger);
+ StringBuilder b = new StringBuilder();
+ String[] parts = yaml.split("\n");
+ for (String part : parts) {
+b.append(part);
+b.append("\n");
}
+ setSwaggerYaml(b.toString());
} catch (Exception e) {
e.printStackTrace();
}
diff --git
a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
index 0da2706..6b214e0 100644
---
a/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
+++
b/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
@@ -19,6 +19,19 @@
package org.apache.ambari.logsearch.rest;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.EXPORT_USER_TALBE_TO_TEXT_FILE_OD;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.GET_AUDIT_CLUSTERS_OD;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.GET_AUDIT_COMPONENTS_OD;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.GET_AUDIT_LINE_GRAPH_DATA_OD;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.GET_AUDIT_LOGS_OD;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.GET_AUDIT_SCHEMA_FIELD_LIST_OD;
+import static
org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.GET_SERVICE_LOAD_OD;
[ambari-logsearch] 22/28: AMBARI-24833. Simplify HDFS client usage + use core-site.xml (#21)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 1485bd6c72dfa8c712077eafcfe17a4ed5e2d7a1
Author: Olivér Szabó
AuthorDate: Tue Nov 13 11:34:51 2018 +0100
AMBARI-24833. Simplify HDFS client usage + use core-site.xml (#21)
---
.../logfeeder/common/LogFeederConstants.java | 6 ++-
.../ambari/logfeeder/conf/LogFeederProps.java | 51 ++
.../ambari/logfeeder/conf/output/BucketConfig.java | 18
.../conf/output/CloudStorageOutputConfig.java | 47
...utConfig.java => ExternalHdfsOutputConfig.java} | 25 +--
.../ambari/logfeeder/conf/output/OutputConfig.java | 32 --
.../logfeeder/conf/output/RolloverConfig.java | 18
.../logfeeder/conf/output/S3OutputConfig.java | 30 +
.../output/cloud/CloudStorageLoggerFactory.java| 9 ++--
.../output/cloud/CloudStorageUploader.java | 6 +--
...CloudClient.java => AbstractS3CloudClient.java} | 10 ++---
...adClient.java => ExternalHDFSUploadClient.java} | 28 +---
.../output/cloud/upload/HDFSS3UploadClient.java| 16 +++
.../output/cloud/upload/HDFSUploadClient.java | 44 ++-
.../output/cloud/upload/S3UploadClient.java| 12 ++---
.../output/cloud/upload/UploadClient.java | 13 +-
.../output/cloud/upload/UploadClientFactory.java | 19 ++--
.../ambari/logfeeder/util/LogFeederHDFSUtil.java | 22 +++---
.../src/main/resources/core-site.xml | 42 ++
.../src/main/resources/logfeeder.properties| 11 +++--
20 files changed, 218 insertions(+), 241 deletions(-)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index f528c45..11d351f 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -113,9 +113,13 @@ public class LogFeederConstants {
public static final String CLOUD_STORAGE_DESTINATION =
"logfeeder.cloud.storage.destination";
public static final String CLOUD_STORAGE_UPLOAD_ON_SHUTDOWN =
"logfeeder.cloud.storage.upload.on.shutdown";
public static final String CLOUD_STORAGE_UPLOADER_INTERVAL_SECONDS =
"logfeeder.cloud.storage.uploader.interval.seconds";
+ public static final String CLOUD_STORAGE_BUCKET =
"logfeeder.cloud.storage.bucket";
public static final String CLOUD_STORAGE_BUCKET_BOOTSTRAP =
"logfeeder.cloud.storage.bucket.bootstrap";
public static final String CLOUD_STORAGE_USE_HDFS_CLIENT =
"logfeeder.cloud.storage.use.hdfs.client";
+ public static final String CLOUD_STORAGE_CUSTOM_FS =
"logfeeder.cloud.storage.custom.fs";
+ public static final String CLOUD_STORAGE_BASE_PATH =
"logfeeder.cloud.storage.base.path";
+ public static final String CLOUD_ROLLOVER_ARCHIVE_LOCATION =
"logfeeder.cloud.rollover.archive.base.dir";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_MIN =
"logfeeder.cloud.rollover.threshold.min";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE =
"logfeeder.cloud.rollover.threshold.size";
public static final String CLOUD_ROLLOVER_USE_GZIP =
"logfeeder.cloud.rollover.use.gzip";
@@ -126,14 +130,12 @@ public class LogFeederConstants {
public static final String HDFS_HOST = "logfeeder.hdfs.host";
public static final String HDFS_PORT = "logfeeder.hdfs.port";
public static final String HDFS_USER = "logfeeder.hdfs.user";
- public static final String HDFS_OUTPUT_BASE_DIR =
"logfeeder.hdfs.output.base.dir";
public static final String HDFS_FILE_PERMISSIONS =
"logfeeder.hdfs.file.permissions";
public static final String HDFS_KERBEROS = "logfeeder.hdfs.kerberos";
public static final String S3_ENDPOINT = "logfeeder.s3.endpoint";
public static final String S3_ENDPOINT_DEFAULT = "https://s3.amazonaws.com;;
public static final String S3_REGION = "logfeeder.s3.region";
- public static final String S3_BUCKET = "logfeeder.s3.bucket";
public static final String S3_OBJECT_ACL = "logfeeder.s3.object.acl";
public static final String S3_PATH_STYLE_ACCESS =
"logfeeder.s3.path.style.access";
public static final String S3_MULTIOBJECT_DELETE_ENABLE =
"logfeeder.s3.multiobject.delete.enable";
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index e89f7f4..d32e1df 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++
[ambari-logsearch] 26/28: AMBARI-24833. Extend cloud log rolling options (#25)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit ff5d318d4d4a76c7eebc48aff13c87326779d7f9
Author: Olivér Szabó
AuthorDate: Fri Nov 16 11:49:15 2018 +0100
AMBARI-24833. Extend cloud log rolling options (#25)
* AMBARI-24833. Extend cloud log rolling options
* Add some more fixes, use default-fs if hdfs-client is used with cloud
storage.
* AMBARI-24833. Overide hdfs username, if core hdfs is used as fs
* AMBARI-24833. Remove aws-java-sdk-bundle dependency (~100MB)
* AMBARI-24833. change size format to size unit
---
ambari-logsearch-logfeeder/pom.xml | 11 +
.../logfeeder/common/LogFeederConstants.java | 9 ++--
.../logfeeder/conf/CloudStorageDestination.java| 2 +-
.../ambari/logfeeder/conf/LogFeederProps.java | 20 +++-
.../conf/output/ExternalHdfsOutputConfig.java | 17 ---
.../logfeeder/conf/output/RolloverConfig.java | 54 ++
.../output/cloud/CloudStorageLoggerFactory.java| 9 +++-
.../output/cloud/CloudStorageUploader.java | 2 +-
.../cloud/upload/ExternalHDFSUploadClient.java | 6 +--
.../output/cloud/upload/HDFSUploadClient.java | 12 -
.../output/cloud/upload/UploadClientFactory.java | 1 +
.../src/main/resources/logfeeder.properties| 4 +-
12 files changed, 105 insertions(+), 42 deletions(-)
diff --git a/ambari-logsearch-logfeeder/pom.xml
b/ambari-logsearch-logfeeder/pom.xml
index e71b3cc..003b233 100644
--- a/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch-logfeeder/pom.xml
@@ -150,6 +150,12 @@
org.apache.hadoop
hadoop-aws
${hadoop.version}
+
+
+ com.amazonaws
+ aws-java-sdk-bundle
+
+
com.amazonaws
@@ -162,6 +168,11 @@
${aws-sdk.version}
+ com.amazonaws
+ aws-java-sdk-dynamodb
+ ${aws-sdk.version}
+
+
org.apache.hadoop
hadoop-azure
${hadoop.version}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index 11d351f..b5fffa8 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -34,10 +34,6 @@ public class LogFeederConstants {
// Event History Constants History
public static final String VALUES = "jsons";
public static final String ROW_TYPE = "rowtype";
-
- // S3 Constants
- public static final String S3_PATH_START_WITH = "s3://";
- public static final String S3_PATH_SEPARATOR = "/";
public static final String IN_MEMORY_TIMESTAMP = "in_memory_timestamp";
@@ -122,14 +118,17 @@ public class LogFeederConstants {
public static final String CLOUD_ROLLOVER_ARCHIVE_LOCATION =
"logfeeder.cloud.rollover.archive.base.dir";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_MIN =
"logfeeder.cloud.rollover.threshold.min";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE =
"logfeeder.cloud.rollover.threshold.size";
+ public static final String CLOUD_ROLLOVER_MAX_BACKUP_FILES =
"logfeeder.cloud.rollover.max.files";
+ public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE_UNIT =
"logfeeder.cloud.rollover.threshold.size.unit";
public static final String CLOUD_ROLLOVER_USE_GZIP =
"logfeeder.cloud.rollover.use.gzip";
public static final String CLOUD_ROLLOVER_IMMEDIATE_FLUSH =
"logfeeder.cloud.rollover.immediate.flush";
public static final String CLOUD_ROLLOVER_ON_SHUTDOWN =
"logfeeder.cloud.rollover.on.shutdown";
public static final String CLOUD_ROLLOVER_ON_STARTUP =
"logfeeder.cloud.rollover.on.startup";
+ public static final String HDFS_USER = "logfeeder.hdfs.user";
+
public static final String HDFS_HOST = "logfeeder.hdfs.host";
public static final String HDFS_PORT = "logfeeder.hdfs.port";
- public static final String HDFS_USER = "logfeeder.hdfs.user";
public static final String HDFS_FILE_PERMISSIONS =
"logfeeder.hdfs.file.permissions";
public static final String HDFS_KERBEROS = "logfeeder.hdfs.kerberos";
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
index 1a7eafa..0246b98 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
@@ -19,7 +19,7 @@
package
[ambari-logsearch] 13/28: Update maven versioning (#11)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 31d3ca18795a0104d578d494080a61a42ce078a2
Author: Olivér Szabó
AuthorDate: Wed Oct 24 15:46:17 2018 +0200
Update maven versioning (#11)
---
Makefile | 2 +-
README.md | 2 +-
ambari-logsearch-appender/pom.xml | 2 +-
ambari-logsearch-assembly/pom.xml | 2 +-
ambari-logsearch-config-api/pom.xml | 2 +-
ambari-logsearch-config-json/pom.xml | 2 +-
ambari-logsearch-config-local/pom.xml | 2 +-
ambari-logsearch-config-solr/pom.xml | 2 +-
ambari-logsearch-config-zookeeper/pom.xml | 2 +-
ambari-logsearch-it/pom.xml | 2 +-
ambari-logsearch-log4j2-appender/pom.xml | 2 +-
ambari-logsearch-logfeeder-container-registry/pom.xml | 2 +-
ambari-logsearch-logfeeder-plugin-api/pom.xml | 2 +-
ambari-logsearch-logfeeder/pom.xml| 2 +-
ambari-logsearch-server/pom.xml | 2 +-
ambari-logsearch-web/pom.xml | 4 +---
pom.xml | 5 +++--
17 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/Makefile b/Makefile
index 6ad38e4..5f9c8b0 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,7 @@ deb:
$(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
update-version:
- $(MAVEN_BINARY) versions:set -DnewVersion=$(new-version)
-DgenerateBackupPoms=false
+ $(MAVEN_BINARY) versions:set-property -Dproperty=revision
-DnewVersion=$(new-version) -DgenerateBackupPoms=false
docker-build:
$(MAVEN_BINARY) clean package docker:build -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
diff --git a/README.md b/README.md
index 07309a1..47d09a1 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Log Search is a sub-project of [Apache
Ambari](https://github.com/apache/ambari)
## Development
-Requires JDK 8 (JDK 11 is recommended)
+Requires JDK 8 (JDK 11 is recommended) and Maven 3.5.x
### Prerequisites
diff --git a/ambari-logsearch-appender/pom.xml
b/ambari-logsearch-appender/pom.xml
index f83ada8..92f68b9 100644
--- a/ambari-logsearch-appender/pom.xml
+++ b/ambari-logsearch-appender/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
ambari-logsearch-appender
diff --git a/ambari-logsearch-assembly/pom.xml
b/ambari-logsearch-assembly/pom.xml
index 19c3d90..21cb491 100644
--- a/ambari-logsearch-assembly/pom.xml
+++ b/ambari-logsearch-assembly/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
Ambari Logsearch Assembly
http://maven.apache.org
diff --git a/ambari-logsearch-config-api/pom.xml
b/ambari-logsearch-config-api/pom.xml
index add7ddf..75e1840 100644
--- a/ambari-logsearch-config-api/pom.xml
+++ b/ambari-logsearch-config-api/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
4.0.0
diff --git a/ambari-logsearch-config-json/pom.xml
b/ambari-logsearch-config-json/pom.xml
index 05565bd..cf65c33 100644
--- a/ambari-logsearch-config-json/pom.xml
+++ b/ambari-logsearch-config-json/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
4.0.0
diff --git a/ambari-logsearch-config-local/pom.xml
b/ambari-logsearch-config-local/pom.xml
index 7e50f7c..c8cf081 100644
--- a/ambari-logsearch-config-local/pom.xml
+++ b/ambari-logsearch-config-local/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
4.0.0
diff --git a/ambari-logsearch-config-solr/pom.xml
b/ambari-logsearch-config-solr/pom.xml
index f47fd75..27be12d 100644
--- a/ambari-logsearch-config-solr/pom.xml
+++ b/ambari-logsearch-config-solr/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
4.0.0
diff --git a/ambari-logsearch-config-zookeeper/pom.xml
b/ambari-logsearch-config-zookeeper/pom.xml
index 343971e..5b5fc8a 100644
--- a/ambari-logsearch-config-zookeeper/pom.xml
+++ b/ambari-logsearch-config-zookeeper/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
org.apache.ambari
-2.0.0.0-SNAPSHOT
+${revision}
4.0.0
diff --git a/ambari-logsearch-it/pom.xml b/ambari-logsearch-it/pom.xml
index a4e160f..eda11d8 100644
--- a/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch-it/pom.xml
@@ -21,7 +21,7 @@
ambari-logsearch
[ambari-logsearch] 16/28: AMBARI-24851 - Logsearch: debug mode using java 8 and 11 (#15)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit ab9e8c0e6228a5c530490ee2b5667d1c7e26bca1
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Wed Oct 31 12:52:55 2018 +0100
AMBARI-24851 - Logsearch: debug mode using java 8 and 11 (#15)
---
ambari-logsearch-server/src/main/scripts/logsearch.sh | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/ambari-logsearch-server/src/main/scripts/logsearch.sh
b/ambari-logsearch-server/src/main/scripts/logsearch.sh
index 1b64832..a821168 100755
--- a/ambari-logsearch-server/src/main/scripts/logsearch.sh
+++ b/ambari-logsearch-server/src/main/scripts/logsearch.sh
@@ -149,7 +149,12 @@ function start() {
LOGSEARCH_DEBUG_PORT=${LOGSEARCH_DEBUG_PORT:-"5005"}
if [ "$LOGSEARCH_DEBUG" = "true" ]; then
-LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug
-Xrunjdwp:transport=dt_socket,address=*:$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND
"
+if [ $java_version == "8" ]; then
+ LOGSEARCH_DEBUG_ADDRESS=$LOGSEARCH_DEBUG_PORT
+else
+ LOGSEARCH_DEBUG_ADDRESS="*:$LOGSEARCH_DEBUG_PORT"
+fi
+LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug
-Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_ADDRESS,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND
"
fi
if [ "$LOGSEARCH_SSL" = "true" ]; then
[ambari-logsearch] 15/28: AMBARI-24825. Log Feeder: Fix HDFS/S3 outputs (#13)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit c3a750a73e9f34f99494dad581563c2ed4307a15
Author: Olivér Szabó
AuthorDate: Thu Oct 25 15:36:31 2018 +0200
AMBARI-24825. Log Feeder: Fix HDFS/S3 outputs (#13)
---
.../apache/ambari/logfeeder/output/OutputFile.java | 2 +-
.../ambari/logfeeder/output/OutputHDFSFile.java| 7 ++
.../ambari/logfeeder/output/OutputS3File.java | 17 ++-
.../ambari/logfeeder/output/spool/LogSpooler.java | 5 +-
.../src/main/scripts/logfeeder.sh | 2 +-
.../src/main/scripts/logsearch.sh | 2 +-
docker/cloud-docker-compose.yml| 129 +
.../logfeeder/shipper-conf/output.config.json | 38 ++
8 files changed, 196 insertions(+), 6 deletions(-)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
index 7dae1b8..7495444 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
@@ -98,7 +98,7 @@ public class OutputFile extends Output {
String outStr = null;
CSVPrinter csvPrinter = null;
try {
- if (codec.equals("csv")) {
+ if ("csv".equals(codec)) {
csvPrinter = new CSVPrinter(outWriter, CSVFormat.RFC4180);
//TODO:
} else {
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
index ed93aa4..93a2643 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
@@ -39,6 +39,7 @@ import java.io.File;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
@@ -114,6 +115,12 @@ public class OutputHDFSFile extends Output impl
}
}
+ @Override
+ public void write(Map jsonObj, InputFileMarker inputMarker)
throws Exception {
+String block = LogFeederUtil.getGson().toJson(jsonObj);
+write(block, inputMarker);
+ }
+
@Override
public String getShortDescription() {
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
index 7d7e6af..a2f6b08 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
@@ -192,12 +192,22 @@ public class OutputS3File extends OutputFile implements
RolloverCondition, Rollo
*/
@Override
public void write(String block, InputFileMarker inputMarker) {
+createLogSpoolerIfRequired(inputMarker);
+logSpooler.add(block);
+ }
+
+ @Override
+ public void write(Map jsonObj, InputFileMarker inputMarker)
throws Exception {
+String block = LogFeederUtil.getGson().toJson(jsonObj);
+write(block, inputMarker);
+ }
+
+ private void createLogSpoolerIfRequired(InputFileMarker inputMarker) {
if (logSpooler == null) {
if (inputMarker.getInput().getClass().isAssignableFrom(InputFile.class))
{
InputFile input = (InputFile) inputMarker.getInput();
logSpooler = createSpooler(input.getFilePath());
s3Uploader = createUploader(input.getInputDescriptor().getType());
-logSpooler.add(block);
} else {
logger.error("Cannot write from non local file...");
}
@@ -261,4 +271,9 @@ public class OutputS3File extends OutputFile implements
RolloverCondition, Rollo
public void handleRollover(File rolloverFile) {
s3Uploader.addFileForUpload(rolloverFile.getAbsolutePath());
}
+
+ @Override
+ public String getShortDescription() {
+return "output:destination=s3,bucket=" +
s3OutputConfiguration.getS3BucketName();
+ }
}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
index 7d7d111..82a3f1b 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
@@ -45,8 +45,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class LogSpooler {
private static
[ambari-logsearch] 27/28: AMBARI-24833. Support for cloud logs to using filters + JSON output (#26)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit f8b80a5193b7f0f71c7e0ceaa7ebb311a435d072
Author: Olivér Szabó
AuthorDate: Mon Nov 19 10:32:15 2018 +0100
AMBARI-24833. Support for cloud logs to using filters + JSON output (#26)
* AMBARI-24833. Support for cloud logs to using filters + JSON output
* AMBARI-24833. Do not filter anything if filters are not enabled
* AMBARI-24833. Fix intermittent issues.
* AMBARI-24833. Edit comment
---
.../local/LogSearchConfigLogFeederLocal.java | 42 ++--
.../config/zookeeper/LogLevelFilterManagerZK.java | 1 +
.../logfeeder/common/LogFeederConstants.java | 1 +
.../ambari/logfeeder/conf/LogFeederProps.java | 18
...andler.java => AbstractInputConfigHandler.java} | 84 ++--
.../impl/CloudStorageInputConfigHandler.java | 14 ++-
.../operations/impl/DefaultInputConfigHandler.java | 62 +---
.../logfeeder/output/OutputLineEnricher.java | 109 +
.../ambari/logfeeder/output/OutputManagerImpl.java | 76 ++
.../output/cloud/CloudStorageLoggerFactory.java| 14 ++-
.../output/cloud/CloudStorageOutputManager.java| 27 -
.../output/cloud/CloudStorageUploader.java | 2 +-
.../src/main/resources/logfeeder.properties| 1 +
13 files changed, 232 insertions(+), 219 deletions(-)
diff --git
a/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
b/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
index f6cb519..12af637 100644
---
a/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
+++
b/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
@@ -84,12 +84,7 @@ public class LogSearchConfigLogFeederLocal extends
LogSearchConfigLocal implemen
File[] inputConfigFiles = new
File(configDir).listFiles(inputConfigFileFilter);
if (inputConfigFiles != null) {
for (File inputConfigFile : inputConfigFiles) {
-String inputConfig = new
String(Files.readAllBytes(inputConfigFile.toPath()));
-Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
-m.find();
-String serviceName = m.group(1);
-JsonElement inputConfigJson =
JsonHelper.mergeGlobalConfigWithInputConfig(parser, inputConfig,
globalConfigNode);
-inputConfigMonitor.loadInputConfigs(serviceName,
InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+tryLoadingInputConfig(inputConfigMonitor, parser, globalConfigNode,
inputConfigFile);
}
}
final FileSystem fs = FileSystems.getDefault();
@@ -100,6 +95,41 @@ public class LogSearchConfigLogFeederLocal extends
LogSearchConfigLocal implemen
executorService.submit(updater);
}
+ private void tryLoadingInputConfig(InputConfigMonitor inputConfigMonitor,
JsonParser parser, JsonArray globalConfigNode, File inputConfigFile) throws
Exception {
+// note: that will try to solve a intermittent issue when the input config
json is a null string (during file generation), that process will re-try to
process the files a few times
+int tries = 0;
+while(true) {
+ tries++;
+ Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
+ m.find();
+ String inputConfig = new
String(Files.readAllBytes(inputConfigFile.toPath()));
+ String serviceName = m.group(1);
+ JsonElement inputConfigJson = null;
+ logger.info("Trying to load '{}' service input config from input file
'{}'", serviceName, inputConfigFile.getAbsolutePath());
+ try {
+inputConfigJson = JsonHelper.mergeGlobalConfigWithInputConfig(parser,
inputConfig, globalConfigNode);
+ } catch (Exception e) {
+final String errorMessage;
+if (tries < 3) {
+ errorMessage = String.format("Cannot parse input config: %s, will
retry in a few seconds again (tries: %s)", inputConfig, String.valueOf(tries));
+ logger.error(errorMessage, e);
+ try {
+Thread.sleep(2000);
+ } catch (Exception ex) {
+// skip
+ }
+ continue;
+} else {
+ errorMessage = String.format("Cannot parse input config: %s, after
%s tries. Will skip to processing it", inputConfig, String.valueOf(tries));
+ logger.error(errorMessage, e);
+ break;
+}
+ }
+ inputConfigMonitor.loadInputConfigs(serviceName,
InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+ break;
+}
+ }
+
@Override
public void close() throws IOException {
}
diff
[ambari-logsearch] 17/28: AMBARI-24851 - ADDENDUM Logsearch: debug mode using java 8 and 11 (#16)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 1425fd26e632715d29e01cc4e6a54b72b76cd576
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Wed Oct 31 17:04:04 2018 +0100
AMBARI-24851 - ADDENDUM Logsearch: debug mode using java 8 and 11 (#16)
---
ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
index eaf00e1..213e564 100755
--- a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
+++ b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
@@ -168,7 +168,12 @@ function start() {
LOGFEEDER_DEBUG_PORT=${LOGFEEDER_DEBUG_PORT:-"5006"}
if [ "$LOGFEEDER_DEBUG" = "true" ]; then
-LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug
-Xrunjdwp:transport=dt_socket,address=*:$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND
"
+if [ $java_version == "8" ]; then
+ LOGFEEDER_DEBUG_ADDRESS=$LOGFEEDER_DEBUG_PORT
+else
+ LOGFEEDER_DEBUG_ADDRESS="*:$LOGFEEDER_DEBUG_PORT"
+fi
+LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug
-Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_ADDRESS,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND
"
fi
if [ "$LOGFEEDER_SSL" = "true" ]; then
[ambari-logsearch] 03/28: AMBARI-24743. Cleanup logsearch: add/cleanup javadoc. (#4)
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit 54b055fde96bb3124ef2667dcef38a07a05094cf Author: Olivér Szabó AuthorDate: Mon Oct 8 12:58:26 2018 +0200 AMBARI-24743. Cleanup logsearch: add/cleanup javadoc. (#4) * AMBARI-24743. Cleanup logsearch: add/cleanup javadoc. * AMBARI-24743. Add some more Javadocs (mainly for Log Feeder) * AMBARI-24743. Fixxes based on review --- Makefile | 3 + .../logsearch/config/api/InputConfigMonitor.java | 2 +- .../config/api/LogLevelFilterManager.java | 11 ++- .../config/api/LogLevelFilterMonitor.java | 4 + .../config/api/LogLevelFilterUpdater.java | 1 + .../logsearch/config/api/LogSearchConfig.java | 4 +- .../config/api/LogSearchConfigLogFeeder.java | 6 +- .../config/api/LogSearchConfigServer.java | 6 +- .../config/api/LogSearchPropertyDescription.java | 10 +-- .../api/ShipperConfigElementDescription.java | 10 +-- .../config/api/ShipperConfigTypeDescription.java | 4 +- ambari-logsearch-config-json/pom.xml | 2 +- .../config/zookeeper/LogLevelFilterManagerZK.java | 3 + .../zookeeper/LogSearchConfigLogFeederZK.java | 3 + .../config/zookeeper/LogSearchConfigServerZK.java | 3 + .../config/zookeeper/LogSearchConfigZK.java| 3 + .../config/zookeeper/LogSearchConfigZKHelper.java | 23 +- .../container/docker/command/ContainerCommand.java | 2 +- .../ambari/logfeeder/plugin/common/AliasUtil.java | 5 +- .../ambari/logfeeder/plugin/common/ConfigItem.java | 22 ++--- .../plugin/common/LogFeederProperties.java | 3 +- .../ambari/logfeeder/plugin/common/MetricData.java | 3 + .../ambari/logfeeder/plugin/filter/Filter.java | 50 +++- .../logfeeder/plugin/filter/mapper/Mapper.java | 45 +- .../ambari/logfeeder/plugin/input/Input.java | 95 -- .../ambari/logfeeder/plugin/input/InputMarker.java | 13 +++ .../logfeeder/plugin/input/cache/LRUCache.java | 11 +++ .../logfeeder/plugin/manager/BlockManager.java | 17 .../plugin/manager/CheckpointManager.java | 40 - .../logfeeder/plugin/manager/InputManager.java | 38 - .../logfeeder/plugin/manager/OutputManager.java| 33 ++-- .../ambari/logfeeder/plugin/output/Output.java | 61 +++--- .../ambari/logfeeder/common/ConfigHandler.java | 3 + .../logfeeder/common/LogEntryParseTester.java | 13 ++- .../common/LogFeederSolrClientFactory.java | 11 +++ .../ambari/logfeeder/filter/DockerLogFilter.java | 8 ++ .../apache/ambari/logfeeder/filter/FilterGrok.java | 30 ++- .../apache/ambari/logfeeder/filter/FilterJSON.java | 19 + .../ambari/logfeeder/filter/FilterKeyValue.java| 37 - .../logfeeder/input/InputConfigUploader.java | 3 + .../apache/ambari/logfeeder/input/InputFile.java | 38 +++-- .../ambari/logfeeder/input/InputManagerImpl.java | 43 ++ .../apache/ambari/logfeeder/input/InputS3File.java | 14 +++- .../ambari/logfeeder/input/InputSimulate.java | 3 + .../apache/ambari/logfeeder/input/InputSocket.java | 3 + .../logfeeder/input/file/ProcessFileHelper.java| 10 +++ .../file/checkpoint/FileCheckpointManager.java | 3 + .../file/checkpoint/util/CheckpointFileReader.java | 15 .../file/checkpoint/util/FileCheckInHelper.java| 8 ++ .../util/FileCheckpointCleanupHelper.java | 8 ++ .../checkpoint/util/ResumeLineNumberHelper.java| 9 ++ .../input/monitor/AbstractLogFileMonitor.java | 3 + .../input/monitor/CheckpointCleanupMonitor.java| 3 + .../input/monitor/DockerLogFileUpdateMonitor.java | 17 ++-- .../ambari/logfeeder/input/reader/GZIPReader.java | 3 + .../input/reader/LogsearchReaderFactory.java | 9 ++ .../loglevelfilter/LogLevelFilterHandler.java | 3 + .../ambari/logfeeder/mapper/MapperAnonymize.java | 18 +++- .../apache/ambari/logfeeder/mapper/MapperDate.java | 15 +++- .../ambari/logfeeder/mapper/MapperFieldCopy.java | 14 +++- .../ambari/logfeeder/mapper/MapperFieldName.java | 14 +++- .../ambari/logfeeder/mapper/MapperFieldValue.java | 13 ++- .../apache/ambari/logfeeder/output/OutputFile.java | 2 +- .../ambari/logfeeder/output/OutputHDFSFile.java| 2 +- .../ambari/logfeeder/output/OutputLineFilter.java | 7 +- .../ambari/logfeeder/output/OutputManagerImpl.java | 33 +++- .../ambari/logfeeder/output/OutputS3File.java | 3 +- .../apache/ambari/logfeeder/output/OutputSolr.java | 38 +++-- .../org/apache/ambari/logfeeder/util/S3Util.java | 6 ++ .../logfeeder/mapper/MapperAnonymizeTest.java | 6 +- .../ambari/logfeeder/mapper/MapperDateTest.java| 12 +-- .../logfeeder/mapper/MapperFieldCopyTest.java | 4
[ambari-logsearch] 12/28: AMBARI-24692. ADDENDUM: fix java version check
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit a5b8b2af271e4e3339a6228c07c1d888d12fe473 Author: Oliver Szabo AuthorDate: Thu Oct 18 18:08:31 2018 +0200 AMBARI-24692. ADDENDUM: fix java version check --- ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh | 2 +- ambari-logsearch-server/src/main/scripts/logsearch.sh| 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh index 37f5b00..e125768 100755 --- a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh +++ b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh @@ -80,7 +80,7 @@ else LOGFEEDER_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGFEEDER_GC_LOGFILE" fi -java_version=$($JVM -version 2>&1 | grep 'java version' | cut -d'"' -f2 | cut -d'.' -f2) +java_version=$($JVM -version 2>&1 | grep 'version' | cut -d'"' -f2 | cut -d'.' -f2) if [ $java_version == "8" ]; then LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE" else diff --git a/ambari-logsearch-server/src/main/scripts/logsearch.sh b/ambari-logsearch-server/src/main/scripts/logsearch.sh index acd183b..b941e25 100755 --- a/ambari-logsearch-server/src/main/scripts/logsearch.sh +++ b/ambari-logsearch-server/src/main/scripts/logsearch.sh @@ -78,7 +78,7 @@ else LOGSEARCH_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGSEARCH_GC_LOGFILE" fi -java_version=$($JVM -version 2>&1 | grep 'java version' | cut -d'"' -f2 | cut -d'.' -f2) +java_version=$($JVM -version 2>&1 | grep 'version' | cut -d'"' -f2 | cut -d'.' -f2) if [ $java_version == "8" ]; then LOGSEARCH_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGSEARCH_GC_LOGFILE" else
[ambari-logsearch] 05/28: AMBARI-24692. Use jdk8 as default compiler version (#7)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 1457c4de1cd983432d3e4ff0bd9328ff70fa3127
Author: Olivér Szabó
AuthorDate: Wed Oct 10 20:59:57 2018 +0200
AMBARI-24692. Use jdk8 as default compiler version (#7)
* AMBARI-24692. Use jdk8 as default compiler version
* AMBARI-24692. Revert jvm gc settings
* AMBARI-24692. Implement TODO: java version check.
* AMBARI-24692. Extend jdk version to env variable in Makefile
---
Makefile | 48 +++---
.../src/main/scripts/logfeeder.sh | 7 +++-
.../src/main/scripts/logsearch.sh | 7 +++-
pom.xml| 2 +-
4 files changed, 28 insertions(+), 36 deletions(-)
diff --git a/Makefile b/Makefile
index 16d3782..6ad38e4 100644
--- a/Makefile
+++ b/Makefile
@@ -19,59 +19,41 @@ else
LOGSEARCH_BUILD_DOCKER_TAG = "latest"
endif
+ifeq ("$(LOGSEARCH_JDK_11)", "true")
+ LOGSEARCH_JAVA_VERSION = "11"
+else
+ LOGSEARCH_JAVA_VERSION = "1.8"
+endif
+
package:
- $(MAVEN_BINARY) clean package
+ $(MAVEN_BINARY) clean package -Djdk.version=$(LOGSEARCH_JAVA_VERSION)
install:
- $(MAVEN_BINARY) clean install -DskipTests
+ $(MAVEN_BINARY) clean install -DskipTests
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
be:
- $(MAVEN_BINARY) clean package -Pbe
+ $(MAVEN_BINARY) clean package -Pbe
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
fe:
- $(MAVEN_BINARY) clean package -Pfe
+ $(MAVEN_BINARY) clean package -Pfe
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
test:
- $(MAVEN_BINARY) clean test
+ $(MAVEN_BINARY) clean test -Djdk.version=$(LOGSEARCH_JAVA_VERSION)
rpm:
- $(MAVEN_BINARY) clean package -Dbuild-rpm -DskipTests
+ $(MAVEN_BINARY) clean package -Dbuild-rpm -DskipTests
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
deb:
- $(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests
+ $(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
update-version:
$(MAVEN_BINARY) versions:set -DnewVersion=$(new-version)
-DgenerateBackupPoms=false
-package-jdk8:
- $(MAVEN_BINARY) clean package -Djdk.version=1.8
-
-install-jdk8:
- $(MAVEN_BINARY) clean install -DskipTests -Djdk.version=1.8
-
-be-jdk8:
- $(MAVEN_BINARY) clean package -Pbe -Djdk.version=1.8
-
-fe-jdk8:
- $(MAVEN_BINARY) clean package -Pfe -Djdk.version=1.8
-
-test-jdk8:
- $(MAVEN_BINARY) clean test -Djdk.version=1.8
-
-rpm-jdk8:
- $(MAVEN_BINARY) clean package -Dbuild-rpm -DskipTests -Djdk.version=1.8
-
-deb-jdk8:
- $(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests -Djdk.version=1.8
-
-javadoc:
- $(MAVEN_BINARY) javadoc:javadoc
-
docker-build:
- $(MAVEN_BINARY) clean package docker:build -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
+ $(MAVEN_BINARY) clean package docker:build -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
docker-push:
- $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
+ $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
-Djdk.version=$(LOGSEARCH_JAVA_VERSION)
docker-dev-start:
cd docker && docker-compose up -d
diff --git a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
index b9141f0..37f5b00 100755
--- a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
+++ b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh
@@ -80,7 +80,12 @@ else
LOGFEEDER_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGFEEDER_GC_LOGFILE"
fi
-LOGFEEDER_GC_OPTS="-Xlog:gc*:file=$LOGFEEDER_GC_LOGFILE:time"
+java_version=$($JVM -version 2>&1 | grep 'java version' | cut -d'"' -f2 | cut
-d'.' -f2)
+if [ $java_version == "8" ]; then
+ LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps
-Xloggc:$LOGFEEDER_GC_LOGFILE"
+else
+ LOGFEEDER_GC_OPTS="-Xlog:gc*:file=$LOGFEEDER_GC_LOGFILE:time"
+fi
function print_usage() {
cat << EOF
diff --git a/ambari-logsearch-server/src/main/scripts/logsearch.sh
b/ambari-logsearch-server/src/main/scripts/logsearch.sh
index 629e8f2..acd183b 100755
--- a/ambari-logsearch-server/src/main/scripts/logsearch.sh
+++ b/ambari-logsearch-server/src/main/scripts/logsearch.sh
@@ -78,7 +78,12 @@ else
LOGSEARCH_GC_LOGFILE="$LOG_PATH_WITHOUT_SLASH/$LOGSEARCH_GC_LOGFILE"
fi
-LOGSEARCH_GC_OPTS="-Xlog:gc*:file=$LOGSEARCH_GC_LOGFILE:time"
+java_version=$($JVM -version 2>&1 | grep 'java version' | cut -d'"' -f2 | cut
-d'.' -f2)
+if [ $java_version == "8" ]; then
+
[ambari-logsearch] 19/28: AMBARI-24875. Log Search: cannot build sub-modules separately. (#20)
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit f26a9f81b30486bcec30a56d7edb6ac3af8f0d6f Author: Olivér Szabó AuthorDate: Sun Nov 11 18:54:32 2018 +0100 AMBARI-24875. Log Search: cannot build sub-modules separately. (#20) --- .gitignore | 1 + pom.xml| 24 2 files changed, 25 insertions(+) diff --git a/.gitignore b/.gitignore index e9491db..5062a44 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ node/ .hg .hgignore .hgtags +.flattened-pom.xml diff --git a/pom.xml b/pom.xml index 21d236c..a5e4ef2 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,30 @@ + +org.codehaus.mojo +flatten-maven-plugin +1.0.0 + + true + + + +flatten +process-resources + + flatten + + + +flatten.clean +clean + + clean + + + + org.codehaus.mojo rpm-maven-plugin
[ambari-logsearch] 07/28: AMBARI-24765. Fix CVE issues for Log Search (#8)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit e4537608ac6c34f8c38ed0351626690c926ad4ae
Author: Olivér Szabó
AuthorDate: Fri Oct 12 11:25:54 2018 +0200
AMBARI-24765. Fix CVE issues for Log Search (#8)
---
ambari-logsearch-logfeeder-container-registry/pom.xml | 4 ++--
pom.xml | 9 +
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/ambari-logsearch-logfeeder-container-registry/pom.xml
b/ambari-logsearch-logfeeder-container-registry/pom.xml
index e601034..6983d09 100644
--- a/ambari-logsearch-logfeeder-container-registry/pom.xml
+++ b/ambari-logsearch-logfeeder-container-registry/pom.xml
@@ -40,12 +40,12 @@
com.fasterxml.jackson.core
jackson-databind
- 2.9.4
+ ${fasterxml-jackson.version}
com.fasterxml.jackson.core
jackson-annotations
- 2.9.4
+ ${fasterxml-jackson.version}
commons-lang
diff --git a/pom.xml b/pom.xml
index e40b81e..7630919 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,6 +93,7 @@
3.8.0
2.7.0.0.0
latest
+2.9.5
@@ -389,22 +390,22 @@
com.fasterxml.jackson.core
jackson-databind
-2.9.4
+${fasterxml-jackson.version}
com.fasterxml.jackson.core
jackson-annotations
-2.9.4
+${fasterxml-jackson.version}
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
-2.9.4
+${fasterxml-jackson.version}
com.fasterxml.jackson.dataformat
jackson-dataformat-xml
-2.9.4
+${fasterxml-jackson.version}
com.fasterxml.woodstox
[ambari-logsearch] 09/28: AMBARI-24799. Update Spring versions
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit 6a9af2664113b10189a2f467db76ad045d9eb0dc Author: Oliver Szabo AuthorDate: Wed Oct 17 23:25:54 2018 +0200 AMBARI-24799. Update Spring versions --- ambari-logsearch-logfeeder/pom.xml | 4 ++-- ambari-logsearch-server/pom.xml| 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch-logfeeder/pom.xml index 1f8a20c..d2efc79 100644 --- a/ambari-logsearch-logfeeder/pom.xml +++ b/ambari-logsearch-logfeeder/pom.xml @@ -33,8 +33,8 @@ UTF-8 -5.1.0.RELEASE -2.0.5.RELEASE +5.1.1.RELEASE +2.0.6.RELEASE diff --git a/ambari-logsearch-server/pom.xml b/ambari-logsearch-server/pom.xml index 5649361..0695d13 100755 --- a/ambari-logsearch-server/pom.xml +++ b/ambari-logsearch-server/pom.xml @@ -27,11 +27,11 @@ http://maven.apache.org Ambari Logsearch Server -2.0.5.RELEASE +2.0.6.RELEASE -5.1.0.RELEASE +5.1.1.RELEASE 1.3.3.RELEASE -5.1.0.RELEASE +5.1.1.RELEASE 2.27 9.4.12.v20180830 1.5.16
[ambari-logsearch] 04/28: AMBARI-24746. Upgrade Solr version to 7.5.0 (#6)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 1436800f27243d4bcd4395fd36c8a69c9ee550fa
Author: Olivér Szabó
AuthorDate: Mon Oct 8 22:36:42 2018 +0200
AMBARI-24746. Upgrade Solr version to 7.5.0 (#6)
---
.../src/main/configsets/audit_logs/conf/solrconfig.xml | 2 +-
.../src/main/configsets/hadoop_logs/conf/solrconfig.xml | 2 +-
ambari-logsearch-server/src/main/configsets/history/conf/solrconfig.xml | 2 +-
docker/Dockerfile | 2 +-
docker/docker-compose.yml | 2 +-
docker/logsearch-docker.sh | 2 +-
docker/solr.yml | 2 +-
docker/sso.yml | 2 +-
pom.xml | 2 +-
9 files changed, 9 insertions(+), 9 deletions(-)
diff --git
a/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
b/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
index 8f54121..ea77a09 100644
--- a/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
+++ b/ambari-logsearch-server/src/main/configsets/audit_logs/conf/solrconfig.xml
@@ -35,7 +35,7 @@
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
- 7.4.0
+ 7.5.0
- 7.4.0
+ 7.5.0
- 7.4.0
+ 7.5.0
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 1c8fac2..41a5ff1 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -55,7 +55,7 @@ RUN echo 'X11DisplayOffset 10\n' /etc/ssh/sshd_config
RUN git config --global url."https://".insteadOf git://
# Install Solr
-ENV SOLR_VERSION 7.4.0
+ENV SOLR_VERSION 7.5.0
RUN wget --no-check-certificate -O /root/solr-$SOLR_VERSION.tgz
http://public-repo-1.hortonworks.com/ARTIFACTS/dist/lucene/solr/$SOLR_VERSION/solr-$SOLR_VERSION.tgz
RUN cd /root && tar -zxvf /root/solr-$SOLR_VERSION.tgz
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index fb14622..dc136bf 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -26,7 +26,7 @@ services:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
-image: solr:${SOLR_VERSION:-7.4.0}
+image: solr:${SOLR_VERSION:-7.5.0}
restart: always
hostname: solr
ports:
diff --git a/docker/logsearch-docker.sh b/docker/logsearch-docker.sh
index 72a332a..8966361 100755
--- a/docker/logsearch-docker.sh
+++ b/docker/logsearch-docker.sh
@@ -109,7 +109,7 @@ AMBARI_LOCATION=$AMBARI_LOCATION
ZOOKEEPER_VERSION=3.4.10
ZOOKEEPER_CONNECTION_STRING=zookeeper:2181
-SOLR_VERSION=7.4.0
+SOLR_VERSION=7.5.0
EOF
echo ".env file has been created. Check it out before starting Log Search.
($sdir/.env)"
exit
diff --git a/docker/solr.yml b/docker/solr.yml
index 2975af6..5bef837 100644
--- a/docker/solr.yml
+++ b/docker/solr.yml
@@ -15,7 +15,7 @@
version: '3.3'
services:
solr:
-image: solr:${SOLR_VERSION:-7.4.0}
+image: solr:${SOLR_VERSION:-7.5.0}
restart: always
networks:
- logsearch-network
diff --git a/docker/sso.yml b/docker/sso.yml
index 0837dd8..9d27d4f 100644
--- a/docker/sso.yml
+++ b/docker/sso.yml
@@ -26,7 +26,7 @@ services:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
-image: solr:${SOLR_VERSION:-7.4.0}
+image: solr:${SOLR_VERSION:-7.5.0}
restart: always
hostname: solr
ports:
diff --git a/pom.xml b/pom.xml
index d6ecde5..649cf3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -82,7 +82,7 @@
python (= 2.6)
amd64
${deb.python.ver}
-7.4.0
+7.5.0
3.0.0
2.5
3.4.6.2.3.0.0-2557
[ambari-logsearch] 14/28: Make rule to set version is update-version, not set-version (#12)
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit 1b9059a6855fd42bedc3dc86210274fb68d9f585 Author: Doroszlai, Attila <6454655+adorosz...@users.noreply.github.com> AuthorDate: Wed Oct 24 16:06:15 2018 +0200 Make rule to set version is update-version, not set-version (#12) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47d09a1..21b94fc 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Also you can run from the IDE, but make sure all of the ambari logsearch modules ## Update version (for release or specific builds) ```bash -make set-version new-version="2.8.0.0-11" +make update-version new-version="2.8.0.0-11" ``` ## Contributing
[ambari-logsearch] 18/28: AMBARI-24833. Create cloud input/output skeleton. (#17)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 9f6b8053242bbd6c2c978b3ecfda149233a604e1
Author: Olivér Szabó
AuthorDate: Thu Nov 1 13:33:30 2018 +0100
AMBARI-24833. Create cloud input/output skeleton. (#17)
* AMBARI-24833. Create cloud input/output skeleton.
* AMBARI-24833. Use LogFeederMode as enum by spring
* AMBARI-24833. Fix review issues + add secret stores.
---
.../ambari/logfeeder/plugin/input/Input.java | 14 +
ambari-logsearch-logfeeder/pom.xml | 6 +-
.../logfeeder/common/LogEntryParseTester.java | 11 +-
.../logfeeder/common/LogFeederConstants.java | 6 +-
.../ambari/logfeeder/conf/ApplicationConfig.java | 136 ++--
.../ambari/logfeeder/conf/LogFeederMode.java | 58 +++
.../ambari/logfeeder/conf/LogFeederProps.java | 18 +
.../logfeeder/conf/LogFeederSecurityConfig.java| 53 +--
.../conf/condition/CloudStorageCondition.java | 37 ++
.../conf/condition/NonCloudStorageCondition.java | 37 ++
.../logfeeder/credential/CompositeSecretStore.java | 39 +++
.../logfeeder/credential/EnvSecretStore.java | 37 ++
.../logfeeder/credential/FileSecretStore.java | 60
.../credential/HadoopCredentialSecretStore.java| 52 +++
.../logfeeder/credential/PropertySecretStore.java | 36 ++
.../ambari/logfeeder/credential/SecretStore.java | 30 ++
.../ambari/logfeeder/filter/FilterDummy.java | 58 +++
.../logfeeder/input/InputConfigUploader.java | 26 +-
.../apache/ambari/logfeeder/input/InputFile.java | 16 +-
.../ambari/logfeeder/input/InputManagerImpl.java | 12 +-
.../apache/ambari/logfeeder/input/InputSocket.java | 2 +-
.../ambari/logfeeder/manager/BlockMerger.java | 66
.../logfeeder/manager/InputConfigHolder.java | 80 +
.../InputConfigManager.java} | 388 ++---
.../manager/operations/InputConfigHandler.java | 53 +++
.../impl/CloudStorageInputConfigHandler.java | 101 ++
.../operations/impl/DefaultInputConfigHandler.java | 166 +
.../ambari/logfeeder/metrics/StatsLogger.java | 16 +-
.../ambari/logfeeder/output/OutputManagerImpl.java | 1 +
.../output/cloud/CloudStorageFactory.java | 32 ++
.../logfeeder/output/cloud/CloudStorageOutput.java | 30 ++
.../output/cloud/CloudStorageOutputManager.java| 102 ++
.../ambari/logfeeder/output/cloud/HDFSOutput.java | 74
.../src/main/resources/logfeeder.properties| 3 +
.../ambari/logfeeder/output/OutputS3FileTest.java | 3 -
.../ambari/logfeeder/output/S3UploaderTest.java| 4 -
.../logfeeder/output/spool/LogSpoolerTest.java | 8 -
.../org/apache/ambari/logsearch/LogSearch.java | 3 +-
docker/test-config/logfeeder/logfeeder.properties | 3 +-
39 files changed, 1471 insertions(+), 406 deletions(-)
diff --git
a/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java
b/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java
index 6228637..9ee4533 100644
---
a/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java
+++
b/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java
@@ -71,6 +71,7 @@ public abstract class Input parse() throws Exception {
InputConfig inputConfig = getInputConfig();
-ConfigHandler configHandler = new ConfigHandler(null);
-configHandler.setInputManager(new InputManagerImpl());
OutputManagerImpl outputManager = new OutputManagerImpl();
LogFeederProps logFeederProps = new LogFeederProps();
LogEntryCacheConfig logEntryCacheConfig = new LogEntryCacheConfig();
@@ -101,8 +101,11 @@ public class LogEntryParseTester {
LogLevelFilterHandler logLevelFilterHandler = new
LogLevelFilterHandler(null);
logLevelFilterHandler.setLogFeederProps(logFeederProps);
outputManager.setLogLevelFilterHandler(logLevelFilterHandler);
-configHandler.setOutputManager(outputManager);
-Input input = configHandler.getTestInput(inputConfig, logId);
+DefaultInputConfigHandler configHandler = new DefaultInputConfigHandler();
+InputConfigManager inputConfigManager = new InputConfigManager(
+ null, new InputManagerImpl(), outputManager,
configHandler,logFeederProps, true
+);
+Input input = inputConfigManager.getTestInput(inputConfig, logId);
input.init(logFeederProps);
final Map result = new HashMap<>();
input.getFirstFilter().init(logFeederProps);
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
[ambari-logsearch] 01/28: AMBARI-24739. Add contribution guide to Log Search.
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit 4250f99b6e72eb254d1d569cefe6bc0edfce6e77 Author: Oliver Szabo AuthorDate: Fri Oct 5 00:34:57 2018 +0200 AMBARI-24739. Add contribution guide to Log Search. --- Makefile | 9 + README.md | 27 +-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 0f026ce..6eb4b22 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,9 @@ rpm: deb: $(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests +update-version: + $(MAVEN_BINARY) versions:set -DnewVersion=$(new-version) -DgenerateBackupPoms=false + package-jdk8: $(MAVEN_BINARY) clean package -Djdk.version=1.8 @@ -66,3 +69,9 @@ docker-build: docker-push: $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests -Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG) + +docker-dev-start: + cd docker && docker-compose up -d + +docker-dev-build-and-start: + $(MAVEN_BINARY) clean package && cd docker && docker-compose up -d diff --git a/README.md b/README.md index 3bab2c3..a2fc3e8 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,17 @@ In case if you started the containers separately and if you would like to access 1. Check out the code from GIT repository -2. On the logsearch root folder (ambari/ambari-logsearch), please execute the following Maven command to build RPM/DPKG: +2. On the logsearch root folder (ambari/ambari-logsearch), please execute the following make command to build RPM/DPKG: ```bash -# for building with jdk 8, use -Djdk.version=1.8 -mvn -Dbuild-rpm clean package +make rpm +# or for jdk8 +make rpm-jdk8 ``` or ```bash -# for building with jdk 8, use -Djdk.version=1.8 -mvn -Dbuild-deb clean package +make deb +# or for jdk8 +make deb-jdk8 ``` 3. Generated RPM/DPKG files will be found in ambari-logsearch-assembly/target folder @@ -75,7 +77,20 @@ mvn clean integration-test -Dselenium-tests failsafe:verify ``` Also you can run from the IDE, but make sure all of the ambari logsearch modules are built. + +## Update version (for release or specific builds) + +```bash +make set-version new-version="2.8.0.0-11" +``` + +## Contributing + +https://cwiki.apache.org/confluence/display/AMBARI/How+to+Contribute + +(That is the ambari contribution guide, everything is the same here except use ambari-logsearch repository instead of ambari) + ## License -- http://ambari.apache.org/license.html +- http://ambari.apache.org/license.html - See more at [Ambari repository](https://github.com/apache/ambari)
[ambari-logsearch] 02/28: [AMBARI-24656] [Log Search UI] Handle the 401 and the 403 response status at login (#2)
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit d8529088869223ee7eee6b33a6b7f9bf6adab92f Author: Istvan Tobias AuthorDate: Fri Oct 5 18:16:16 2018 +0200 [AMBARI-24656] [Log Search UI] Handle the 401 and the 403 response status at login (#2) --- ambari-logsearch-web/karma.conf.js | 1 + ambari-logsearch-web/package.json | 3 + ambari-logsearch-web/src/app/app.module.ts | 16 +- .../src/app/classes/models/app-state.ts| 10 +- .../src/app/classes/models/store.ts| 35 ++-- .../models/user.ts}| 9 +- ambari-logsearch-web/src/app/classes/string.ts | 1 - .../action-menu/action-menu.component.html | 2 +- .../action-menu/action-menu.component.spec.ts | 16 +- .../action-menu/action-menu.component.ts | 12 +- .../src/app/components/app.component.html | 11 +- .../src/app/components/app.component.less | 7 + .../src/app/components/app.component.ts| 59 -- .../audit-logs-entries.component.spec.ts | 14 +- .../audit-logs-table.component.spec.ts | 16 +- .../cluster-filter.component.spec.ts | 5 +- .../cluster-filter/cluster-filter.component.ts | 6 +- .../context-menu/context-menu.component.spec.ts| 10 +- .../filters-panel/filters-panel.component.html | 62 +-- .../filters-panel/filters-panel.component.spec.ts | 14 +- .../log-context/log-context.component.spec.ts | 14 +- .../log-index-filter.component.html| 4 +- .../log-index-filter.component.less| 6 + .../log-index-filter.component.spec.ts | 16 +- .../log-index-filter/log-index-filter.component.ts | 4 +- .../login-form/login-form.component.html | 19 +- .../login-form/login-form.component.less | 13 +- .../login-form/login-form.component.spec.ts| 56 ++ .../components/login-form/login-form.component.ts | 69 +++ .../logs-container.component.spec.ts | 19 +- .../logs-container/logs-container.component.ts | 2 +- .../main-container/main-container.component.ts | 29 +-- .../service-logs-table.component.spec.ts | 10 +- .../time-range-picker.component.spec.ts| 16 +- .../timezone-picker.component.spec.ts | 10 +- .../components/top-menu/top-menu.component.spec.ts | 10 +- .../app/components/top-menu/top-menu.component.ts | 15 +- .../src/app/modules/app-load/app-load.module.ts| 23 ++- .../modules/app-load/services/app-load.service.ts | 41 +++-- .../dropdown-button/dropdown-button.component.ts | 10 +- .../dropdown-list/dropdown-list.component.html | 8 +- .../dropdown-list/dropdown-list.component.spec.ts | 10 +- .../dropdown-list/dropdown-list.component.ts | 6 +- .../filter-dropdown.component.spec.ts | 10 +- .../modal-dialog/modal-dialog.component.spec.ts| 20 ++- .../shared/interfaces/notification.interface.ts| 7 +- .../shared/services/notification.service.ts| 2 +- .../src/app/modules/shared/variables.less | 3 +- .../src/app/services/auth-guard.service.ts | 12 +- .../src/app/services/auth.service.spec.ts | 84 - .../src/app/services/auth.service.ts | 143 ++- .../services/component-generator.service.spec.ts | 14 +- .../app/services/history-manager.service.spec.ts | 16 +- .../src/app/services/http-client.service.ts| 18 +- .../app/services/log-index-filter.service.spec.ts | 7 +- .../src/app/services/login-screen-guard.service.ts | 11 +- .../app/services/logs-container.service.spec.ts| 14 +- .../src/app/services/logs-container.service.ts | 98 +- .../src/app/services/storage/reducers.service.ts | 5 +- .../src/app/services/user-settings.service.spec.ts | 12 +- .../src/app/store/actions/auth.actions.ts | 101 +++ .../actions/notification.actions.ts} | 19 +- .../src/app/store/effects/auth.effects.ts | 198 + .../effects/notification.effects.ts} | 40 +++-- .../src/app/store/reducers/auth.reducers.ts| 120 + .../src/app/store/selectors/auth.selectors.ts | 47 + ambari-logsearch-web/src/app/test-config.spec.ts | 41 +++-- ambari-logsearch-web/src/assets/i18n/en.json | 8 +- ambari-logsearch-web/yarn.lock | 31 69 files changed, 1256 insertions(+), 544 deletions(-) diff --git a/ambari-logsearch-web/karma.conf.js b/ambari-logsearch-web/karma.conf.js index 08608d8..b7e9d03 100644 --- a/ambari-logsearch-web/karma.conf.js +++ b/ambari-logsearch-web/karma.conf.js @@ -26,6 +26,7 @@ module.exports =
[ambari-logsearch] 10/28: Add repo definitions to logsearch server
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit 2402f1c416e2faef7c822986a936189267f4346d Author: Oliver Szabo AuthorDate: Thu Oct 18 15:09:44 2018 +0200 Add repo definitions to logsearch server --- ambari-logsearch-server/pom.xml | 36 +++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/ambari-logsearch-server/pom.xml b/ambari-logsearch-server/pom.xml index 0695d13..d18a47c 100755 --- a/ambari-logsearch-server/pom.xml +++ b/ambari-logsearch-server/pom.xml @@ -41,6 +41,40 @@ 0.6.0 4.0.1 + + + apache-hadoop + hdp + http://repo.hortonworks.com/content/groups/public/ + + + oss.sonatype.org + OSS Sonatype Staging + https://oss.sonatype.org/content/groups/staging + + + spring-milestones + Spring Milestones + http://repo.spring.io/milestone + +false + + + + ASF Staging + https://repository.apache.org/content/groups/staging/ + + + ASF Snapshots + https://repository.apache.org/content/repositories/snapshots/ + +true + + +false + + + @@ -307,7 +341,7 @@ org.springframework.boot spring-boot-starter-jersey - 2.1.0.M2 + 2.1.0.M4 tomcat-embed-el
[ambari-logsearch] 28/28: AMBARI-24833. HDFS client kerberos support + small fixes (#27)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit d7499927f4dc2349a45261a23b0b955b173250d8
Author: Olivér Szabó
AuthorDate: Mon Nov 19 14:19:57 2018 +0100
AMBARI-24833. HDFS client kerberos support + small fixes (#27)
* AMBARI-24833. HDFS client kerberos support + small fixes
* AMBARI-24833. Fix principal description
---
.../logfeeder/common/LogFeederConstants.java | 2 +
.../ambari/logfeeder/conf/LogFeederProps.java | 27 ++--
...HdfsOutputConfig.java => HdfsOutputConfig.java} | 65 +--
.../impl/AbstractInputConfigHandler.java | 2 +-
.../impl/CloudStorageInputConfigHandler.java | 2 +-
.../operations/impl/DefaultInputConfigHandler.java | 2 +-
.../logfeeder/output/OutputLineEnricher.java | 2 +-
.../cloud/upload/ExternalHDFSUploadClient.java | 73 --
.../output/cloud/upload/HDFSUploadClient.java | 55
.../output/cloud/upload/UploadClientFactory.java | 4 +-
10 files changed, 114 insertions(+), 120 deletions(-)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index f9ef32d..a15ac74 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -132,6 +132,8 @@ public class LogFeederConstants {
public static final String HDFS_PORT = "logfeeder.hdfs.port";
public static final String HDFS_FILE_PERMISSIONS =
"logfeeder.hdfs.file.permissions";
public static final String HDFS_KERBEROS = "logfeeder.hdfs.kerberos";
+ public static final String HDFS_KERBEROS_KEYTAB = "logfeeder.hdfs.keytab";
+ public static final String HDFS_KERBEROS_PRINCIPAL =
"logfeeder.hdfs.principal";
public static final String S3_ENDPOINT = "logfeeder.s3.endpoint";
public static final String S3_ENDPOINT_DEFAULT = "https://s3.amazonaws.com;;
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index f2eb6c7..b6ab4c7 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -19,7 +19,7 @@
package org.apache.ambari.logfeeder.conf;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.conf.output.ExternalHdfsOutputConfig;
+import org.apache.ambari.logfeeder.conf.output.HdfsOutputConfig;
import org.apache.ambari.logfeeder.conf.output.RolloverConfig;
import org.apache.ambari.logfeeder.conf.output.S3OutputConfig;
import org.apache.ambari.logfeeder.plugin.common.LogFeederProperties;
@@ -53,7 +53,7 @@ public class LogFeederProps implements LogFeederProperties {
private S3OutputConfig s3OutputConfig;
@Inject
- private ExternalHdfsOutputConfig hdfsOutputConfig;
+ private HdfsOutputConfig hdfsOutputConfig;
private Properties properties;
@@ -258,7 +258,7 @@ public class LogFeederProps implements LogFeederProperties {
defaultValue = "false",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
- @Value("${" + LogFeederConstants.CLOUD_STORAGE_USE_HDFS_CLIENT + ":false}")
+ @Value("${" + LogFeederConstants.CLOUD_STORAGE_USE_HDFS_CLIENT + ":true}")
private boolean useCloudHdfsClient;
@LogSearchPropertyDescription(
@@ -281,15 +281,6 @@ public class LogFeederProps implements LogFeederProperties
{
private String cloudBasePath;
@LogSearchPropertyDescription(
-name = LogFeederConstants.HDFS_USER,
-description = "Overrides HADOOP_USER_NAME variable at runtime",
-examples = {"hdfs"},
-sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
- )
- @Value("${"+ LogFeederConstants.HDFS_USER + ":}")
- private String logfeederHdfsUser;
-
- @LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_STORAGE_USE_FILTERS,
description = "Use filters for inputs (with filters the output format will
be JSON)",
examples = {"true"},
@@ -460,7 +451,7 @@ public class LogFeederProps implements LogFeederProperties {
this.cloudStorageMode = cloudStorageMode;
}
- public ExternalHdfsOutputConfig getHdfsOutputConfig() {
+ public HdfsOutputConfig getHdfsOutputConfig() {
return hdfsOutputConfig;
}
@@ -480,7 +471,7 @@ public class LogFeederProps implements LogFeederProperties {
this.rolloverConfig = rolloverConfig;
}
- public void setHdfsOutputConfig(ExternalHdfsOutputConfig hdfsOutputConfig) {
+
[ambari-logsearch] 08/28: AMBARI-24789. LogFeeder: fix CVE-2018-11771 - upgrade commons-compress lib (#9)
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git commit d9afa7cd670cb3e37e723cbc31517a1ef397a7f3 Author: Olivér Szabó AuthorDate: Tue Oct 16 19:32:48 2018 +0200 AMBARI-24789. LogFeeder: fix CVE-2018-11771 - upgrade commons-compress lib (#9) --- ambari-logsearch-logfeeder/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch-logfeeder/pom.xml index 8b89ff6..1f8a20c 100644 --- a/ambari-logsearch-logfeeder/pom.xml +++ b/ambari-logsearch-logfeeder/pom.xml @@ -152,7 +152,7 @@ org.apache.commons commons-compress - 1.16.1 + 1.18 com.fasterxml.jackson.dataformat
[ambari-logsearch] branch cloudbreak updated (5670040 -> d749992)
This is an automated email from the ASF dual-hosted git repository. oleewere pushed a change to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git. omit 5670040 [AMBARI-24551] [Log Search UI] get rid of redundant requests after undoing or redoing several history steps (#22) omit db5fb3d AMBARI-24833. Simplify HDFS client usage + use core-site.xml (#21) omit 2bd96e1 AMBARI-24833. Re-implement S3/HDFS outputs as global cloud outputs (#19) omit 9c83460 AMBARI-24847 - Logsearch: Cannot search for a term that includes spaces or dashes (#18) omit 2c7d5f0 AMBARI-24875. Log Search: cannot build sub-modules separately. (#20) omit 9bce50d AMBARI-24833. Create cloud input/output skeleton. (#17) omit f0c35f3 AMBARI-24851 - ADDENDUM Logsearch: debug mode using java 8 and 11 (#16) omit 845e0c6 AMBARI-24851 - Logsearch: debug mode using java 8 and 11 (#15) omit 879ba60 AMBARI-24825. Log Feeder: Fix HDFS/S3 outputs (#13) omit 5fec024 Make rule to set version is update-version, not set-version (#12) omit f694bdd Update maven versioning (#11) omit 521b17b AMBARI-24692. ADDENDUM: fix java version check omit 49408e4 AMBARI-24802 - Logsearch: upgrade swagger-ui to 3.19.0 (#10) omit 4471753 Add repo definitions to logsearch server omit 3f31765 AMBARI-24799. Update Spring versions omit efb573c AMBARI-24789. LogFeeder: fix CVE-2018-11771 - upgrade commons-compress lib (#9) omit 98af5df AMBARI-24765. Fix CVE issues for Log Search (#8) omit 1d68049 Update README.md omit 302042c AMBARI-24692. Use jdk8 as default compiler version (#7) omit bac521f AMBARI-24746. Upgrade Solr version to 7.5.0 (#6) omit 365fd0a AMBARI-24743. Cleanup logsearch: add/cleanup javadoc. (#4) omit 2ba9216 [AMBARI-24656] [Log Search UI] Handle the 401 and the 403 response status at login (#2) omit 7742a73 Merge pull request #3 from oleewere/AMBARI-24739-master omit d619fa1 AMBARI-24739. Add contribution guide to Log Search. new 4250f99 AMBARI-24739. Add contribution guide to Log Search. new d852908 [AMBARI-24656] [Log Search UI] Handle the 401 and the 403 response status at login (#2) new 54b055f AMBARI-24743. Cleanup logsearch: add/cleanup javadoc. (#4) new 1436800 AMBARI-24746. Upgrade Solr version to 7.5.0 (#6) new 1457c4d AMBARI-24692. Use jdk8 as default compiler version (#7) new b8c217c Update README.md new e453760 AMBARI-24765. Fix CVE issues for Log Search (#8) new d9afa7c AMBARI-24789. LogFeeder: fix CVE-2018-11771 - upgrade commons-compress lib (#9) new 6a9af26 AMBARI-24799. Update Spring versions new 2402f1c Add repo definitions to logsearch server new 3053cff AMBARI-24802 - Logsearch: upgrade swagger-ui to 3.19.0 (#10) new a5b8b2a AMBARI-24692. ADDENDUM: fix java version check new 31d3ca1 Update maven versioning (#11) new 1b9059a Make rule to set version is update-version, not set-version (#12) new c3a750a AMBARI-24825. Log Feeder: Fix HDFS/S3 outputs (#13) new ab9e8c0 AMBARI-24851 - Logsearch: debug mode using java 8 and 11 (#15) new 1425fd2 AMBARI-24851 - ADDENDUM Logsearch: debug mode using java 8 and 11 (#16) new 9f6b805 AMBARI-24833. Create cloud input/output skeleton. (#17) new f26a9f8 AMBARI-24875. Log Search: cannot build sub-modules separately. (#20) new a06f977 AMBARI-24847 - Logsearch: Cannot search for a term that includes spaces or dashes (#18) new 3a0e80a AMBARI-24833. Re-implement S3/HDFS outputs as global cloud outputs (#19) new 1485bd6 AMBARI-24833. Simplify HDFS client usage + use core-site.xml (#21) new 9c7c5ba [AMBARI-24551] [Log Search UI] get rid of redundant requests after undoing or redoing several history steps (#22) new 94c71eb [AMBARI-24891] [Log Serach UI] The Log Index Filter panel does not work (#23) new 324d8f4 [AMBARI-24899] [Log Search UI] Change capture countdown text (#24) new ff5d318 AMBARI-24833. Extend cloud log rolling options (#25) new f8b80a5 AMBARI-24833. Support for cloud logs to using filters + JSON output (#26) new d749992 AMBARI-24833. HDFS client kerberos support + small fixes (#27) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (5670040) \ N -- N -- N refs/heads/cloudbreak (d749992) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone
[ambari-logsearch] 20/28: AMBARI-24847 - Logsearch: Cannot search for a term that includes spaces or dashes (#18)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit a06f977aef19d6227c4b0380e7cb39ae176c6f31
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Mon Nov 12 14:45:13 2018 +0100
AMBARI-24847 - Logsearch: Cannot search for a term that includes spaces or
dashes (#18)
---
ambari-logsearch-it/pom.xml| 10 +
.../org/apache/ambari/logsearch/solr/Solr.java | 115 +++
.../ambari/logsearch/solr/SolrDocumentMatcher.java | 60
.../solr/search/SearchServiceLogsTest.java | 376 +
.../main/configsets/audit_logs/conf/managed-schema | 2 +-
.../configsets/hadoop_logs/conf/managed-schema | 2 +-
.../AbstractDateRangeFacetQueryConverter.java | 10 +-
.../AbstractOperationHolderConverter.java | 72 ++--
.../app/services/logs-filtering-utils.service.ts | 2 +-
9 files changed, 598 insertions(+), 51 deletions(-)
diff --git a/ambari-logsearch-it/pom.xml b/ambari-logsearch-it/pom.xml
index eda11d8..889525e 100644
--- a/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch-it/pom.xml
@@ -60,6 +60,16 @@
${solr.version}
+ org.apache.solr
+ solr-core
+ ${solr.version}
+
+
+ org.apache.solr
+ solr-dataimporthandler
+ ${solr.version}
+
+
junit
junit
diff --git
a/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/Solr.java
b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/Solr.java
new file mode 100644
index 000..ee0bc8c
--- /dev/null
+++
b/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/solr/Solr.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.solr;
+
+import static
org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_EVTTIME;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.ID;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.LOG_MESSAGE;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LEVEL;
+import static
org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants.LOGTIME;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.PivotField;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.NodeConfig;
+import org.apache.solr.core.SolrResourceLoader;
+
+public class Solr {
+ public static final DateTimeFormatter SOLR_DATETIME_FORMATTER =
DateTimeFormatter.ofPattern("-MM-dd'T'HH:mm:ss.SSSX");
+
+ public static Solr core(String coreName) throws IOException,
SolrServerException {
+assert coreName != null;
+
+String targetLocation =
Solr.class.getProtectionDomain().getCodeSource().getLocation().getFile() + "..";
+String logsearchConfigSetDir = targetLocation +
"/../../ambari-logsearch-server/src/main/configsets";
+File targetConfigSetDir = new File(targetLocation + "/configsets");
+if (targetConfigSetDir.exists())
+ FileUtils.deleteDirectory(targetConfigSetDir);
+FileUtils.copyDirectory(new File(logsearchConfigSetDir),
targetConfigSetDir);
+String solrHome = targetLocation + "/solr";
+File solrHomeDir = new File(solrHome);
+if (solrHomeDir.exists())
+ FileUtils.deleteDirectory(solrHomeDir);
+solrHomeDir.mkdirs();
+
+SolrResourceLoader solrResourceLoader = new
SolrResourceLoader(solrHomeDir.toPath());
+
+
[ambari-logsearch] 24/28: [AMBARI-24891] [Log Serach UI] The Log Index Filter panel does not work (#23)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch cloudbreak
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit 94c71ebda3f9929221405e94c33f9d3c8e4dca14
Author: Istvan Tobias
AuthorDate: Fri Nov 16 10:36:47 2018 +0100
[AMBARI-24891] [Log Serach UI] The Log Index Filter panel does not work
(#23)
* [AMBARI-24891] [Log Serach UI] The Log Index Filter panel does not work -
dropdown selection fixes
* [AMBARI-24891] [Log Serach UI] The Log Index Filter panel does not work -
auto selection fix
---
.../action-menu/action-menu.component.ts | 3 +++
.../filter-button/filter-button.component.ts | 2 +-
.../menu-button/menu-button.component.html | 2 +-
.../menu-button/menu-button.component.ts | 8
.../dropdown-button/dropdown-button.component.html | 8 +---
.../dropdown-button/dropdown-button.component.ts | 23 +++---
.../dropdown-list/dropdown-list.component.ts | 5 ++---
.../filter-dropdown/filter-dropdown.component.ts | 16 +--
8 files changed, 41 insertions(+), 26 deletions(-)
diff --git
a/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
b/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
index 721ae93..a5d9ae2 100644
---
a/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
+++
b/ambari-logsearch-web/src/app/components/action-menu/action-menu.component.ts
@@ -75,6 +75,9 @@ export class ActionMenuComponent implements OnInit,
OnDestroy {
this.selectedClusterName$.takeUntil(this.destroyed$).subscribe(
(clusterName: string) => this.setModalSubmitDisabled(!clusterName)
);
+this.clustersListItems$.filter((items: ListItem[]) => items.some((item:
ListItem) => item.isChecked)).first()
+ .map((items: ListItem[]) => items.find((item: ListItem) =>
item.isChecked))
+ .subscribe((item) => this.selectedClusterName$.next(item.value));
}
ngOnDestroy() {
diff --git
a/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.ts
b/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.ts
index d6f24e5..21cc151 100644
---
a/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.ts
+++
b/ambari-logsearch-web/src/app/components/filter-button/filter-button.component.ts
@@ -51,7 +51,7 @@ export class FilterButtonComponent extends
MenuButtonComponent implements Contro
}
writeValue(items: ListItem[]) {
-this.selection = items;
+this.selection = items ? (Array.isArray(items) ? items : [items] ) : [];
}
registerOnChange(callback: any): void {
diff --git
a/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
b/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
index 7061def..28cde2b 100644
---
a/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
+++
b/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.html
@@ -15,7 +15,7 @@
limitations under the License.
-->
-
+
diff --git
a/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
b/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
index faf2165..3174991 100644
---
a/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
+++
b/ambari-logsearch-web/src/app/components/menu-button/menu-button.component.ts
@@ -127,6 +127,14 @@ export class MenuButtonComponent {
return this.subItems && this.subItems.filter((option: ListItem): boolean
=> option.isChecked);
}
+ get hasSelection(): boolean {
+return this.subItems && this.subItems.filter((option: ListItem): boolean
=> option.isChecked).length > 0;
+ }
+
+ get totalSelection(): number {
+return this.subItems ? this.subItems.filter((option: ListItem): boolean =>
option.isChecked).length : 0;
+ }
+
constructor(private utils: UtilsService) {}
findItemIndexInList(item: ListItem, itemList: ListItem[] = this.subItems):
number {
diff --git
a/ambari-logsearch-web/src/app/modules/shared/components/dropdown-button/dropdown-button.component.html
b/ambari-logsearch-web/src/app/modules/shared/components/dropdown-button/dropdown-button.component.html
index 7140028..f4fdf4f 100644
---
a/ambari-logsearch-web/src/app/modules/shared/components/dropdown-button/dropdown-button.component.html
+++
b/ambari-logsearch-web/src/app/modules/shared/components/dropdown-button/dropdown-button.component.html
@@ -15,17 +15,19 @@
limitations under the License.
-->
-
+
-
{{label}}
- {{selection[0].label |
translate}}
+
+{{ item.label | translate }}
+
diff --git
[GitHub] asfgit commented on issue #29: AMBARI-24833. Use clustername and hostname in cloud log archive dir
asfgit commented on issue #29: AMBARI-24833. Use clustername and hostname in cloud log archive dir URL: https://github.com/apache/ambari-logsearch/pull/29#issuecomment-439942525 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/Ambari-LogSearch-Github-PR-Builder/60/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] oleewere opened a new pull request #29: AMBARI-24833. Use clustername and hostname in cloud log archive dir
oleewere opened a new pull request #29: AMBARI-24833. Use clustername and hostname in cloud log archive dir URL: https://github.com/apache/ambari-logsearch/pull/29 # What changes were proposed in this pull request? Use hostname + cluster name as well in archive dir (to make it work properly with mounts) ## How was this patch tested? FT manually with docker env please review @g-boros This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[ambari] branch trunk updated: AMBARI-24919 external Namenode HA (benyoka) (#2625)
This is an automated email from the ASF dual-hosted git repository.
benyoka pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 93576b7 AMBARI-24919 external Namenode HA (benyoka) (#2625)
93576b7 is described below
commit 93576b7c9bc25361ba1e43c9dfb08fc136667520
Author: benyoka
AuthorDate: Mon Nov 19 16:50:28 2018 +0100
AMBARI-24919 external Namenode HA (benyoka) (#2625)
---
.../internal/BlueprintConfigurationProcessor.java | 26 ++-
.../ambari/server/topology/ClusterTopology.java| 5 +
.../server/topology/ClusterTopologyImpl.java | 27 +--
.../topology/validators/NameNodeHaValidator.java | 128 ++
.../validators/TopologyValidatorFactory.java | 9 +-
.../BlueprintConfigurationProcessorTest.java | 257 +
.../server/topology/ClusterTopologyImplTest.java | 55 +
.../validators/NameNodeHaValidatorTest.java| 219 ++
8 files changed, 649 insertions(+), 77 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index 1daf1f5..46d7e9b 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -57,6 +57,8 @@ import
org.apache.ambari.server.topology.ConfigRecommendationStrategy;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.HostGroup;
import org.apache.ambari.server.topology.HostGroupInfo;
+import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.validators.NameNodeHaValidator;
import org.apache.ambari.server.topology.validators.UnitValidatedProperty;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -480,6 +482,23 @@ public class BlueprintConfigurationProcessor {
private void doNameNodeHAUpdateOnClusterCreation(Configuration clusterConfig,
Map> clusterProps,
Set
configTypesUpdated) throws ConfigurationTopologyException {
+
+final Collection nnHosts =
clusterTopology.getHostAssignmentsForComponent("NAMENODE");
+
+// external namenodes
+if (nnHosts.isEmpty()) {
+ LOG.info("NAMENODE HA is enabled but there are no NAMENODE components in
the cluster. Assuming external name nodes.");
+ // need to redo validation here as required information (explicit
hostnames for some host groups) may have been
+ // missing by the time the ClusterTopology object was validated
+ try {
+new NameNodeHaValidator().validateExternalNamenodeHa(clusterTopology);
+ }
+ catch (InvalidTopologyException ex) {
+throw new ConfigurationTopologyException(ex.getMessage(), ex);
+ }
+ return;
+}
+
// add "dfs.internal.nameservices" if it's not specified
Map hdfsSiteConfig =
clusterConfig.getFullProperties().get("hdfs-site");
String nameservices = hdfsSiteConfig.get("dfs.nameservices");
@@ -496,10 +515,9 @@ public class BlueprintConfigurationProcessor {
LOG.info("Processing a single HDFS NameService, which indicates a
default HDFS NameNode HA deployment");
// if the active/standby namenodes are not specified, assign them
automatically
if (! isNameNodeHAInitialActiveNodeSet(clusterProps) && !
isNameNodeHAInitialStandbyNodeSet(clusterProps)) {
-Collection nnHosts =
clusterTopology.getHostAssignmentsForComponent("NAMENODE");
-if (nnHosts.size() < 2) {
- throw new ConfigurationTopologyException("NAMENODE HA requires at
least 2 hosts running NAMENODE but there are: " +
-nnHosts.size() + " Hosts: " + nnHosts);
+if (nnHosts.size() == 1) { // can't be 0 as in that case was handled
above
+ throw new ConfigurationTopologyException("NAMENODE HA requires at
least two hosts running NAMENODE but there is " +
+"only one: " + nnHosts.iterator().next());
}
// set the properties that configure which namenode is active,
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
index e1cab2b..4d8bba3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java
@@ -20,6 +20,7 @@ package org.apache.ambari.server.topology;
import java.util.Collection;
import java.util.Map;
+import
[ambari] branch trunk updated: AMBARI-24913. New LDAP related properties to indicate if Ambari should manage LDAP configuration for certain services (#2629)
This is an automated email from the ASF dual-hosted git repository.
smolnar pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 153d5f9 AMBARI-24913. New LDAP related properties to indicate if
Ambari should manage LDAP configuration for certain services (#2629)
153d5f9 is described below
commit 153d5f96cf1c1c68084d308bb98314b3d5231153
Author: Sandor Molnar
AuthorDate: Mon Nov 19 15:54:41 2018 +0100
AMBARI-24913. New LDAP related properties to indicate if Ambari should
manage LDAP configuration for certain services (#2629)
---
.../AmbariServerConfigurationKey.java | 3 +
.../AmbariServerLDAPConfigurationHandler.java | 2 +-
.../ldap/domain/AmbariLdapConfiguration.java | 8 ++
.../ambari/server/upgrade/UpgradeCatalog270.java | 8 ++
.../main/resources/stacks/ambari_configuration.py | 59
.../server/upgrade/UpgradeCatalog270Test.java | 3 +
.../src/test/python/TestAmbariConfiguration.py | 104 +
7 files changed, 186 insertions(+), 1 deletion(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfigurationKey.java
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfigurationKey.java
index 8599a0d0..05caa75 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfigurationKey.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/configuration/AmbariServerConfigurationKey.java
@@ -25,6 +25,9 @@ public enum AmbariServerConfigurationKey {
/*
* LDAP Configuration Keys
* */
+
AMBARI_MANAGES_LDAP_CONFIGURATION(AmbariServerConfigurationCategory.LDAP_CONFIGURATION,
"ambari.ldap.manage_services", PLAINTEXT, "false", "A Boolean value indicating
whether Ambari is to manage the LDAP configuration for services or not."),
+ LDAP_ENABLED_SERVICES(AmbariServerConfigurationCategory.LDAP_CONFIGURATION,
"ambari.ldap.enabled_services", PLAINTEXT, null, "A comma-delimited list of
services that are expected to be configured for LDAP. A \"*\" indicates all
services."),
+
LDAP_ENABLED(AmbariServerConfigurationCategory.LDAP_CONFIGURATION,
"ambari.ldap.authentication.enabled", PLAINTEXT, "false", "An internal property
used for unit testing and development purposes."),
SERVER_HOST(AmbariServerConfigurationCategory.LDAP_CONFIGURATION,
"ambari.ldap.connectivity.server.host", PLAINTEXT, "localhost", "The LDAP URL
host used for connecting to an LDAP server when authenticating users."),
SERVER_PORT(AmbariServerConfigurationCategory.LDAP_CONFIGURATION,
"ambari.ldap.connectivity.server.port", PLAINTEXT, "33389", "The LDAP URL port
used for connecting to an LDAP server when authenticating users."),
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
index f08c1de..2cc79b2 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AmbariServerLDAPConfigurationHandler.java
@@ -66,7 +66,7 @@ public class AmbariServerLDAPConfigurationHandler extends
AmbariServerStackAdvis
public void updateComponentCategory(String categoryName, Map
properties, boolean removePropertiesIfNotSpecified) throws AmbariException {
super.updateComponentCategory(categoryName, properties,
removePropertiesIfNotSpecified);
final AmbariLdapConfiguration ldapConfiguration = new
AmbariLdapConfiguration(getConfigurationProperties(AmbariServerConfigurationCategory.LDAP_CONFIGURATION.getCategoryName()));
-if (ldapConfiguration.ldapEnabled()) {
+if (ldapConfiguration.isAmbariManagesLdapConfiguration()) {
processClusters(LDAP_CONFIGURATIONS);
}
}
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfiguration.java
b/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfiguration.java
index c55f337..0647138 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfiguration.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/ldap/domain/AmbariLdapConfiguration.java
@@ -71,6 +71,14 @@ public class AmbariLdapConfiguration {
this.configurationMap = configuration;
}
+ public boolean isAmbariManagesLdapConfiguration() {
+return
Boolean.valueOf(configValue(AmbariServerConfigurationKey.AMBARI_MANAGES_LDAP_CONFIGURATION));
+ }
+
+ public String getLdapEnabledServices() {
+return
[GitHub] oleewere closed pull request #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold
oleewere closed pull request #28: AMBARI-24833. Use seconds instead of minutes
for cloud log threshold
URL: https://github.com/apache/ambari-logsearch/pull/28
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index a15ac7468a..1b0aa534d6 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -117,7 +117,7 @@
public static final String CLOUD_STORAGE_BASE_PATH =
"logfeeder.cloud.storage.base.path";
public static final String CLOUD_ROLLOVER_ARCHIVE_LOCATION =
"logfeeder.cloud.rollover.archive.base.dir";
- public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_MIN =
"logfeeder.cloud.rollover.threshold.min";
+ public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SECONDS =
"logfeeder.cloud.rollover.threshold.min";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE =
"logfeeder.cloud.rollover.threshold.size";
public static final String CLOUD_ROLLOVER_MAX_BACKUP_FILES =
"logfeeder.cloud.rollover.max.files";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE_UNIT =
"logfeeder.cloud.rollover.threshold.size.unit";
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
index 282792ae86..bdc908bd5e 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
@@ -37,14 +37,14 @@
private String rolloverArchiveBaseDir;
@LogSearchPropertyDescription(
-name = LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_MIN,
-description = "Rollover cloud log files after an interval (minutes)",
-examples = {"1"},
-defaultValue = "60",
+name = LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SECONDS,
+description = "Rollover cloud log files after an interval (seconds)",
+examples = {"60"},
+defaultValue = "3600",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
- @Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_MIN + ":60}")
- private int rolloverThresholdTimeMins;
+ @Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SECONDS +
":3600}")
+ private int rolloverThresholdTimeSeconds;
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE,
@@ -116,12 +116,12 @@
@Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_ON_STARTUP + ":false}")
private boolean rolloverOnStartup;
- public int getRolloverThresholdTimeMins() {
-return rolloverThresholdTimeMins;
+ public int getRolloverThresholdTimeSeconds() {
+return rolloverThresholdTimeSeconds;
}
- public void setRolloverThresholdTimeMins(int rolloverThresholdTimeMins) {
-this.rolloverThresholdTimeMins = rolloverThresholdTimeMins;
+ public void setRolloverThresholdTimeSeconds(int
rolloverThresholdTimeSeconds) {
+this.rolloverThresholdTimeSeconds = rolloverThresholdTimeSeconds;
}
public Integer getRolloverMaxBackupFiles() {
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
index 0cfdbcc1e2..d39349f4d1 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
@@ -78,7 +78,7 @@ public static Logger createLogger(Input input, LoggerContext
loggerContext, LogF
String rolloverSize =
logFeederProps.getRolloverConfig().getRolloverSize().toString() +
logFeederProps.getRolloverConfig().getRolloverSizeFormat();
SizeBasedTriggeringPolicy sizeBasedTriggeringPolicy =
SizeBasedTriggeringPolicy.createPolicy(rolloverSize);
CustomTimeBasedTriggeringPolicy customTimeBasedTriggeringPolicy =
CustomTimeBasedTriggeringPolicy
-
.createPolicy(String.valueOf(logFeederProps.getRolloverConfig().getRolloverThresholdTimeMins()));
+
.createPolicy(String.valueOf(logFeederProps.getRolloverConfig().getRolloverThresholdTimeSeconds()));
final CompositeTriggeringPolicy
[GitHub] oleewere commented on issue #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold
oleewere commented on issue #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold URL: https://github.com/apache/ambari-logsearch/pull/28#issuecomment-439908578 i will close this, let's keep as minutes This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[ambari] branch trunk updated: AMBARI-24921 If service does not have client service component created then "Run Service Check" option should be made hidden
This is an automated email from the ASF dual-hosted git repository.
atkach pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new f4bd455 AMBARI-24921 If service does not have client service
component created then "Run Service Check" option should be made hidden
f4bd455 is described below
commit f4bd455ed5eb67df592169201455e95c614a69d6
Author: Andrii Tkach
AuthorDate: Mon Nov 19 13:22:22 2018 +0200
AMBARI-24921 If service does not have client service component created then
"Run Service Check" option should be made hidden
---
ambari-web/app/views/main/service/item.js | 3 ++-
ambari-web/test/views/main/service/item_test.js | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/ambari-web/app/views/main/service/item.js
b/ambari-web/app/views/main/service/item.js
index 0186b86..430d62a 100644
--- a/ambari-web/app/views/main/service/item.js
+++ b/ambari-web/app/views/main/service/item.js
@@ -158,7 +158,8 @@ App.MainServiceItemView =
Em.View.extend(App.HiveInteractiveCheck, {
var allMasters =
service.get('hostComponents').filterProperty('isMaster').mapProperty('componentName').uniq();
var allSlaves =
service.get('slaveComponents').rejectProperty('totalCount',
0).mapProperty('componentName');
var actionMap = App.HostComponentActionMap.getMap(this);
-var serviceCheckSupported =
App.get('services.supportsServiceCheck').contains(service.get('serviceName'));
+var serviceCheckSupported =
App.get('services.supportsServiceCheck').contains(service.get('serviceName'))
+ && service.get('installedClients') > 0;
var hasConfigTab = this.get('hasConfigTab');
var excludedCommands = this.get('mastersExcludedCommands');
var serviceName = service.get('serviceName');
diff --git a/ambari-web/test/views/main/service/item_test.js
b/ambari-web/test/views/main/service/item_test.js
index 6d91e3a..4071dc3 100644
--- a/ambari-web/test/views/main/service/item_test.js
+++ b/ambari-web/test/views/main/service/item_test.js
@@ -563,6 +563,7 @@ describe('App.MainServiceItemView', function () {
serviceName: testCase.serviceName,
displayName: testCase.displayName,
serviceTypes: testCase.serviceTypes,
+installedClients: 1,
passiveState: 'OFF'
}),
isSeveralClients: false,
[GitHub] asfgit commented on issue #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold
asfgit commented on issue #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold URL: https://github.com/apache/ambari-logsearch/pull/28#issuecomment-439897680 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/Ambari-LogSearch-Github-PR-Builder/59/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] oleewere opened a new pull request #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold
oleewere opened a new pull request #28: AMBARI-24833. Use seconds instead of minutes for cloud log threshold URL: https://github.com/apache/ambari-logsearch/pull/28 # What changes were proposed in this pull request? Use seconds instead of minutes for cloud log threshold ## How was this patch tested? waiting for UTs This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[ambari-logsearch] branch master updated: AMBARI-24833. HDFS client kerberos support + small fixes (#27)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new c94d3b2 AMBARI-24833. HDFS client kerberos support + small fixes (#27)
c94d3b2 is described below
commit c94d3b20576a179f7a4e04491e423c77a3976ce5
Author: Olivér Szabó
AuthorDate: Mon Nov 19 14:19:57 2018 +0100
AMBARI-24833. HDFS client kerberos support + small fixes (#27)
* AMBARI-24833. HDFS client kerberos support + small fixes
* AMBARI-24833. Fix principal description
---
.../logfeeder/common/LogFeederConstants.java | 2 +
.../ambari/logfeeder/conf/LogFeederProps.java | 27 ++--
...HdfsOutputConfig.java => HdfsOutputConfig.java} | 65 +--
.../impl/AbstractInputConfigHandler.java | 2 +-
.../impl/CloudStorageInputConfigHandler.java | 2 +-
.../operations/impl/DefaultInputConfigHandler.java | 2 +-
.../logfeeder/output/OutputLineEnricher.java | 2 +-
.../cloud/upload/ExternalHDFSUploadClient.java | 73 --
.../output/cloud/upload/HDFSUploadClient.java | 55
.../output/cloud/upload/UploadClientFactory.java | 4 +-
10 files changed, 114 insertions(+), 120 deletions(-)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index f9ef32d..a15ac74 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -132,6 +132,8 @@ public class LogFeederConstants {
public static final String HDFS_PORT = "logfeeder.hdfs.port";
public static final String HDFS_FILE_PERMISSIONS =
"logfeeder.hdfs.file.permissions";
public static final String HDFS_KERBEROS = "logfeeder.hdfs.kerberos";
+ public static final String HDFS_KERBEROS_KEYTAB = "logfeeder.hdfs.keytab";
+ public static final String HDFS_KERBEROS_PRINCIPAL =
"logfeeder.hdfs.principal";
public static final String S3_ENDPOINT = "logfeeder.s3.endpoint";
public static final String S3_ENDPOINT_DEFAULT = "https://s3.amazonaws.com;;
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index f2eb6c7..b6ab4c7 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -19,7 +19,7 @@
package org.apache.ambari.logfeeder.conf;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.conf.output.ExternalHdfsOutputConfig;
+import org.apache.ambari.logfeeder.conf.output.HdfsOutputConfig;
import org.apache.ambari.logfeeder.conf.output.RolloverConfig;
import org.apache.ambari.logfeeder.conf.output.S3OutputConfig;
import org.apache.ambari.logfeeder.plugin.common.LogFeederProperties;
@@ -53,7 +53,7 @@ public class LogFeederProps implements LogFeederProperties {
private S3OutputConfig s3OutputConfig;
@Inject
- private ExternalHdfsOutputConfig hdfsOutputConfig;
+ private HdfsOutputConfig hdfsOutputConfig;
private Properties properties;
@@ -258,7 +258,7 @@ public class LogFeederProps implements LogFeederProperties {
defaultValue = "false",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
- @Value("${" + LogFeederConstants.CLOUD_STORAGE_USE_HDFS_CLIENT + ":false}")
+ @Value("${" + LogFeederConstants.CLOUD_STORAGE_USE_HDFS_CLIENT + ":true}")
private boolean useCloudHdfsClient;
@LogSearchPropertyDescription(
@@ -281,15 +281,6 @@ public class LogFeederProps implements LogFeederProperties
{
private String cloudBasePath;
@LogSearchPropertyDescription(
-name = LogFeederConstants.HDFS_USER,
-description = "Overrides HADOOP_USER_NAME variable at runtime",
-examples = {"hdfs"},
-sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
- )
- @Value("${"+ LogFeederConstants.HDFS_USER + ":}")
- private String logfeederHdfsUser;
-
- @LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_STORAGE_USE_FILTERS,
description = "Use filters for inputs (with filters the output format will
be JSON)",
examples = {"true"},
@@ -460,7 +451,7 @@ public class LogFeederProps implements LogFeederProperties {
this.cloudStorageMode = cloudStorageMode;
}
- public ExternalHdfsOutputConfig getHdfsOutputConfig() {
+ public HdfsOutputConfig getHdfsOutputConfig() {
return hdfsOutputConfig;
}
@@ -480,7 +471,7 @@ public class
[GitHub] oleewere closed pull request #27: AMBARI-24833. HDFS client kerberos support + small fixes
oleewere closed pull request #27: AMBARI-24833. HDFS client kerberos support +
small fixes
URL: https://github.com/apache/ambari-logsearch/pull/27
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index f9ef32d688..a15ac7468a 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -132,6 +132,8 @@
public static final String HDFS_PORT = "logfeeder.hdfs.port";
public static final String HDFS_FILE_PERMISSIONS =
"logfeeder.hdfs.file.permissions";
public static final String HDFS_KERBEROS = "logfeeder.hdfs.kerberos";
+ public static final String HDFS_KERBEROS_KEYTAB = "logfeeder.hdfs.keytab";
+ public static final String HDFS_KERBEROS_PRINCIPAL =
"logfeeder.hdfs.principal";
public static final String S3_ENDPOINT = "logfeeder.s3.endpoint";
public static final String S3_ENDPOINT_DEFAULT = "https://s3.amazonaws.com;;
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index f2eb6c741e..b6ab4c7342 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -19,7 +19,7 @@
package org.apache.ambari.logfeeder.conf;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
-import org.apache.ambari.logfeeder.conf.output.ExternalHdfsOutputConfig;
+import org.apache.ambari.logfeeder.conf.output.HdfsOutputConfig;
import org.apache.ambari.logfeeder.conf.output.RolloverConfig;
import org.apache.ambari.logfeeder.conf.output.S3OutputConfig;
import org.apache.ambari.logfeeder.plugin.common.LogFeederProperties;
@@ -53,7 +53,7 @@
private S3OutputConfig s3OutputConfig;
@Inject
- private ExternalHdfsOutputConfig hdfsOutputConfig;
+ private HdfsOutputConfig hdfsOutputConfig;
private Properties properties;
@@ -258,7 +258,7 @@
defaultValue = "false",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
- @Value("${" + LogFeederConstants.CLOUD_STORAGE_USE_HDFS_CLIENT + ":false}")
+ @Value("${" + LogFeederConstants.CLOUD_STORAGE_USE_HDFS_CLIENT + ":true}")
private boolean useCloudHdfsClient;
@LogSearchPropertyDescription(
@@ -280,15 +280,6 @@
@Value("${" + LogFeederConstants.CLOUD_STORAGE_BASE_PATH + ":}")
private String cloudBasePath;
- @LogSearchPropertyDescription(
-name = LogFeederConstants.HDFS_USER,
-description = "Overrides HADOOP_USER_NAME variable at runtime",
-examples = {"hdfs"},
-sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
- )
- @Value("${"+ LogFeederConstants.HDFS_USER + ":}")
- private String logfeederHdfsUser;
-
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_STORAGE_USE_FILTERS,
description = "Use filters for inputs (with filters the output format will
be JSON)",
@@ -460,7 +451,7 @@ public void setCloudStorageMode(LogFeederMode
cloudStorageMode) {
this.cloudStorageMode = cloudStorageMode;
}
- public ExternalHdfsOutputConfig getHdfsOutputConfig() {
+ public HdfsOutputConfig getHdfsOutputConfig() {
return hdfsOutputConfig;
}
@@ -480,7 +471,7 @@ public void setRolloverConfig(RolloverConfig
rolloverConfig) {
this.rolloverConfig = rolloverConfig;
}
- public void setHdfsOutputConfig(ExternalHdfsOutputConfig hdfsOutputConfig) {
+ public void setHdfsOutputConfig(HdfsOutputConfig hdfsOutputConfig) {
this.hdfsOutputConfig = hdfsOutputConfig;
}
@@ -512,14 +503,6 @@ public boolean isUseCloudHdfsClient() {
return useCloudHdfsClient;
}
- public String getLogfeederHdfsUser() {
-return logfeederHdfsUser;
- }
-
- public void setLogfeederHdfsUser(String logfeederHdfsUser) {
-this.logfeederHdfsUser = logfeederHdfsUser;
- }
-
public void setUseCloudHdfsClient(boolean useCloudHdfsClient) {
this.useCloudHdfsClient = useCloudHdfsClient;
}
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/ExternalHdfsOutputConfig.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/HdfsOutputConfig.java
similarity index 59%
rename from
ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/ExternalHdfsOutputConfig.java
rename to
[GitHub] asfgit commented on issue #27: AMBARI-24833. HDFS client kerberos support + small fixes
asfgit commented on issue #27: AMBARI-24833. HDFS client kerberos support + small fixes URL: https://github.com/apache/ambari-logsearch/pull/27#issuecomment-43945 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/Ambari-LogSearch-Github-PR-Builder/58/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] kasakrisz commented on a change in pull request #27: AMBARI-24833. HDFS client kerberos support + small fixes
kasakrisz commented on a change in pull request #27: AMBARI-24833. HDFS client
kerberos support + small fixes
URL: https://github.com/apache/ambari-logsearch/pull/27#discussion_r234610512
##
File path:
ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/HdfsOutputConfig.java
##
@@ -62,7 +71,27 @@
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
@Value("${"+ LogFeederConstants.HDFS_KERBEROS + ":false}")
- private boolean secure;
+ private boolean hdfsKerberos;
+
+ @LogSearchPropertyDescription(
+name = LogFeederConstants.HDFS_KERBEROS_KEYTAB,
+description = "Kerberos keytab location for Log Feeder for communicating
with secure HDFS. ",
+examples = {"/etc/security/keytabs/mykeytab.keytab"},
+defaultValue = "/etc/security/keytabs/logfeeder.service.keytab",
+sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.HDFS_KERBEROS_KEYTAB +
":/etc/security/keytabs/logfeeder.service.keytab}")
+ private String keytab;
+
+ @LogSearchPropertyDescription(
+name = LogFeederConstants.HDFS_KERBEROS_PRINCIPAL,
+description = "Kerberos keytab location for Log Feeder for communicating
with secure HDFS. ",
Review comment:
This is the same as in line 78
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] kasakrisz closed pull request #16: AMBARI-24912 - Infra Manager: scheduled job fails with dateparse exception
kasakrisz closed pull request #16: AMBARI-24912 - Infra Manager: scheduled job
fails with dateparse exception
URL: https://github.com/apache/ambari-infra/pull/16
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
index f0563f95..40fa19b4 100644
---
a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
+++
b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
@@ -25,6 +25,7 @@
import java.time.Duration;
import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
import java.util.HashMap;
import java.util.Map;
@@ -33,7 +34,7 @@
public class SolrQueryBuilder {
public static String computeEnd(String end, Duration ttl) {
-return computeEnd(end, OffsetDateTime.now(), ttl);
+return computeEnd(end, OffsetDateTime.now(ZoneOffset.UTC), ttl);
}
public static String computeEnd(String end, OffsetDateTime now, Duration
ttl) {
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[ambari-infra] branch master updated: AMBARI-24912 - Infra Manager: scheduled job fails with dateparse exception (#16)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-infra.git
The following commit(s) were added to refs/heads/master by this push:
new c818680 AMBARI-24912 - Infra Manager: scheduled job fails with
dateparse exception (#16)
c818680 is described below
commit c818680578cb686460d0a6fd398bb4b990cd47d8
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Mon Nov 19 13:34:01 2018 +0100
AMBARI-24912 - Infra Manager: scheduled job fails with dateparse exception
(#16)
---
.../java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
index f0563f9..40fa19b 100644
---
a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
+++
b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/SolrQueryBuilder.java
@@ -25,6 +25,7 @@ import static
org.apache.solr.client.solrj.SolrQuery.ORDER.asc;
import java.time.Duration;
import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
import java.util.HashMap;
import java.util.Map;
@@ -33,7 +34,7 @@ import org.apache.solr.client.solrj.SolrQuery;
public class SolrQueryBuilder {
public static String computeEnd(String end, Duration ttl) {
-return computeEnd(end, OffsetDateTime.now(), ttl);
+return computeEnd(end, OffsetDateTime.now(ZoneOffset.UTC), ttl);
}
public static String computeEnd(String end, OffsetDateTime now, Duration
ttl) {
[ambari] 01/02: AMBARI-24920. LocalFS (file:///) directory creation fails (aonishuk)
This is an automated email from the ASF dual-hosted git repository.
aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 3da344ab8e7d878868998ee514dd0dbc99a2be55
Author: Andrew Onishuk
AuthorDate: Mon Nov 19 12:16:10 2018 +0200
AMBARI-24920. LocalFS (file:///) directory creation fails (aonishuk)
---
.../libraries/providers/hdfs_resource.py | 49 ++
.../libraries/providers/hdfs_resource.py.rej | 11 +
2 files changed, 43 insertions(+), 17 deletions(-)
diff --git
a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 52b501d..33aa96a 100644
---
a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++
b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -113,8 +113,14 @@ class HdfsResourceJar:
def action_delayed_for_nameservice(self, nameservice, action_name,
main_resource):
resource = {}
env = Environment.get_instance()
-if not 'hdfs_files' in env.config:
- env.config['hdfs_files'] = []
+env_dict_key = 'hdfs_files_sudo' if main_resource.create_as_root else
'hdfs_files'
+
+if main_resource.create_as_root:
+ Logger.info("Will create {0} as root
user".format(main_resource.resource.target))
+
+
+if not env_dict_key in env.config:
+ env.config[env_dict_key] = []
# Put values in dictionary-resource
for field_name, json_field_name in RESOURCE_TO_JSON_FIELDS.iteritems():
@@ -130,22 +136,25 @@ class HdfsResourceJar:
resource['nameservice'] = nameservice
# Add resource to create
-env.config['hdfs_files'].append(resource)
+env.config[env_dict_key].append(resource)
- def action_execute(self, main_resource):
+ def action_execute(self, main_resource, sudo=False):
env = Environment.get_instance()
+env_dict_key = 'hdfs_files_sudo' if sudo else 'hdfs_files'
+if not env_dict_key in env.config or not env.config[env_dict_key]:
+ return
+
# Check required parameters
-if main_resource.has_core_configs:
+if not sudo:
main_resource.assert_parameter_is_set('user')
+ user = main_resource.resource.user
+else:
+ user = None
+
-if not 'hdfs_files' in env.config or not env.config['hdfs_files']:
- Logger.info("No resources to create. 'create_on_execute' or
'delete_on_execute' or 'download_on_execute' wasn't triggered before this
'execute' action.")
- return
-
hadoop_bin_dir = main_resource.resource.hadoop_bin_dir
hadoop_conf_dir = main_resource.resource.hadoop_conf_dir
-user = main_resource.resource.user if main_resource.has_core_configs else
None
security_enabled = main_resource.resource.security_enabled
keytab_file = main_resource.resource.keytab
kinit_path = main_resource.resource.kinit_path_local
@@ -161,18 +170,19 @@ class HdfsResourceJar:
# Write json file to disk
File(json_path,
owner = user,
- content = json.dumps(env.config['hdfs_files'])
+ content = json.dumps(env.config[env_dict_key])
)
# Execute jar to create/delete resources in hadoop
-Execute(format("hadoop --config {hadoop_conf_dir} jar {jar_path}
{json_path}"),
+Execute(('hadoop', '--config', hadoop_conf_dir, 'jar', jar_path,
json_path),
user=user,
path=[hadoop_bin_dir],
logoutput=logoutput,
+sudo=sudo,
)
# Clean
-env.config['hdfs_files'] = []
+env.config[env_dict_key] = []
class WebHDFSCallException(Fail):
@@ -618,7 +628,8 @@ class HdfsResourceProvider(Provider):
self.has_core_configs = not is_empty(getattr(resource, 'default_fs'))
self.ignored_resources_list =
HdfsResourceProvider.get_ignored_resources_list(self.resource.hdfs_resource_ignore_file)
-
+self.create_as_root = False
+
if not self.has_core_configs:
self.webhdfs_enabled = False
self.fsType = None
@@ -670,10 +681,12 @@ class HdfsResourceProvider(Provider):
def action_delayed(self, action_name):
self.assert_parameter_is_set('type')
-
+
if self.has_core_configs:
path_protocol = urlparse(self.resource.target).scheme.lower()
default_fs_protocol = urlparse(self.resource.default_fs).scheme.lower()
+
+ self.create_as_root = path_protocol == 'file' or default_fs_protocol ==
'file' and path_protocol == None
# for protocols which are different that defaultFs webhdfs will not be
able to create directories
# so for them fast-hdfs-resource.jar should be used
@@ -682,6 +695,7 @@ class HdfsResourceProvider(Provider):
Logger.info("Cannot use webhdfs for {0} defaultFs = {1} has different
protocol".format(self.resource.target, self.resource.default_fs))
else:
[ambari] branch trunk updated: AMBARI-24904. JAR does not exist: /var/lib/ambari-agent/lib/fast-hdfs-resource.jar (aonishuk)
This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/trunk by this push: new 9e6965d AMBARI-24904. JAR does not exist: /var/lib/ambari-agent/lib/fast-hdfs-resource.jar (aonishuk) 9e6965d is described below commit 9e6965d0bff7d31fc188d9b5be42aac80f94e286 Author: Andrew Onishuk AuthorDate: Mon Nov 19 12:37:27 2018 +0200 AMBARI-24904. JAR does not exist: /var/lib/ambari-agent/lib/fast-hdfs-resource.jar (aonishuk) --- .../python/resource_management/libraries/providers/hdfs_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py index 33aa96a..887b4fb 100644 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py +++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py @@ -216,7 +216,7 @@ class WebHDFSUtil: @staticmethod def is_webhdfs_available(is_webhdfs_enabled, dfs_type): # only hdfs seems to support webHDFS -return (is_webhdfs_enabled and dfs_type == 'hdfs') +return (is_webhdfs_enabled and dfs_type.lower() == 'hdfs') def run_command(self, *args, **kwargs): """
[ambari] branch trunk updated (72213a6 -> 5bc2e57)
This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git. from 72213a6 AMBARI-24905 Service display name on left navigation bar should be suffixed with "Client" if only client service component is present for a service new 3da344a AMBARI-24920. LocalFS (file:///) directory creation fails (aonishuk) new 5bc2e57 Delete hdfs_resource.py.rej The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../libraries/providers/hdfs_resource.py | 49 ++ 1 file changed, 32 insertions(+), 17 deletions(-)
[ambari] 02/02: Delete hdfs_resource.py.rej
This is an automated email from the ASF dual-hosted git repository. aonishuk pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git commit 5bc2e578942ef3af46c7be86d11803a58fbb126c Author: aonishuk AuthorDate: Mon Nov 19 12:34:24 2018 +0200 Delete hdfs_resource.py.rej --- .../libraries/providers/hdfs_resource.py.rej | 11 --- 1 file changed, 11 deletions(-) diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej deleted file mode 100644 index 0220898..000 --- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py.rej +++ /dev/null @@ -1,11 +0,0 @@ ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py -+++ ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py -@@ -686,7 +686,7 @@ class HdfsResourceProvider(Provider): - path_protocol = urlparse(self.resource.target).scheme.lower() - default_fs_protocol = urlparse(self.resource.default_fs).scheme.lower() - -- self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and path_protocol == None -+ self.create_as_root = path_protocol == 'file' or default_fs_protocol == 'file' and not path_protocol - - # for protocols which are different that defaultFs webhdfs will not be able to create directories - # so for them fast-hdfs-resource.jar should be used
[GitHub] asfgit commented on issue #27: AMBARI-24833. HDFS client kerberos support + small fixes
asfgit commented on issue #27: AMBARI-24833. HDFS client kerberos support + small fixes URL: https://github.com/apache/ambari-logsearch/pull/27#issuecomment-439831367 Refer to this link for build results (access rights to CI server needed): https://builds.apache.org/job/Ambari-LogSearch-Github-PR-Builder/57/ Test PASSed. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[ambari] branch trunk updated: AMBARI-24905 Service display name on left navigation bar should be suffixed with "Client" if only client service component is present for a service
This is an automated email from the ASF dual-hosted git repository.
atkach pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 72213a6 AMBARI-24905 Service display name on left navigation bar
should be suffixed with "Client" if only client service component is present
for a service
72213a6 is described below
commit 72213a6452550921485e26f2e64230ad26cf8a16
Author: Andrii Tkach
AuthorDate: Fri Nov 16 15:27:27 2018 +0200
AMBARI-24905 Service display name on left navigation bar should be suffixed
with "Client" if only client service component is present for a service
---
ambari-web/app/models/service.js | 22 +-
.../app/templates/main/service/menu_item.hbs | 2 +-
2 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/ambari-web/app/models/service.js b/ambari-web/app/models/service.js
index 47eb606..a0158c8 100644
--- a/ambari-web/app/models/service.js
+++ b/ambari-web/app/models/service.js
@@ -22,7 +22,14 @@ require('utils/config');
App.Service = DS.Model.extend({
serviceName: DS.attr('string', {defaultValue: ''}),
- displayName: Em.computed.formatRole('serviceName', true),
+ displayName: function() {
+const displayName = App.format.role(this.get('serviceName'), true);
+if (this.get('hasMasterOrSlaveComponent') ||
displayName.endsWith('Client')) {
+ return displayName;
+} else {
+ return displayName + ' Client';
+}
+ }.property('serviceName'),
passiveState: DS.attr('string', {defaultValue: "OFF"}),
workStatus: DS.attr('string'),
rand: DS.attr('string'),
@@ -69,10 +76,15 @@ App.Service = DS.Model.extend({
* @type {bool}
*/
hasMasterOrSlaveComponent: function() {
-return this.get('slaveComponents').toArray()
-.concat(this.get('masterComponents').toArray())
-.mapProperty('totalCount')
-.reduce((a, b) => a + b, 0) > 0;
+if (App.router.get('clusterController.isHostComponentMetricsLoaded')) {
+ return this.get('slaveComponents').toArray()
+ .concat(this.get('masterComponents').toArray())
+ .mapProperty('totalCount')
+ .reduce((a, b) => a + b, 0) > 0;
+} else {
+ //Assume that service has master or/and slave components until data
loaded
+ return true;
+}
}.property('slaveComponents.@each.totalCount',
'masterComponents.@each.totalCount'),
serviceComponents: function() {
diff --git a/ambari-web/app/templates/main/service/menu_item.hbs
b/ambari-web/app/templates/main/service/menu_item.hbs
index 9ac8d76..430d012 100644
--- a/ambari-web/app/templates/main/service/menu_item.hbs
+++ b/ambari-web/app/templates/main/service/menu_item.hbs
@@ -33,6 +33,6 @@
{{/unless}}
- {{unbound view.displayName}}
+ {{view.content.displayName}}
[GitHub] oleewere opened a new pull request #27: AMBARI-24833. HDFS client kerberos support + small fixes
oleewere opened a new pull request #27: AMBARI-24833. HDFS client kerberos support + small fixes URL: https://github.com/apache/ambari-logsearch/pull/27 # What changes were proposed in this pull request? - add kerberos support for hdfs client - remove external hdfs client -> merge it with hdfs client - small fixes ## How was this patch tested? FT: manually - docker/vagrant/gce env This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[ambari-logsearch] branch master updated: AMBARI-24833. Support for cloud logs to using filters + JSON output (#26)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new 72b3dd6 AMBARI-24833. Support for cloud logs to using filters + JSON
output (#26)
72b3dd6 is described below
commit 72b3dd6d39d233db5b02536b3c6a3215971f6cd6
Author: Olivér Szabó
AuthorDate: Mon Nov 19 10:32:15 2018 +0100
AMBARI-24833. Support for cloud logs to using filters + JSON output (#26)
* AMBARI-24833. Support for cloud logs to using filters + JSON output
* AMBARI-24833. Do not filter anything if filters are not enabled
* AMBARI-24833. Fix intermittent issues.
* AMBARI-24833. Edit comment
---
.../local/LogSearchConfigLogFeederLocal.java | 42 ++--
.../config/zookeeper/LogLevelFilterManagerZK.java | 1 +
.../logfeeder/common/LogFeederConstants.java | 1 +
.../ambari/logfeeder/conf/LogFeederProps.java | 18
...andler.java => AbstractInputConfigHandler.java} | 84 ++--
.../impl/CloudStorageInputConfigHandler.java | 14 ++-
.../operations/impl/DefaultInputConfigHandler.java | 62 +---
.../logfeeder/output/OutputLineEnricher.java | 109 +
.../ambari/logfeeder/output/OutputManagerImpl.java | 76 ++
.../output/cloud/CloudStorageLoggerFactory.java| 14 ++-
.../output/cloud/CloudStorageOutputManager.java| 27 -
.../output/cloud/CloudStorageUploader.java | 2 +-
.../src/main/resources/logfeeder.properties| 1 +
13 files changed, 232 insertions(+), 219 deletions(-)
diff --git
a/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
b/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
index f6cb519..12af637 100644
---
a/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
+++
b/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
@@ -84,12 +84,7 @@ public class LogSearchConfigLogFeederLocal extends
LogSearchConfigLocal implemen
File[] inputConfigFiles = new
File(configDir).listFiles(inputConfigFileFilter);
if (inputConfigFiles != null) {
for (File inputConfigFile : inputConfigFiles) {
-String inputConfig = new
String(Files.readAllBytes(inputConfigFile.toPath()));
-Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
-m.find();
-String serviceName = m.group(1);
-JsonElement inputConfigJson =
JsonHelper.mergeGlobalConfigWithInputConfig(parser, inputConfig,
globalConfigNode);
-inputConfigMonitor.loadInputConfigs(serviceName,
InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+tryLoadingInputConfig(inputConfigMonitor, parser, globalConfigNode,
inputConfigFile);
}
}
final FileSystem fs = FileSystems.getDefault();
@@ -100,6 +95,41 @@ public class LogSearchConfigLogFeederLocal extends
LogSearchConfigLocal implemen
executorService.submit(updater);
}
+ private void tryLoadingInputConfig(InputConfigMonitor inputConfigMonitor,
JsonParser parser, JsonArray globalConfigNode, File inputConfigFile) throws
Exception {
+// note: that will try to solve a intermittent issue when the input config
json is a null string (during file generation), that process will re-try to
process the files a few times
+int tries = 0;
+while(true) {
+ tries++;
+ Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
+ m.find();
+ String inputConfig = new
String(Files.readAllBytes(inputConfigFile.toPath()));
+ String serviceName = m.group(1);
+ JsonElement inputConfigJson = null;
+ logger.info("Trying to load '{}' service input config from input file
'{}'", serviceName, inputConfigFile.getAbsolutePath());
+ try {
+inputConfigJson = JsonHelper.mergeGlobalConfigWithInputConfig(parser,
inputConfig, globalConfigNode);
+ } catch (Exception e) {
+final String errorMessage;
+if (tries < 3) {
+ errorMessage = String.format("Cannot parse input config: %s, will
retry in a few seconds again (tries: %s)", inputConfig, String.valueOf(tries));
+ logger.error(errorMessage, e);
+ try {
+Thread.sleep(2000);
+ } catch (Exception ex) {
+// skip
+ }
+ continue;
+} else {
+ errorMessage = String.format("Cannot parse input config: %s, after
%s tries. Will skip to processing it", inputConfig, String.valueOf(tries));
+ logger.error(errorMessage, e);
+ break;
+}
+ }
+
[GitHub] oleewere closed pull request #26: AMBARI-24833. Support for cloud logs to using filters + JSON output
oleewere closed pull request #26: AMBARI-24833. Support for cloud logs to using
filters + JSON output
URL: https://github.com/apache/ambari-logsearch/pull/26
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
b/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
index f6cb519b95..12af637d8c 100644
---
a/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
+++
b/ambari-logsearch-config-local/src/main/java/org/apache/ambari/logsearch/config/local/LogSearchConfigLogFeederLocal.java
@@ -84,12 +84,7 @@ public void monitorInputConfigChanges(final
InputConfigMonitor inputConfigMonito
File[] inputConfigFiles = new
File(configDir).listFiles(inputConfigFileFilter);
if (inputConfigFiles != null) {
for (File inputConfigFile : inputConfigFiles) {
-String inputConfig = new
String(Files.readAllBytes(inputConfigFile.toPath()));
-Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
-m.find();
-String serviceName = m.group(1);
-JsonElement inputConfigJson =
JsonHelper.mergeGlobalConfigWithInputConfig(parser, inputConfig,
globalConfigNode);
-inputConfigMonitor.loadInputConfigs(serviceName,
InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+tryLoadingInputConfig(inputConfigMonitor, parser, globalConfigNode,
inputConfigFile);
}
}
final FileSystem fs = FileSystems.getDefault();
@@ -100,6 +95,41 @@ public void monitorInputConfigChanges(final
InputConfigMonitor inputConfigMonito
executorService.submit(updater);
}
+ private void tryLoadingInputConfig(InputConfigMonitor inputConfigMonitor,
JsonParser parser, JsonArray globalConfigNode, File inputConfigFile) throws
Exception {
+// note: that will try to solve a intermittent issue when the input config
json is a null string (during file generation), that process will re-try to
process the files a few times
+int tries = 0;
+while(true) {
+ tries++;
+ Matcher m = serviceNamePattern.matcher(inputConfigFile.getName());
+ m.find();
+ String inputConfig = new
String(Files.readAllBytes(inputConfigFile.toPath()));
+ String serviceName = m.group(1);
+ JsonElement inputConfigJson = null;
+ logger.info("Trying to load '{}' service input config from input file
'{}'", serviceName, inputConfigFile.getAbsolutePath());
+ try {
+inputConfigJson = JsonHelper.mergeGlobalConfigWithInputConfig(parser,
inputConfig, globalConfigNode);
+ } catch (Exception e) {
+final String errorMessage;
+if (tries < 3) {
+ errorMessage = String.format("Cannot parse input config: %s, will
retry in a few seconds again (tries: %s)", inputConfig, String.valueOf(tries));
+ logger.error(errorMessage, e);
+ try {
+Thread.sleep(2000);
+ } catch (Exception ex) {
+// skip
+ }
+ continue;
+} else {
+ errorMessage = String.format("Cannot parse input config: %s, after
%s tries. Will skip to processing it", inputConfig, String.valueOf(tries));
+ logger.error(errorMessage, e);
+ break;
+}
+ }
+ inputConfigMonitor.loadInputConfigs(serviceName,
InputConfigGson.gson.fromJson(inputConfigJson, InputConfigImpl.class));
+ break;
+}
+ }
+
@Override
public void close() throws IOException {
}
diff --git
a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java
b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java
index fd08e07fed..0975c395fa 100644
---
a/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java
+++
b/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/LogLevelFilterManagerZK.java
@@ -48,6 +48,7 @@
public LogLevelFilterManagerZK(Map properties) throws
Exception {
this.client = LogSearchConfigZKHelper.createZKClient(properties);
+this.client.start();
this.serverCache = new TreeCache(client, "/");
this.aclList = LogSearchConfigZKHelper.getAcls(properties);
this.gson = LogSearchConfigZKHelper.createGson();
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
[GitHub] oleewere commented on a change in pull request #26: AMBARI-24833. Support for cloud logs to using filters + JSON output
oleewere commented on a change in pull request #26: AMBARI-24833. Support for
cloud logs to using filters + JSON output
URL: https://github.com/apache/ambari-logsearch/pull/26#discussion_r23455
##
File path:
ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/manager/operations/impl/AbstractInputConfigHandler.java
##
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.manager.operations.impl;
+
+import org.apache.ambari.logfeeder.manager.InputConfigHolder;
+import org.apache.ambari.logfeeder.manager.operations.InputConfigHandler;
+import org.apache.ambari.logfeeder.plugin.common.AliasUtil;
+import org.apache.ambari.logfeeder.plugin.filter.Filter;
+import org.apache.ambari.logfeeder.plugin.input.Input;
+import
org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Holds common operations for input config handlers
+ */
+public abstract class AbstractInputConfigHandler implements InputConfigHandler
{
+
+ private static final Logger logger =
LogManager.getLogger(AbstractInputConfigHandler.class);
+
+ protected void loadFilters(String serviceName, InputConfigHolder
inputConfigHolder) {
+sortFilters(inputConfigHolder);
+
+List toRemoveInputList = new ArrayList<>();
+for (Input input :
inputConfigHolder.getInputManager().getInputList(serviceName)) {
+ for (FilterDescriptor filterDescriptor :
inputConfigHolder.getFilterConfigList()) {
+if (filterDescriptor == null) {
+ logger.warn("Filter descriptor is smpty. Skipping...");
Review comment:
i will solve that in a next commit
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] oleewere commented on a change in pull request #26: AMBARI-24833. Support for cloud logs to using filters + JSON output
oleewere commented on a change in pull request #26: AMBARI-24833. Support for
cloud logs to using filters + JSON output
URL: https://github.com/apache/ambari-logsearch/pull/26#discussion_r234544396
##
File path:
ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
##
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.output;
+
+import com.google.common.hash.Hashing;
+import org.apache.ambari.logfeeder.plugin.common.MetricData;
+import org.apache.ambari.logfeeder.plugin.input.Input;
+import org.apache.ambari.logfeeder.plugin.input.InputMarker;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility class for fill output with other fields
+ */
+public class OutputLineEnricher {
+
+ private static final Logger logger =
LogManager.getLogger(OutputLineEnricher.class);
+
+ private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1
+
+ public void enrichFields(final Map jsonObj, final
InputMarker inputMarker, final MetricData messageTruncateMetric) {
+Input input = inputMarker.getInput();
+// Update the block with the context fields
+for (Map.Entry entry :
input.getInputDescriptor().getAddFields().entrySet()) {
+ if (jsonObj.get(entry.getKey()) == null ||
entry.getKey().equals("cluster") && "null".equals(jsonObj.get(entry.getKey(
{
Review comment:
i will solve that in a next commit
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] kasakrisz commented on a change in pull request #26: AMBARI-24833. Support for cloud logs to using filters + JSON output
kasakrisz commented on a change in pull request #26: AMBARI-24833. Support for
cloud logs to using filters + JSON output
URL: https://github.com/apache/ambari-logsearch/pull/26#discussion_r234520580
##
File path:
ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/manager/operations/impl/AbstractInputConfigHandler.java
##
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.manager.operations.impl;
+
+import org.apache.ambari.logfeeder.manager.InputConfigHolder;
+import org.apache.ambari.logfeeder.manager.operations.InputConfigHandler;
+import org.apache.ambari.logfeeder.plugin.common.AliasUtil;
+import org.apache.ambari.logfeeder.plugin.filter.Filter;
+import org.apache.ambari.logfeeder.plugin.input.Input;
+import
org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Holds common operations for input config handlers
+ */
+public abstract class AbstractInputConfigHandler implements InputConfigHandler
{
+
+ private static final Logger logger =
LogManager.getLogger(AbstractInputConfigHandler.class);
+
+ protected void loadFilters(String serviceName, InputConfigHolder
inputConfigHolder) {
+sortFilters(inputConfigHolder);
+
+List toRemoveInputList = new ArrayList<>();
+for (Input input :
inputConfigHolder.getInputManager().getInputList(serviceName)) {
+ for (FilterDescriptor filterDescriptor :
inputConfigHolder.getFilterConfigList()) {
+if (filterDescriptor == null) {
+ logger.warn("Filter descriptor is smpty. Skipping...");
Review comment:
typo: smpty -> empty
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] kasakrisz commented on a change in pull request #26: AMBARI-24833. Support for cloud logs to using filters + JSON output
kasakrisz commented on a change in pull request #26: AMBARI-24833. Support for
cloud logs to using filters + JSON output
URL: https://github.com/apache/ambari-logsearch/pull/26#discussion_r234521392
##
File path:
ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
##
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.output;
+
+import com.google.common.hash.Hashing;
+import org.apache.ambari.logfeeder.plugin.common.MetricData;
+import org.apache.ambari.logfeeder.plugin.input.Input;
+import org.apache.ambari.logfeeder.plugin.input.InputMarker;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility class for fill output with other fields
+ */
+public class OutputLineEnricher {
+
+ private static final Logger logger =
LogManager.getLogger(OutputLineEnricher.class);
+
+ private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1
+
+ public void enrichFields(final Map jsonObj, final
InputMarker inputMarker, final MetricData messageTruncateMetric) {
+Input input = inputMarker.getInput();
+// Update the block with the context fields
+for (Map.Entry entry :
input.getInputDescriptor().getAddFields().entrySet()) {
+ if (jsonObj.get(entry.getKey()) == null ||
entry.getKey().equals("cluster") && "null".equals(jsonObj.get(entry.getKey(
{
Review comment:
```"cluster".equals(entry.getKey())```
would be safer
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
