[ambari-logsearch] branch master updated: AMBARI-24833. Let logfeeder.properties override core-site configs. (#30)

[oleewere]

This is an automated email from the ASF dual-hosted git repository.

oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git


The following commit(s) were added to refs/heads/master by this push:
     new defc854  AMBARI-24833. Let logfeeder.properties override core-site 
configs. (#30)
defc854 is described below

commit defc854b8d1948745a07a1a6e07322670ffef340
Author: Olivér Szabó <oleew...@gmail.com>
AuthorDate: Mon Nov 19 21:49:56 2018 +0100

    AMBARI-24833. Let logfeeder.properties override core-site configs. (#30)
---
 .../output/cloud/upload/HDFSS3UploadClient.java       |  1 +
 .../output/cloud/upload/HDFSUploadClient.java         |  1 +
 .../ambari/logfeeder/util/LogFeederHDFSUtil.java      | 19 +++++++++++++++++++
 .../src/main/resources/logfeeder.properties           |  2 ++
 4 files changed, 23 insertions(+)

diff --git 
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
 
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
index 5405a10..df1b8a5 100644
--- 
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
+++ 
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSS3UploadClient.java
@@ -57,6 +57,7 @@ public class HDFSS3UploadClient extends AbstractS3CloudClient 
implements UploadC
     conf.set("fs.s3a.endpoint", s3OutputConfig.getEndpoint());
     conf.set("fs.s3a.path.style.access", 
String.valueOf(s3OutputConfig.isPathStyleAccess()));
     conf.set("fs.s3a.multiobjectdelete.enable", 
String.valueOf(s3OutputConfig.isMultiobjectDeleteEnable()));
+    LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, conf);
     this.fs = LogFeederHDFSUtil.buildFileSystem(conf);
   }
 
diff --git 
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
 
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
index 421c4c5..7e1b471 100644
--- 
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
+++ 
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
@@ -84,6 +84,7 @@ public class HDFSUploadClient implements UploadClient {
       }
     }
     logger.info("HDFS client - will use '{}' permission for uploaded files", 
hdfsOutputConfig.getHdfsFilePermissions());
+    LogFeederHDFSUtil.overrideFileSystemConfigs(logFeederProps, configuration);
     this.fs = LogFeederHDFSUtil.buildFileSystem(configuration);
   }
 
diff --git 
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
 
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
index 3549e04..b741229 100644
--- 
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
+++ 
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederHDFSUtil.java
@@ -19,7 +19,10 @@
 package org.apache.ambari.logfeeder.util;
 
 import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
 
+import org.apache.ambari.logfeeder.conf.LogFeederProps;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -95,4 +98,20 @@ public class LogFeederHDFSUtil {
       }
     }
   }
+
+  /**
+   * Override Hadoop configuration object based on logfeeder.properties 
configurations (with keys that starts with "fs." or "hadoop.*")
+   * @param logFeederProps global property holder
+   * @param configuration hadoop configuration holder
+   */
+  public static void overrideFileSystemConfigs(LogFeederProps logFeederProps, 
Configuration configuration) {
+    Properties properties = logFeederProps.getProperties();
+    for (Map.Entry<Object, Object> prop : properties.entrySet()) {
+      String propertyName = prop.getKey().toString();
+      if (propertyName.startsWith("fs.")) {
+        logger.info("Override {} configuration (by logfeeder.properties)", 
propertyName);
+        configuration.set(propertyName, prop.getValue().toString());
+      }
+    }
+  }
 }
\ No newline at end of file
diff --git a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties 
b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 45c05f3..2537913 100644
--- a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -68,3 +68,5 @@ logfeeder.s3.access.key=MyAccessKey
 logfeeder.s3.object.acl=public-read
 logfeeder.s3.path.style.access=true
 logfeeder.s3.multiobject.delete.enable=false
+
+fs.s3a.acl.default=PublicReadWrite