Repository: metron Updated Branches: refs/heads/master ae1dc3a4a -> 222685760
METRON-1047 REST should use core-site.xml for Hadoop configuration (merrimanr) closes apache/metron#672 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/22268576 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/22268576 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/22268576 Branch: refs/heads/master Commit: 22268576039babc1dd9ee65a5a6a11ade49a7258 Parents: ae1dc3a Author: merrimanr <[email protected]> Authored: Mon Aug 21 15:02:57 2017 -0500 Committer: merrimanr <[email protected]> Committed: Mon Aug 21 15:02:57 2017 -0500 ---------------------------------------------------------------------- .../common-services/METRON/CURRENT/package/templates/metron.j2 | 2 +- .../metron-rest/src/main/config/rest_application.yml | 4 ---- .../main/java/org/apache/metron/rest/MetronRestConstants.java | 5 ----- .../main/java/org/apache/metron/rest/config/HadoopConfig.java | 2 -- metron-interface/metron-rest/src/main/scripts/metron-rest | 3 ++- .../java/org/apache/metron/rest/config/HadoopConfigTest.java | 6 ------ 6 files changed, 3 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 ---------------------------------------------------------------------- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 06dddd3..7233b54 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 @@ -31,7 +31,7 @@ METRON_DEFAULT_GROK_PATH="{{metron_default_grok_path}}" METRON_SPRING_OPTIONS="{{metron_spring_options}}" ZOOKEEPER="{{zookeeper_quorum}}" BROKERLIST="{{kafka_brokers}}" -HDFS_URL="{{default_fs}}" +HADOOP_CONF_DIR="/etc/hadoop/conf/" STORM_REST_URL="{{storm_rest_addr}}" SECURITY_ENABLED={{security_enabled|lower}} {% if metron_principal_name is defined %}METRON_PRINCIPAL_NAME="{{metron_principal_name}}" http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/config/rest_application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml index 2afbb8a..6d12e95 100644 --- a/metron-interface/metron-rest/src/main/config/rest_application.yml +++ b/metron-interface/metron-rest/src/main/config/rest_application.yml @@ -31,10 +31,6 @@ kafka: security: protocol: ${KAFKA_SECURITY_PROTOCOL} -hdfs: - namenode: - url: ${HDFS_URL} - grok: path: temp: ${METRON_TEMP_GROK_PATH} http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index a080f77..11310d4 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -30,8 +30,6 @@ public class MetronRestConstants { public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp"; public static final String GROK_CLASS_NAME = GrokParser.class.getName(); public static final String GROK_PATH_KEY = "grokPath"; - public static final String GROK_STATEMENT_KEY = "grokStatement"; - public static final String GROK_PATTERN_LABEL_KEY = "patternLabel"; public static final String STORM_UI_SPRING_PROPERTY = "storm.ui.url"; public static final String SUPERVISOR_SUMMARY_URL = "/api/v1/supervisor/summary"; @@ -53,9 +51,6 @@ public class MetronRestConstants { public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url"; - public static final String HDFS_URL_SPRING_PROPERTY = "hdfs.namenode.url"; - public static final String DEFAULT_HDFS_URL = "file:///"; - public static final String KERBEROS_ENABLED_SPRING_PROPERTY = "kerberos.enabled"; public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY = "kerberos.principal"; public static final String KERBEROS_KEYTAB_SPRING_PROPERTY = "kerberos.keytab"; http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java index 7b1bf2f..98c8108 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java @@ -39,9 +39,7 @@ public class HadoopConfig { @Bean public org.apache.hadoop.conf.Configuration configuration() throws IOException { org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration(); - configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)); if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) { - configuration.set("hadoop.security.authentication", "KERBEROS"); UserGroupInformation.setConfiguration(configuration); String keyTabLocation = environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY); String userPrincipal = environment.getProperty(MetronRestConstants.KERBEROS_PRINCIPLE_SPRING_PROPERTY); http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/main/scripts/metron-rest ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest b/metron-interface/metron-rest/src/main/scripts/metron-rest index 08cdfd4..0d99617 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest @@ -46,8 +46,9 @@ fi HBASE_HOME=${HBASE_HOME:-/usr/hdp/current/hbase-client} PIDFILE="$METRON_PID_DIR/$NAME.pid" +LOCKFILE=/var/lock/subsys/$NAME -METRON_REST_CLASSPATH="${HBASE_HOME}/conf:$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar" +METRON_REST_CLASSPATH="$HADOOP_CONF_DIR:${HBASE_HOME}/conf:$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar" # the vagrant Spring profile provides configuration values, otherwise configuration is provided by rest_application.yml if [[ !($METRON_SPRING_PROFILES_ACTIVE == *"vagrant"*) ]]; then http://git-wip-us.apache.org/repos/asf/metron/blob/22268576/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java index c262783..7504e3e 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/HadoopConfigTest.java @@ -56,7 +56,6 @@ public class HadoopConfigTest { @Test public void configurationShouldReturnProperKerberosConfiguration() throws IOException { - when(environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)).thenReturn("default filesystem"); when(environment.getProperty(MetronRestConstants.KERBEROS_KEYTAB_SPRING_PROPERTY)).thenReturn("metron keytabLocation"); when(environment.getProperty(MetronRestConstants.KERBEROS_PRINCIPLE_SPRING_PROPERTY)).thenReturn("metron principal"); @@ -67,14 +66,10 @@ public class HadoopConfigTest { verifyStatic(); UserGroupInformation.setConfiguration(any(Configuration.class)); UserGroupInformation.loginUserFromKeytab("metron keytabLocation", "metron principal"); - - assertEquals("default filesystem", configuration.get("fs.defaultFS")); - assertEquals("KERBEROS", configuration.get("hadoop.security.authentication")); } @Test public void configurationShouldReturnProperConfiguration() throws IOException { - when(environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)).thenReturn("default filesystem"); when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(false); Configuration configuration = hadoopConfig.configuration(); @@ -83,7 +78,6 @@ public class HadoopConfigTest { UserGroupInformation.setConfiguration(any(Configuration.class)); UserGroupInformation.loginUserFromKeytab(anyString(), anyString()); - assertEquals("default filesystem", configuration.get("fs.defaultFS")); assertEquals("simple", configuration.get("hadoop.security.authentication")); } }
