Improvements to filemgr and config-publisher bash scripts
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9ee8af48 Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9ee8af48 Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9ee8af48 Branch: refs/heads/development Commit: 9ee8af48ba45904a44d4e32b3030e8ad7d0bee21 Parents: 57a9235 Author: Imesha Sudasingha <[email protected]> Authored: Thu Aug 10 18:32:27 2017 +0530 Committer: Imesha Sudasingha <[email protected]> Committed: Thu Aug 10 18:32:27 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/oodt/config/Component.java | 4 +-- .../config/ConfigurationManagerFactory.java | 37 ++++++++++++++++---- .../java/org/apache/oodt/config/Constants.java | 3 ++ config/src/main/resources/log4j.xml | 1 - core/pom.xml | 11 +++--- filemgr/pom.xml | 6 +++- filemgr/src/main/bin/filemgr | 6 ++-- .../TestDistributedXmlRpcFileManager.java | 1 + .../distributed/config/filemgr.properties | 8 ++--- 9 files changed, 54 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/Component.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/Component.java b/config/src/main/java/org/apache/oodt/config/Component.java index e2ea223..2c7123e 100644 --- a/config/src/main/java/org/apache/oodt/config/Component.java +++ b/config/src/main/java/org/apache/oodt/config/Component.java @@ -24,8 +24,8 @@ package org.apache.oodt.config; * @author Imesha Sudasingha */ public enum Component { - FILE_MANAGER("filemgr", "OODT_FILEMGR_HOME"), - RESOURCE_MANAGER("resmgr", "OODT_RESMGR_HOME"); + FILE_MANAGER("filemgr", "FILEMGR_HOME"), + RESOURCE_MANAGER("resmgr", "RESMGR_HOME"); /** Shorthand name of the component. Will be used when creating ZNodes in zookeeper */ String name; http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java index 9eecfaa..f3a5dc5 100644 --- a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java +++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java @@ -17,6 +17,7 @@ package org.apache.oodt.config; +import org.apache.oodt.config.Constants.Env; import org.apache.oodt.config.distributed.DistributedConfigurationManager; import org.apache.oodt.config.standalone.StandaloneConfigurationManager; import org.slf4j.Logger; @@ -24,7 +25,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import static org.apache.oodt.config.Constants.Env.CONNECT_STRING; import static org.apache.oodt.config.Constants.Properties.ENABLE_DISTRIBUTED_CONFIGURATION; +import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; /** * Factory class to be used to get the {@link ConfigurationManager} instances accordingly. @@ -40,18 +43,38 @@ public class ConfigurationManagerFactory { } /** - * Returns the {@link ConfigurationManager} to be used by the calling class. Whether to use the standalone version or - * the distributed version of the configuration manager will be determined by the value of the property + * Returns the {@link ConfigurationManager} to be used by the calling class. Whether to use the standalone version + * or the distributed version of the configuration manager will be determined by the value of the property * <pre>org.apache.oodt.config.zookeeper == true</pre> * - * @param component Name of the OODT component, to which the created configuration manager instance will be providing - * configuration support - * @param propertiesFiles List of <pre>.properties</pre> files which are to be used in the case of standalone configuration - * management + * @param component Name of the OODT component, to which the created configuration manager instance will be + * providing configuration support + * @param propertiesFiles List of <pre>.properties</pre> files which are to be used in the case of standalone + * configuration management * @return ConfigurationManager instance to used by the corresponding component. */ public static ConfigurationManager getConfigurationManager(Component component, List<String> propertiesFiles) { - if (Boolean.getBoolean(ENABLE_DISTRIBUTED_CONFIGURATION)) { + boolean isDistributed = Boolean.getBoolean(ENABLE_DISTRIBUTED_CONFIGURATION); + if (!isDistributed) { + String env = System.getenv(Env.ENABLE_DISTRIBUTED_CONFIGURATION); + if (env != null) { + isDistributed = Boolean.parseBoolean(env); + } + } + + if (isDistributed) { + String connectString = System.getProperty(ZK_CONNECT_STRING); + if (connectString == null) { + connectString = System.getenv(CONNECT_STRING); + } + + if (connectString == null) { + throw new IllegalArgumentException( + String.format("%s environment variable need to be set for distributed configuration management", CONNECT_STRING)); + } + + System.setProperty(ZK_CONNECT_STRING, connectString); + logger.info("Using distributed configuration management for {}", component); return new DistributedConfigurationManager(component); } http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/java/org/apache/oodt/config/Constants.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java index 611f81e..9cd217f 100644 --- a/config/src/main/java/org/apache/oodt/config/Constants.java +++ b/config/src/main/java/org/apache/oodt/config/Constants.java @@ -43,6 +43,9 @@ public class Constants { public static class Env { /** Environment variable name to specify OODT project name */ public static final String OODT_PROJECT = "OODT_PROJECT"; + /** The environment variable to be set, if to enable distributed configuration management */ + public static final String ENABLE_DISTRIBUTED_CONFIGURATION = "OODT_DISTRIBUTED_CONF"; + public static final String CONNECT_STRING = "ZK_CONNECT_STRING"; } public static class Properties { http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/config/src/main/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/config/src/main/resources/log4j.xml b/config/src/main/resources/log4j.xml index 8cc1919..0cd8348 100644 --- a/config/src/main/resources/log4j.xml +++ b/config/src/main/resources/log4j.xml @@ -39,7 +39,6 @@ <root> <priority value="INFO"/> <appender-ref ref="console"/> - <appender-ref ref="file"/> </root> <logger name="org.apache.zookeeper"> http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index b9f07e1..5e99f9e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -253,6 +253,11 @@ the License. <version>3.3.0</version> </dependency> <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.5.1-alpha</version> + </dependency> + <dependency> <groupId>org.apache.cxf</groupId> <artifactId>cxf-rt-frontend-jaxrs</artifactId> <version>2.6.0</version> @@ -412,12 +417,6 @@ the License. <groupId>org.apache.solr</groupId> <artifactId>solr-solrj</artifactId> <version>6.2.1</version> - <exclusions> - <exclusion> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.tika</groupId> http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/pom.xml ---------------------------------------------------------------------- diff --git a/filemgr/pom.xml b/filemgr/pom.xml index 4fadcb8..2086dc1 100644 --- a/filemgr/pom.xml +++ b/filemgr/pom.xml @@ -163,6 +163,10 @@ <groupId>org.apache.solr</groupId> <artifactId>solr-lucene-core</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -236,7 +240,7 @@ </property> </systemProperties> <environmentVariables> - <OODT_FILEMGR_HOME>${project.basedir}</OODT_FILEMGR_HOME> + <FILEMGR_HOME>${project.basedir}</FILEMGR_HOME> <OODT_PROJECT>primary</OODT_PROJECT> </environmentVariables> <forkedProcessTimeoutInSeconds>0</forkedProcessTimeoutInSeconds> http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/main/bin/filemgr ---------------------------------------------------------------------- diff --git a/filemgr/src/main/bin/filemgr b/filemgr/src/main/bin/filemgr index 1a4af47..02f5b90 100644 --- a/filemgr/src/main/bin/filemgr +++ b/filemgr/src/main/bin/filemgr @@ -34,7 +34,7 @@ else fi export JAVA_HOME -FILEMGR_HOME=.. +FILEMGR_HOME=`cd ..; pwd` export FILEMGR_HOME RUN_HOME=${FILEMGR_HOME}/../run export RUN_HOME @@ -52,6 +52,8 @@ for file in `find ../lib/*.jar`; do LIB_DEPS="${file}:${LIB_DEPS}" done +LIB_DEPS="${FILEMGR_HOME}/etc/log4j.xml:${LIB_DEPS}" + if [ ! -z $OODT_DISTRIBUTED_CONF ] then if [ ! -z $ZK_CONNECT_STRING ] @@ -73,7 +75,7 @@ case "$1" in $JAVA_HOME/bin/java \ -cp ${LIB_DEPS} \ ${DISTRIBUTED_CONF_PROPERTIES} \ - -Dlog4j.configuration=./${FILEMGR_HOME}/etc/log4j.xml \ + -Dlog4j.configuration=log4j.xml \ -Djava.util.logging.config.file=${FILEMGR_HOME}/etc/logging.properties \ -Dorg.apache.oodt.cas.filemgr.properties=${CAS_FILEMGR_PROPS} \ org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \ http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java ---------------------------------------------------------------------- diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java index afbac03..3dd158b 100644 --- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java +++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/config/distributed/TestDistributedXmlRpcFileManager.java @@ -44,6 +44,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +// TODO: 8/10/17 Cleanup Dirs once done public class TestDistributedXmlRpcFileManager extends AbstractDistributedConfigurationTest { private static final int FM_PORT = 9001; http://git-wip-us.apache.org/repos/asf/oodt/blob/9ee8af48/filemgr/src/test/resources/distributed/config/filemgr.properties ---------------------------------------------------------------------- diff --git a/filemgr/src/test/resources/distributed/config/filemgr.properties b/filemgr/src/test/resources/distributed/config/filemgr.properties index 50d09a7..2ca5096 100644 --- a/filemgr/src/test/resources/distributed/config/filemgr.properties +++ b/filemgr/src/test/resources/distributed/config/filemgr.properties @@ -63,7 +63,7 @@ org.apache.oodt.cas.filemgr.catalog.science.jdbc.pass= org.apache.oodt.cas.filemgr.catalog.science.jdbc.driver=org.hsqldb.jdbc.JDBCDriver # lucene catalog configuration -org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=file:target/tmp +org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=target/tmp org.apache.oodt.cas.filemgr.catalog.lucene.pageSize=20 org.apache.oodt.cas.filemgr.catalog.lucene.commitLockTimeout.seconds=60 org.apache.oodt.cas.filemgr.catalog.lucene.writeLockTimeout.seconds=60 @@ -90,10 +90,10 @@ org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.pass= org.apache.oodt.cas.filemgr.repositorymgr.science.jdbc.driver=org.hsqldb.jdbc.JDBCDriver # XML repository manager configuration -org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[OODT_FILEMGR_HOME]/target/filemgr/policy/oodt +org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[FILEMGR_HOME]/target/filemgr/policy/oodt # XML validation layer configuration -org.apache.oodt.cas.filemgr.validation.dirs=file:[OODT_FILEMGR_HOME]/target/filemgr/policy/oodt +org.apache.oodt.cas.filemgr.validation.dirs=file:[FILEMGR_HOME]/target/filemgr/policy/oodt # set the following property to 'true' to allow dynamic metadata fields, # effectively bypassing the validation layer. @@ -125,7 +125,7 @@ org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key # location of Mime-Type repository -org.apache.oodt.cas.filemgr.mime.type.repository=[OODT_FILEMGR_HOME]/target/filemgr/policy/mime-types.xml +org.apache.oodt.cas.filemgr.mime.type.repository=[FILEMGR_HOME]/target/filemgr/policy/mime-types.xml # tells the file manager system layer to include product instance metadata # NOTE: here are the expected field mappings
