Altered configuration publishing mechanism to use spring configuration
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/fadf149c Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/fadf149c Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/fadf149c Branch: refs/heads/master Commit: fadf149cb77dd943af9e81c0f127ce74f203c478 Parents: 4b88f71 Author: Imesha Sudasingha <[email protected]> Authored: Sat Jun 17 21:09:58 2017 +0530 Committer: Imesha Sudasingha <[email protected]> Committed: Sat Jun 17 21:09:58 2017 +0530 ---------------------------------------------------------------------- config/pom.xml | 15 +++ .../oodt/config/ConfigurationManager.java | 19 +-- .../config/ConfigurationManagerFactory.java | 6 +- .../java/org/apache/oodt/config/Constants.java | 3 + .../DistributedConfigurationManager.java | 68 +--------- .../DistributedConfigurationPublisher.java | 135 ++++++++++--------- .../oodt/config/distributed/ZNodePaths.java | 80 +++++++++++ .../apache/oodt/config/distributed/ZPaths.java | 67 --------- .../StandaloneConfigurationManager.java | 27 +--- .../apache/oodt/config/utils/CuratorUtils.java | 46 +++++++ .../DistributedConfigurationPublisherTest.java | 55 +++++--- .../resources/distributed-config-publisher.xml | 56 ++++++++ .../cas/filemgr/system/XmlRpcFileManager.java | 13 +- 13 files changed, 329 insertions(+), 261 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/pom.xml ---------------------------------------------------------------------- diff --git a/config/pom.xml b/config/pom.xml index 0ab4457..891f520 100644 --- a/config/pom.xml +++ b/config/pom.xml @@ -28,6 +28,21 @@ </dependency> <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-beans</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-core</artifactId> + </dependency> + + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java index e5dc82f..1911889 100644 --- a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java +++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java @@ -17,11 +17,7 @@ package org.apache.oodt.config; -import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; /** * The abstract class to define functionalities of the configuration managers. @@ -31,23 +27,10 @@ import java.util.List; public abstract class ConfigurationManager { protected String component; - protected List<String> propertiesFiles; - public ConfigurationManager(String component, List<String> propertiesFiles) { + public ConfigurationManager(String component) { this.component = component; - this.propertiesFiles = propertiesFiles != null ? propertiesFiles : new ArrayList<String>(); } - /** - * Retrieves a given property from the underlying configuration storage. For example, If we want to get the - * value of the property org.foo.bar, we have to call this method with <pre>org.foo.bar</pre> as the parameter. - * - * @param key Name of the property to be retrieved. - * @return Value of the requested property | null - */ - public abstract String getProperty(String key); - public abstract void loadProperties() throws IOException; - - public abstract File getPropertiesFile(String filePath) throws FileNotFoundException; } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java index 0fd995d..4d01e35 100644 --- a/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java +++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManagerFactory.java @@ -43,11 +43,15 @@ public class ConfigurationManagerFactory { * the distributed version of the configuration manager will be determined by the value of the property * <pre>org.apache.oodt.config.zookeeper == true</pre> * + * @param component Name of the OODT component, to which the created configuration manager instance will be providing + * configuration support + * @param propertiesFiles List of <pre>.properties</pre> files which are to be used in the case of standalone configuration + * management * @return ConfigurationManager instance to used by the corresponding component. */ public static ConfigurationManager getConfigurationManager(String component, List<String> propertiesFiles) { if (System.getProperty(ENABLE_DISTRIBUTED_CONFIGURATION) != null) { - return new DistributedConfigurationManager(component, propertiesFiles); + return new DistributedConfigurationManager(component); } return new StandaloneConfigurationManager(component, propertiesFiles); } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/Constants.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java index 6f74f97..033101b 100644 --- a/config/src/main/java/org/apache/oodt/config/Constants.java +++ b/config/src/main/java/org/apache/oodt/config/Constants.java @@ -70,5 +70,8 @@ public class Constants { /** Where properties files are stored inside each component */ public static final String PROPERTIES_PATH_NAME = "properties"; + + /** Where other configuration files will be stored */ + public static final String CONFIGURATION_PATH_NAME = "configuration"; } } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java index f6502bb..9ab0e32 100644 --- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java +++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java @@ -18,21 +18,14 @@ package org.apache.oodt.config.distributed; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.api.ACLProvider; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.oodt.config.ConfigurationManager; import org.apache.oodt.config.Constants; import org.apache.oodt.config.Constants.Properties; import org.apache.oodt.config.utils.CuratorUtils; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; -import java.util.List; import java.util.concurrent.TimeUnit; import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; @@ -52,8 +45,8 @@ public class DistributedConfigurationManager extends ConfigurationManager { private String connectString; private CuratorFramework client; - public DistributedConfigurationManager(String component, List<String> propertiesFiles) { - super(component, propertiesFiles); + public DistributedConfigurationManager(String component) { + super(component); if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) { throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set"); @@ -82,45 +75,12 @@ public class DistributedConfigurationManager extends ConfigurationManager { * {@link Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble. */ private void startZookeeper() { - int connectionTimeoutMs = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "15")); - int sessionTimeoutMs = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "60")); - int retryInitialWaitMs = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "1000")); - int maxRetryCount = Integer.parseInt(System.getProperty(Properties.ZK_CONNECTION_TIMEOUT, "3")); - int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000")); - - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() - .connectString(connectString) - .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount)) - .connectionTimeoutMs(connectionTimeoutMs) - .sessionTimeoutMs(sessionTimeoutMs); - - /* - * If authorization information is available, those will be added to the client. NOTE: These auth info are - * for access control, therefore no authentication will happen when the client is being started. These - * info will only be required whenever a client is accessing an already create ZNode. For another client of - * another node to make use of a ZNode created by this node, it should also provide the same auth info. - */ - if (System.getProperty(Properties.ZK_USERNAME) != null && System.getProperty(Properties.ZK_PASSWORD) != null) { - String authenticationString = System.getProperty(Properties.ZK_USERNAME) + ":" + System.getProperty(Properties.ZK_PASSWORD); - builder.authorization("digest", authenticationString.getBytes()) - .aclProvider(new ACLProvider() { - public List<ACL> getDefaultAcl() { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - - public List<ACL> getAclForPath(String path) { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - }); - } - - client = builder.build(); - logger.debug("CuratorFramework client built successfully with connectString: {}, sessionTimeout: {} and connectionTimeout: {}", - connectString, sessionTimeoutMs, connectionTimeoutMs); + client = CuratorUtils.getCuratorFrameworkClient(connectString, logger); client.start(); logger.info("Curator framework start operation invoked"); + int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000")); try { logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs); client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS); @@ -136,27 +96,7 @@ public class DistributedConfigurationManager extends ConfigurationManager { } @Override - public String getProperty(String key) { - // Todo Implement using curator - return null; - } - - @Override public void loadProperties() { // todo Implement the logic with Curator } - - public File getPropertiesFile(String filePath) { - return null; - } - - public File getConfigurationFile(String filePath) { - return null; - } - - public void publishConfiguration() { - for (String propertyFile : propertiesFiles) { - - } - } } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java index 38aea56..8970641 100644 --- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java +++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java @@ -19,24 +19,19 @@ package org.apache.oodt.config.distributed; import org.apache.commons.io.FileUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.api.ACLProvider; -import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.oodt.config.Constants; import org.apache.oodt.config.utils.CuratorUtils; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -44,7 +39,8 @@ import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; /** - * The class to publish configuration to Zookeeper + * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per + * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that. * * @author Imesha Sudasingha */ @@ -53,13 +49,15 @@ public class DistributedConfigurationPublisher { private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class); private Map<String, String> propertiesFiles; + private Map<String, String> configFiles; private String connectString; private CuratorFramework client; - private ZPaths zPaths; + private ZNodePaths zNodePaths; + private String componentName; - public DistributedConfigurationPublisher(String component, Map<String, String> propertiesFiles) { - this.propertiesFiles = propertiesFiles; - this.zPaths = new ZPaths(component); + public DistributedConfigurationPublisher(String componentName) { + this.componentName = componentName; + this.zNodePaths = new ZNodePaths(componentName); if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) { throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set"); @@ -88,45 +86,12 @@ public class DistributedConfigurationPublisher { * {@link Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble. */ private void startZookeeper() { - int connectionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "15000")); - int sessionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "60000")); - int retryInitialWaitMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "1000")); - int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3")); - int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000")); - - CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() - .connectString(connectString) - .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount)) - .connectionTimeoutMs(connectionTimeoutMs) - .sessionTimeoutMs(sessionTimeoutMs); - - /* - * If authorization information is available, those will be added to the client. NOTE: These auth info are - * for access control, therefore no authentication will happen when the client is being started. These - * info will only be required whenever a client is accessing an already create ZNode. For another client of - * another node to make use of a ZNode created by this node, it should also provide the same auth info. - */ - if (System.getProperty(Constants.Properties.ZK_USERNAME) != null && System.getProperty(Constants.Properties.ZK_PASSWORD) != null) { - String authenticationString = System.getProperty(Constants.Properties.ZK_USERNAME) + ":" + System.getProperty(Constants.Properties.ZK_PASSWORD); - builder.authorization("digest", authenticationString.getBytes()) - .aclProvider(new ACLProvider() { - public List<ACL> getDefaultAcl() { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - - public List<ACL> getAclForPath(String path) { - return ZooDefs.Ids.CREATOR_ALL_ACL; - } - }); - } - - client = builder.build(); - logger.debug("CuratorFramework client built successfully with connectString: {}, sessionTimeout: {} and connectionTimeout: {}", - connectString, sessionTimeoutMs, connectionTimeoutMs); + client = CuratorUtils.getCuratorFrameworkClient(connectString, logger); client.start(); logger.info("Curator framework start operation invoked"); + int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000")); try { logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs); client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS); @@ -142,8 +107,18 @@ public class DistributedConfigurationPublisher { } public void publishConfiguration() throws Exception { - for (Map.Entry<String, String> entry : propertiesFiles.entrySet()) { - logger.info("Publishing configuration {} - {}", entry.getKey(), entry.getValue()); + logger.debug("Publishing properties files : {}", propertiesFiles); + publishConfiguration(propertiesFiles, true); + logger.info("Properties files published successfully"); + + logger.debug("Publishing config files : {}", configFiles); + publishConfiguration(configFiles, false); + logger.info("Config files published successfully"); + } + + private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception { + for (Map.Entry<String, String> entry : fileMapping.entrySet()) { + logger.debug("Publishing configuration {} - {}", entry.getKey(), entry.getValue()); URL resource = Thread.currentThread().getContextClassLoader().getResource(entry.getKey()); String content; @@ -154,17 +129,17 @@ public class DistributedConfigurationPublisher { continue; } - String zNodePath = zPaths.getPropertiesZNodePath(entry.getValue()); + String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(entry.getValue()) : zNodePaths.getConfigurationZNodePath(entry.getValue()); if (client.checkExists().forPath(zNodePath) != null) { byte[] bytes = client.getData().forPath(zNodePath); String existingData = new String(bytes); if (content.equals(existingData)) { - logger.info("{} already exists in zookeeper at {}", entry.getKey(), entry.getValue()); + logger.warn("{} already exists in zookeeper at {}", entry.getKey(), entry.getValue()); continue; } else { Stat stat = client.setData().forPath(zNodePath, content.getBytes()); if (stat != null) { - logger.info("Published {} to {}", entry.getKey(), entry.getValue()); + logger.info("Published configuration file {} to {}", entry.getKey(), entry.getValue()); } } } else { @@ -173,28 +148,56 @@ public class DistributedConfigurationPublisher { } } - public static void main(String[] args) { - if (args.length < 2) { - logger.warn("Component name and at least one properties file needs to be given configuration publishing"); - throw new IllegalArgumentException("Requires at least two arguments, component and one properties file"); - } + public Map<String, String> getPropertiesFiles() { + return propertiesFiles; + } - logger.debug("Starting publishing configuration. Component : {}", args[0]); + public void setPropertiesFiles(Map<String, String> propertiesFiles) { + this.propertiesFiles = propertiesFiles; + } + + public Map<String, String> getConfigFiles() { + return configFiles; + } + + public void setConfigFiles(Map<String, String> configFiles) { + this.configFiles = configFiles; + } + + public ZNodePaths getZNodePaths() { + return zNodePaths; + } - String[] fileMapping = Arrays.copyOfRange(args, 1, args.length); - Map<String, String> propertiesFiles = new HashMap<>(); - for (String entry : fileMapping) { - String[] strings = entry.split("="); - propertiesFiles.put(strings[0], strings[1]); + public static void main(String[] args) { + if (args.length < 1) { + logger.error("Spring configuration file needs to be given as an argument"); + return; } - DistributedConfigurationPublisher distributedConfigurationPublisher = new DistributedConfigurationPublisher(args[0], propertiesFiles); + logger.info("Starting publishing configuration. Spring conf : {}", args[0]); + try { - distributedConfigurationPublisher.publishConfiguration(); + ApplicationContext applicationContext = new ClassPathXmlApplicationContext(args[0]); + Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class); + for (Object bean : distributedConfigurationPublisher.values()) { + DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; + + logger.debug("Publishing configuration for : {}", publisher.getComponentName()); + publisher.publishConfiguration(); + logger.info("Published configuration for : {}", publisher.getComponentName()); + } + } catch (BeansException e) { + logger.error("Error occurred when obtaining configuration publisher beans", e); + return; } catch (Exception e) { logger.error("Error occurred when publishing configuration to zookeeper", e); + return; } logger.info("Published configuration successfully"); } + + public String getComponentName() { + return componentName; + } } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java new file mode 100644 index 0000000..f390838 --- /dev/null +++ b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.config.distributed; + +import static org.apache.oodt.config.Constants.ZPaths.*; + +/** + * Class responsible for handling all the Zookeeper ZNode paths related to configuration + * + * @author Imesha Sudasingha + */ +public class ZNodePaths { + + /** ZNode for distinct components. /components/${component} */ + private String componentZNodePath; + private String componentZNodeRoot; + + /** ZNode path for properties files. /components/${component}/properties */ + private String propertiesZNodePath; + private String propertiesZNodeRoot; + + /** ZNode path for other configuration files. /components/${component}/configuration */ + private String configurationZNodePath; + private String configurationZNodeRoot; + + /** + * Creates the ZNode path structure accordingly to the <pre>componentName</pre> and <pre>propertiesFileNames</pre> given. + * + * @param componentName Name of the OODT component + */ + public ZNodePaths(String componentName) { + if (componentName == null) { + throw new IllegalArgumentException("Component name cannot be null"); + } + + componentZNodePath = SEPARATOR + COMPONENTS_PATH_NAME + SEPARATOR + componentName; + componentZNodeRoot = componentZNodePath + SEPARATOR; + + propertiesZNodePath = componentZNodeRoot + PROPERTIES_PATH_NAME; + propertiesZNodeRoot = propertiesZNodePath + SEPARATOR; + + configurationZNodePath = componentZNodeRoot + CONFIGURATION_PATH_NAME; + configurationZNodeRoot = configurationZNodePath + SEPARATOR; + } + + public String getComponentZNodePath() { + return componentZNodePath; + } + + public String getPropertiesZNodePath() { + return propertiesZNodePath; + } + + public String getConfigurationZNodePath() { + return configurationZNodePath; + } + + public String getPropertiesZNodePath(String subPath) { + return propertiesZNodeRoot + (subPath.startsWith(SEPARATOR) ? subPath.substring(1) : subPath); + } + + public String getConfigurationZNodePath(String subPath) { + return configurationZNodeRoot + (subPath.startsWith(SEPARATOR) ? subPath.substring(1) : subPath); + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java b/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java deleted file mode 100644 index 0e1d10c..0000000 --- a/config/src/main/java/org/apache/oodt/config/distributed/ZPaths.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oodt.config.distributed; - -import java.util.List; - -import static org.apache.oodt.config.Constants.ZPaths.*; - -/** - * Class responsible for handling all the Zookeeper ZNode paths related to configuration - * - * @author Imesha Sudasingha - */ -public class ZPaths { - - /** ZNode for distinct components. /components/${component} */ - public String componentZNodePath; - public String componentZNodeRoot; - - /** ZNode path for properties files. /components/${component}/properties */ - public String propertiesZNodePath; - public String propertiesZNodeRoot; - - /** - * Creates the ZNode path structure accordingly to the <pre>componentName</pre> and <pre>propertiesFileNames</pre> given. - * - * @param componentName Name of the OODT component - */ - public ZPaths(String componentName) { - if (componentName == null) { - throw new IllegalArgumentException("Component name cannot be null"); - } - - componentZNodePath = SEPARATOR + COMPONENTS_PATH_NAME + SEPARATOR + componentName; - componentZNodeRoot = componentZNodePath + SEPARATOR; - - propertiesZNodePath = componentZNodeRoot + PROPERTIES_PATH_NAME; - propertiesZNodeRoot = propertiesZNodePath + SEPARATOR; - } - - public String getComponentZNodePath() { - return componentZNodePath; - } - - public String getPropertiesZNodePath() { - return propertiesZNodePath; - } - - public String getPropertiesZNodePath(String subPath) { - return propertiesZNodeRoot + (subPath.startsWith(SEPARATOR) ? subPath.substring(1) : subPath); - } -} http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java index 23cf0a2..6ecb1b7 100644 --- a/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java +++ b/config/src/main/java/org/apache/oodt/config/standalone/StandaloneConfigurationManager.java @@ -23,8 +23,8 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; +import java.util.ArrayList; import java.util.List; /** @@ -34,35 +34,22 @@ import java.util.List; */ public class StandaloneConfigurationManager extends ConfigurationManager { - /** Logger instance for logging */ private static final Logger logger = LoggerFactory.getLogger(StandaloneConfigurationManager.class); - public StandaloneConfigurationManager(String component, List<String> propertiesFiles) { - super(component, propertiesFiles); - } + private List<String> propertiesFiles; - /** {@inheritDoc} */ - @Override - public String getProperty(String key) { - return System.getProperty(key); + public StandaloneConfigurationManager(String component, List<String> propertiesFiles) { + super(component); + this.propertiesFiles = propertiesFiles != null ? propertiesFiles : new ArrayList<String>(); } /** {@inheritDoc} */ @Override public void loadProperties() throws IOException { for (String file : propertiesFiles) { + logger.debug("Loading properties from file : {}", file); System.getProperties().load(new FileInputStream(new File(file))); + logger.debug("Properties loaded from file : {}", file); } } - - /** {@inheritDoc} */ - @Override - public File getPropertiesFile(String filePath) throws FileNotFoundException { - File file = new File(filePath); - if (!propertiesFiles.contains(filePath) || !file.exists()) { - throw new FileNotFoundException("Couldn't find properties file located at: " + filePath); - } - - return file; - } } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java b/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java index d064080..c73d48b 100644 --- a/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java +++ b/config/src/main/java/org/apache/oodt/config/utils/CuratorUtils.java @@ -18,13 +18,20 @@ package org.apache.oodt.config.utils; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.api.ACLProvider; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.oodt.config.Constants; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileInputStream; import java.io.IOException; +import java.util.List; import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; @@ -77,4 +84,43 @@ public class CuratorUtils { return client.checkExists().forPath(path) != null ? new String(client.getData().forPath(path)) : null; } + public static CuratorFramework getCuratorFrameworkClient(String connectString, Logger logger) { + int connectionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "15000")); + int sessionTimeoutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "60000")); + int retryInitialWaitMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "1000")); + int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3")); + + CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .connectString(connectString) + .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount)) + .connectionTimeoutMs(connectionTimeoutMs) + .sessionTimeoutMs(sessionTimeoutMs); + + /* + * If authorization information is available, those will be added to the client. NOTE: These auth info are + * for access control, therefore no authentication will happen when the client is being started. These + * info will only be required whenever a client is accessing an already create ZNode. For another client of + * another node to make use of a ZNode created by this node, it should also provide the same auth info. + */ + if (System.getProperty(Constants.Properties.ZK_USERNAME) != null && System.getProperty(Constants.Properties.ZK_PASSWORD) != null) { + String authenticationString = System.getProperty(Constants.Properties.ZK_USERNAME) + ":" + System.getProperty(Constants.Properties.ZK_PASSWORD); + builder.authorization("digest", authenticationString.getBytes()) + .aclProvider(new ACLProvider() { + public List<ACL> getDefaultAcl() { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + + public List<ACL> getAclForPath(String path) { + return ZooDefs.Ids.CREATOR_ALL_ACL; + } + }); + } + + CuratorFramework client = builder.build(); + logger.debug("CuratorFramework client built successfully with connectString: {}, sessionTimeout: {} and connectionTimeout: {}", + connectString, sessionTimeoutMs, connectionTimeoutMs); + + return client; + } + } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java ---------------------------------------------------------------------- diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java index 665f785..2a3c903 100644 --- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java +++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisherTest.java @@ -26,17 +26,20 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; import java.io.File; import java.io.IOException; import java.net.URI; -import java.util.Arrays; +import java.util.Map; -import static org.apache.oodt.config.Constants.Components.FILE_MANAGER; import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; public class DistributedConfigurationPublisherTest { + private static final String DISTRIBUTED_CONFIG_PUBLISHER_SPRING_CONFIG = "distributed-config-publisher.xml"; + private static TestingServer zookeeper; private static CuratorFramework client; @@ -56,27 +59,43 @@ public class DistributedConfigurationPublisherTest { @Test public void publishConfiguration() throws Exception { - String args[] = new String[]{ - FILE_MANAGER, - "filemgr.properties=/etc/filemgr.properties", - "mime-types.xml=/etc/mime-types.xml" - }; - DistributedConfigurationPublisher.main(args); + DistributedConfigurationPublisher.main(new String[]{DISTRIBUTED_CONFIG_PUBLISHER_SPRING_CONFIG}); + + ApplicationContext applicationContext = new ClassPathXmlApplicationContext(DISTRIBUTED_CONFIG_PUBLISHER_SPRING_CONFIG); + Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class); + + for (Object bean : distributedConfigurationPublisher.values()) { + DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; + ZNodePaths zNodePaths = publisher.getZNodePaths(); + + // Checking for configuration files + for (Map.Entry<String, String> entry : publisher.getPropertiesFiles().entrySet()) { + String zNodePath = zNodePaths.getPropertiesZNodePath(entry.getValue()); + + Assert.assertNotNull(client.checkExists().forPath(zNodePath)); + + String storedContent = new String(client.getData().forPath(zNodePath)); + + URI file = Thread.currentThread().getContextClassLoader().getResource(entry.getKey()).toURI(); + String fileContent = FileUtils.readFileToString(new File(file)); + + Assert.assertEquals(fileContent, storedContent); + } + + // Checking for configuration files + for (Map.Entry<String, String> entry : publisher.getConfigFiles().entrySet()) { + String zNodePath = zNodePaths.getConfigurationZNodePath(entry.getValue()); - String[] fileMappings = Arrays.copyOfRange(args, 1, args.length); - ZPaths zPaths = new ZPaths(FILE_MANAGER); - for (String mapping : fileMappings) { - String[] parts = mapping.split("="); - String zNodePath = zPaths.getPropertiesZNodePath(parts[1]); + Assert.assertNotNull(client.checkExists().forPath(zNodePath)); - Assert.assertNotNull(client.checkExists().forPath(zNodePath)); + String storedContent = new String(client.getData().forPath(zNodePath)); - String storedContent = new String(client.getData().forPath(zNodePath)); + URI file = Thread.currentThread().getContextClassLoader().getResource(entry.getKey()).toURI(); + String fileContent = FileUtils.readFileToString(new File(file)); - URI file = Thread.currentThread().getContextClassLoader().getResource(parts[0]).toURI(); - String fileContent = FileUtils.readFileToString(new File(file)); + Assert.assertEquals(fileContent, storedContent); + } - Assert.assertEquals(fileContent, storedContent); } } http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/config/src/test/resources/distributed-config-publisher.xml ---------------------------------------------------------------------- diff --git a/config/src/test/resources/distributed-config-publisher.xml b/config/src/test/resources/distributed-config-publisher.xml new file mode 100644 index 0000000..d938b6d --- /dev/null +++ b/config/src/test/resources/distributed-config-publisher.xml @@ -0,0 +1,56 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd";> + + <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher"> + + <constructor-arg value="filemgr"/> + + <property name="propertiesFiles"> + <map> + <entry key="filemgr.properties" value="/etc/filemgr.properties"/> + </map> + </property> + + <property name="configFiles"> + <map> + <entry key="mime-types.xml" value="/etc/mime-types.xml"/> + </map> + </property> + </bean> + + <bean id="resmgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher"> + + <constructor-arg value="resmgr"/> + + <property name="propertiesFiles"> + <map> + <entry key="filemgr.properties" value="/etc/resmgr.properties"/> + </map> + </property> + + <property name="configFiles"> + <map> + <entry key="mime-types.xml" value="/etc/mime-types.xml"/> + </map> + </property> + </bean> + +</beans> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/fadf149c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java ---------------------------------------------------------------------- diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java index d43369a..9d7d740 100644 --- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java +++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java @@ -125,18 +125,17 @@ public class XmlRpcFileManager { webServer.addHandler("filemgr", this); webServer.start(); + List<String> propertiesFiles = new ArrayList<>(); + // set up the configuration, if there is any if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) { - String configFile = System.getProperty("org.apache.oodt.cas.filemgr.properties"); - LOG.log(Level.INFO, "Loading File Manager Configuration Properties from: [" + configFile + "]"); - - List<String> propertiesFiles = new ArrayList<String>(); + String propertiesFile = System.getProperty("org.apache.oodt.cas.filemgr.properties"); + LOG.log(Level.INFO, "Using File Manager Configuration Properties file : [" + propertiesFile + "]"); propertiesFiles.add(System.getProperty("org.apache.oodt.cas.filemgr.properties")); - configurationManager = ConfigurationManagerFactory.getConfigurationManager(componentName, propertiesFiles); - } else { - configurationManager = ConfigurationManagerFactory.getConfigurationManager(componentName,null); } + configurationManager = ConfigurationManagerFactory.getConfigurationManager(componentName, propertiesFiles); + this.loadConfiguration(); LOG.log(Level.INFO, "File Manager started by " + System.getProperty("user.name", "unknown")); }
