Added zip packaging for distributed configuration publisher
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/27a28daa Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/27a28daa Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/27a28daa Branch: refs/heads/master Commit: 27a28daa3d3c448b462bd9a33eb6ac4fd859b75a Parents: 5fcf222 Author: Imesha Sudasingha <[email protected]> Authored: Sun Jul 2 20:53:49 2017 +0530 Committer: Imesha Sudasingha <[email protected]> Committed: Sun Jul 2 20:53:49 2017 +0530 ---------------------------------------------------------------------- config/pom.xml | 36 +- config/src/main/assembly/assembly.xml | 56 + .../java/org/apache/oodt/config/Constants.java | 7 + .../DistributedConfigurationManager.java | 1 - .../DistributedConfigurationPublisher.java | 208 - .../config/distributed/cli/CmdLineOptions.java | 57 + .../cli/DistributedConfigurationPublisher.java | 300 ++ .../config/distributed/utils/CuratorUtils.java | 1 + .../src/main/resources/etc/config-publisher.xml | 36 + config/src/main/resources/etc/log4j.xml | 34 + .../main/resources/examples/filemgr.properties | 114 + .../src/main/resources/examples/mime-types.xml | 4119 ++++++++++++++++++ .../DistributedConfigurationPublisherTest.java | 54 +- .../oodt/config/distributed/TestServer.java | 45 + .../resources/distributed-config-publisher.xml | 56 - .../src/test/resources/etc/config-publisher.xml | 50 + config/src/test/resources/filemgr.properties | 114 - config/src/test/resources/log4j.xml | 34 - config/src/test/resources/mime-types.xml | 4119 ------------------ core/pom.xml | 6 + 20 files changed, 4862 insertions(+), 4585 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/pom.xml ---------------------------------------------------------------------- diff --git a/config/pom.xml b/config/pom.xml index 891f520..d75e032 100644 --- a/config/pom.xml +++ b/config/pom.xml @@ -16,50 +16,68 @@ <description>OODT project for distributed configuration management support</description> <dependencies> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-framework</artifactId> </dependency> - <dependency> <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> - <dependency> <groupId>org.springframework</groupId> <artifactId>spring-beans</artifactId> </dependency> - <dependency> <groupId>org.springframework</groupId> <artifactId>spring-context</artifactId> </dependency> - <dependency> <groupId>org.springframework</groupId> <artifactId>spring-core</artifactId> </dependency> - + <dependency> + <groupId>args4j</groupId> + <artifactId>args4j</artifactId> + </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </dependency> - <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-test</artifactId> </dependency> </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.2-beta-2</version> + <configuration> + <descriptors> + <descriptor>src/main/assembly/assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <goals> + <goal>single</goal> + </goals> + <phase>package</phase> + </execution> + </executions> + </plugin> + </plugins> + </build> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/assembly/assembly.xml ---------------------------------------------------------------------- diff --git a/config/src/main/assembly/assembly.xml b/config/src/main/assembly/assembly.xml new file mode 100644 index 0000000..2743cab --- /dev/null +++ b/config/src/main/assembly/assembly.xml @@ -0,0 +1,56 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<assembly> + <id>dist</id> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>${project.artifactId}-${project.version}</baseDirectory> + <includeSiteDirectory>false</includeSiteDirectory> + <fileSets> + <fileSet> + <directory>${basedir}</directory> + <outputDirectory>.</outputDirectory> + <includes> + <include>LICENSE.txt</include> + <include>CHANGES.txt</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/bin</directory> + <outputDirectory>bin</outputDirectory> + <includes/> + <fileMode>755</fileMode> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources</directory> + <outputDirectory>.</outputDirectory> + <includes/> + </fileSet> + </fileSets> + <dependencySets> + <dependencySet> + <outputDirectory>lib</outputDirectory> + <unpack>false</unpack> + <useProjectArtifact>true</useProjectArtifact> + <useTransitiveDependencies>true</useTransitiveDependencies> + <unpackOptions/> + </dependencySet> + </dependencySets> +</assembly> http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/Constants.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java index 033101b..d6969fe 100644 --- a/config/src/main/java/org/apache/oodt/config/Constants.java +++ b/config/src/main/java/org/apache/oodt/config/Constants.java @@ -17,6 +17,8 @@ package org.apache.oodt.config; +import java.io.File; + /** * Constants to be used by the config package * @@ -27,12 +29,17 @@ public class Constants { private Constants() { } + public static final String SEPARATOR = File.separator; + /** Node name to be used when the configuration manager is the standalone version */ public static final String STANDALONE_NODE_NAME = "local"; /** Default environment name to be used */ public static final String DEFAULT_ENVIRONMENT = "default"; + /** The XML file name in which the configuration to be published to zookeeper is defined */ + public static final String CONFIG_PUBLISHER_XML = "etc" + SEPARATOR + "config-publisher.xml"; + public static class Properties { /** The system property to be set in order to enable distributed configuration management */ public static final String ENABLE_DISTRIBUTED_CONFIGURATION = "org.apache.oodt.config.distributed"; http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java index d16df7e..1a3f390 100644 --- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java +++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java @@ -114,7 +114,6 @@ public class DistributedConfigurationManager extends ConfigurationManager { logger.debug("Saving configuration files for : {}", componentName); saveConfigFiles(); logger.info("Configuration files saved for : {}", componentName); - } /** http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java deleted file mode 100644 index 37b57c9..0000000 --- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oodt.config.distributed; - -import org.apache.commons.io.FileUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.oodt.config.Constants; -import org.apache.oodt.config.distributed.utils.CuratorUtils; -import org.apache.zookeeper.data.Stat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.support.ClassPathXmlApplicationContext; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; -import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; - -/** - * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per - * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that. - * - * @author Imesha Sudasingha - */ -public class DistributedConfigurationPublisher { - - private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class); - - private Map<String, String> propertiesFiles; - private Map<String, String> configFiles; - private String connectString; - private CuratorFramework client; - private ZNodePaths zNodePaths; - private String componentName; - - public DistributedConfigurationPublisher(String componentName) { - this.componentName = componentName; - this.zNodePaths = new ZNodePaths(componentName); - - if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) { - throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set"); - } - - if (System.getProperty(ZK_PROPERTIES_FILE) != null) { - try { - CuratorUtils.loadZookeeperProperties(); - } catch (IOException e) { - logger.error("Error occurred when loading properties from properties file"); - } - } - - if (System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) { - throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble"); - } - - connectString = System.getProperty(Constants.Properties.ZK_CONNECT_STRING); - logger.info("Using zookeeper connect string : {}", connectString); - - startZookeeper(); - } - - /** - * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of - * {@link Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble. - */ - private void startZookeeper() { - client = CuratorUtils.getCuratorFrameworkClient(connectString, logger); - - client.start(); - logger.info("Curator framework start operation invoked"); - - int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000")); - try { - logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs); - client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException ex) { - logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString); - } - - if (!client.getZookeeperClient().isConnected()) { - throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString); - } - - logger.info("CuratorFramework client started successfully"); - } - - public void publishConfiguration() throws Exception { - logger.debug("Publishing properties files : {}", propertiesFiles); - publishConfiguration(propertiesFiles, true); - logger.info("Properties files published successfully"); - - logger.debug("Publishing config files : {}", configFiles); - publishConfiguration(configFiles, false); - logger.info("Config files published successfully"); - - // TODO: 6/17/17 Verify whether the given configuration are published correctly - } - - private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception { - for (Map.Entry<String, String> entry : fileMapping.entrySet()) { - logger.debug("Publishing configuration {} - {}", entry.getKey(), entry.getValue()); - URL resource = Thread.currentThread().getContextClassLoader().getResource(entry.getKey()); - - String content; - try { - content = FileUtils.readFileToString(new File(resource.toURI())); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to read file : {}", entry.getKey(), e); - continue; - } - - String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(entry.getValue()) : zNodePaths.getConfigurationZNodePath(entry.getValue()); - if (client.checkExists().forPath(zNodePath) != null) { - byte[] bytes = client.getData().forPath(zNodePath); - String existingData = new String(bytes); - if (content.equals(existingData)) { - logger.warn("{} already exists in zookeeper at {}", entry.getKey(), entry.getValue()); - } else { - Stat stat = client.setData().forPath(zNodePath, content.getBytes()); - if (stat != null) { - logger.info("Published configuration file {} to {}", entry.getKey(), entry.getValue()); - } - } - } else { - /* - * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted - * when no child node is present under them. - */ - client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes()); - } - } - } - - public Map<String, String> getPropertiesFiles() { - return propertiesFiles; - } - - public void setPropertiesFiles(Map<String, String> propertiesFiles) { - this.propertiesFiles = propertiesFiles; - } - - public Map<String, String> getConfigFiles() { - return configFiles; - } - - public void setConfigFiles(Map<String, String> configFiles) { - this.configFiles = configFiles; - } - - public ZNodePaths getZNodePaths() { - return zNodePaths; - } - - public static void main(String[] args) { - if (args.length < 1) { - logger.error("Spring configuration file needs to be given as an argument"); - return; - } - - logger.info("Starting publishing configuration. Spring conf : {}", args[0]); - - try { - ApplicationContext applicationContext = new ClassPathXmlApplicationContext(args[0]); - Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class); - for (Object bean : distributedConfigurationPublisher.values()) { - DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; - - logger.debug("Publishing configuration for : {}", publisher.getComponentName()); - publisher.publishConfiguration(); - logger.info("Published configuration for : {}", publisher.getComponentName()); - } - } catch (BeansException e) { - logger.error("Error occurred when obtaining configuration publisher beans", e); - return; - } catch (Exception e) { - logger.error("Error occurred when publishing configuration to zookeeper", e); - return; - } - - logger.info("Published configuration successfully"); - } - - public String getComponentName() { - return componentName; - } -} http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java new file mode 100644 index 0000000..afa6350 --- /dev/null +++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/CmdLineOptions.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.config.distributed.cli; + +import org.kohsuke.args4j.Option; + +/** + * Bean class used to map CLI options and arguments provided by the user when publishing configuration to zookeeper + * + * @author Imesha Sudaingha + */ +public class CmdLineOptions { + + @Option(name = "-connectString", usage = "Zookeeper connect string", required = true) + private String connectString; + + @Option(name = "-publish", usage = "Publishes configuration specified in the spring config file to zookeeper. " + + "Any current similar config in zookeeper will be overwritten. If not specified, command will be assumed as a publish") + private boolean publish = true; + + @Option(name = "-verify", usage = "Verifies the content in the local files and the published ones. Results will be printed.") + private boolean verify = true; + + @Option(name = "-clear", usage = "Unpublish any configuration which has been published earlier using the same spring config file") + private boolean clear = false; + + public String getConnectString() { + return connectString; + } + + public boolean isPublish() { + return publish; + } + + public boolean isVerify() { + return verify; + } + + public boolean isClear() { + return clear; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java new file mode 100644 index 0000000..03cdfc7 --- /dev/null +++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/DistributedConfigurationPublisher.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.config.distributed.cli; + +import org.apache.commons.io.FileUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.oodt.config.Constants; +import org.apache.oodt.config.distributed.ZNodePaths; +import org.apache.oodt.config.distributed.utils.CuratorUtils; +import org.apache.zookeeper.data.Stat; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING; +import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE; + +/** + * The class to publish configuration to Zookeeper. When using distributed configuration with OODT, configuration per + * each component type needs to be stored in zookeeper beforehand. This class, provides the means to do that. + * + * @author Imesha Sudasingha + */ +public class DistributedConfigurationPublisher { + + private static final Logger logger = LoggerFactory.getLogger(DistributedConfigurationPublisher.class); + + private Map<String, String> propertiesFiles; + private Map<String, String> configFiles; + private String connectString; + private CuratorFramework client; + private ZNodePaths zNodePaths; + private String componentName; + + public DistributedConfigurationPublisher(String componentName) { + this.componentName = componentName; + this.zNodePaths = new ZNodePaths(componentName); + + if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) { + throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set"); + } + + if (System.getProperty(ZK_PROPERTIES_FILE) != null) { + try { + CuratorUtils.loadZookeeperProperties(); + } catch (IOException e) { + logger.error("Error occurred when loading properties from properties file"); + } + } + + if (System.getProperty(ZK_CONNECT_STRING) == null) { + throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble"); + } + + connectString = System.getProperty(ZK_CONNECT_STRING); + logger.info("Using zookeeper connect string : {}", connectString); + + startZookeeper(); + } + + /** + * Creates a {@link CuratorFramework} instance and start it. This method will wait a maximum amount of + * {@link Constants.Properties#ZK_STARTUP_TIMEOUT} milli-seconds until the client connects to the zookeeper ensemble. + */ + private void startZookeeper() { + client = CuratorUtils.getCuratorFrameworkClient(connectString, logger); + + client.start(); + logger.info("Curator framework start operation invoked"); + + int startupTimeOutMs = Integer.parseInt(System.getProperty(Constants.Properties.ZK_STARTUP_TIMEOUT, "30000")); + try { + logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs); + client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException ex) { + logger.error("Interrupted while waiting to connect zookeeper (connectString : {}) : {}", ex, connectString); + } + + if (!client.getZookeeperClient().isConnected()) { + throw new IllegalStateException("Could not connect to ZooKeeper : " + connectString); + } + + logger.info("CuratorFramework client started successfully"); + } + + /** + * Publishes the configuration files specified to zookeeper. If an exception is thrown while configuration being + * published, no further publishing attempts will be carried on. Error will be reported to user. + * + * @throws Exception Zookeeper errors + */ + public void publishConfiguration() throws Exception { + logger.debug("Publishing properties files : {}", propertiesFiles); + publishConfiguration(propertiesFiles, true); + logger.info("Properties files published successfully"); + + logger.debug("Publishing config files : {}", configFiles); + publishConfiguration(configFiles, false); + logger.info("Config files published successfully"); + } + + /** + * Verified whether the actual content of the local files specified to be published are 100% similar to the ones that + * has been published and stored in zookeeper at the moment. + * + * @return true | if content are up to date and similar + */ + public boolean verifyPublishedConfiguration() { + try { + return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false); + } catch (Exception e) { + logger.error("Error occurred when checking published config", e); + return false; + } + } + + private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception { + for (Map.Entry<String, String> entry : fileMapping.entrySet()) { + String filePath = entry.getKey(); + String relativeZNodePath = entry.getValue(); + logger.info("Publishing configuration {} - {}", filePath, relativeZNodePath); + + String content = getFileContent(filePath); + + String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath); + if (client.checkExists().forPath(zNodePath) != null) { + byte[] bytes = client.getData().forPath(zNodePath); + String existingData = new String(bytes); + if (content.equals(existingData)) { + logger.warn("{} already exists in zookeeper at {}", filePath, relativeZNodePath); + } else { + Stat stat = client.setData().forPath(zNodePath, content.getBytes()); + if (stat != null) { + logger.info("Published configuration file {} to {}", filePath, relativeZNodePath); + } else { + logger.warn("Unable to publish configuration file {} to {}", filePath, relativeZNodePath); + } + } + } else { + /* + * Creating these ZNodes with parent 'Containers' is important since containers are automatically deleted + * when no child node is present under them. + */ + client.create().creatingParentContainersIfNeeded().forPath(zNodePath, content.getBytes()); + logger.info("Replaced old published configuration at {} with content of file : {}", relativeZNodePath, filePath); + } + } + } + + private boolean verifyPublishedConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception { + boolean noError = true; + for (Map.Entry<String, String> entry : fileMapping.entrySet()) { + String filePath = entry.getKey(); + String relativeZNodePath = entry.getValue(); + logger.info("Checking published configuration for {} - {}", filePath, relativeZNodePath); + + String originalContent = getFileContent(filePath); + + String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath); + if (client.checkExists().forPath(zNodePath) == null) { + logger.error("File : {} hasn't been published to ZNode : {}", filePath, relativeZNodePath); + noError = false; + continue; + } + + String publishedContent = new String(client.getData().forPath(zNodePath)); + if (!publishedContent.equals(originalContent)) { + logger.error("Content of local file : {} and content published to {} are not similar", filePath, relativeZNodePath); + noError = false; + continue; + } + + logger.info("{} - {} configuration checked and OK", filePath, relativeZNodePath); + } + + return noError; + } + + private String getFileContent(String file) { + String content; + try { + content = FileUtils.readFileToString(new File(file)); + } catch (IOException e) { + logger.error("Unable to read file : {}", file, e); + throw new IllegalArgumentException("Unable to read content of the file : " + file); + } + + return content; + } + + public Map<String, String> getPropertiesFiles() { + return propertiesFiles; + } + + public void setPropertiesFiles(Map<String, String> propertiesFiles) { + this.propertiesFiles = propertiesFiles; + } + + public Map<String, String> getConfigFiles() { + return configFiles; + } + + public void setConfigFiles(Map<String, String> configFiles) { + this.configFiles = configFiles; + } + + public ZNodePaths getZNodePaths() { + return zNodePaths; + } + + public static void main(String[] args) throws Exception { + CmdLineOptions cmdLineOptions = new CmdLineOptions(); + CmdLineParser parser = new CmdLineParser(cmdLineOptions); + + try { + parser.parseArgument(args); + } catch (CmdLineException e) { + System.err.println("There's an error in your command"); + parser.printUsage(System.err); + return; + } + + if (cmdLineOptions.getConnectString() == null && System.getProperty(ZK_CONNECT_STRING) == null) { + System.err.println("Zookeeper connect string is not found"); + parser.printUsage(System.err); + return; + } else { + System.setProperty(ZK_CONNECT_STRING, cmdLineOptions.getConnectString()); + } + + System.out.println("Starting configuration publishing"); + + try { + ApplicationContext applicationContext = new ClassPathXmlApplicationContext(Constants.CONFIG_PUBLISHER_XML); + Map distributedConfigurationPublisher = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class); + + for (Object bean : distributedConfigurationPublisher.values()) { + DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean; + System.out.println(String.format("\nProcessing commands for component : %s", publisher.getComponentName())); + + if (cmdLineOptions.isPublish()) { + System.out.println(String.format("Publishing configuration for : %s", publisher.getComponentName())); + publisher.publishConfiguration(); + System.out.println(String.format("Published configuration for : %s", publisher.getComponentName())); + System.out.printf("\n"); + } + + if (cmdLineOptions.isVerify()) { + System.out.println(String.format("Verifying configuration for : %s", publisher.getComponentName())); + publisher.verifyPublishedConfiguration(); + System.out.println(String.format("Verified configuration for : %s", publisher.getComponentName())); + System.out.printf("\n"); + } + + if (cmdLineOptions.isClear()) { + System.out.println(String.format("Clearing configuration for : %s", publisher.getComponentName())); + // TODO: 7/2/17 Implement configuration removal + System.out.println(String.format("Cleared configuration for : %s", publisher.getComponentName())); + System.out.printf("\n"); + } + } + } catch (BeansException e) { + logger.error("Error occurred when obtaining configuration publisher beans", e); + throw e; + } catch (Exception e) { + logger.error("Error occurred when publishing configuration to zookeeper", e); + throw e; + } + + logger.info("Published configuration successfully"); + } + + public String getComponentName() { + return componentName; + } +} http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java ---------------------------------------------------------------------- diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java index 6f9c97f..1b10ef0 100644 --- a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java +++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java @@ -100,6 +100,7 @@ public class CuratorUtils { int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3")); CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() + .namespace(NAMESPACE) .connectString(connectString) .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount)) .connectionTimeoutMs(connectionTimeoutMs) http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/resources/etc/config-publisher.xml ---------------------------------------------------------------------- diff --git a/config/src/main/resources/etc/config-publisher.xml b/config/src/main/resources/etc/config-publisher.xml new file mode 100644 index 0000000..cc1261e --- /dev/null +++ b/config/src/main/resources/etc/config-publisher.xml @@ -0,0 +1,36 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd";> + + <bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.cli.DistributedConfigurationPublisher"> + <constructor-arg value="filemgr"/> + <property name="propertiesFiles"> + <map key-type="java.lang.String" value-type="java.lang.String"> + <entry key="../examples/filemgr.properties" value="/etc/filemgr.properties"/> + </map> + </property> + <property name="configFiles"> + <map key-type="java.lang.String" value-type="java.lang.String"> + <entry key="../examples/mime-types.xml" value="/etc/mime-types.xml"/> + </map> + </property> + </bean> + +</beans> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/resources/etc/log4j.xml ---------------------------------------------------------------------- diff --git a/config/src/main/resources/etc/log4j.xml b/config/src/main/resources/etc/log4j.xml new file mode 100644 index 0000000..8f49451 --- /dev/null +++ b/config/src/main/resources/etc/log4j.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/";> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out"/> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%-5p %c{1} - %m%n"/> + </layout> + </appender> + + <root> + <priority value="INFO"/> + <appender-ref ref="console"/> + </root> + +</log4j:configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oodt/blob/27a28daa/config/src/main/resources/examples/filemgr.properties ---------------------------------------------------------------------- diff --git a/config/src/main/resources/examples/filemgr.properties b/config/src/main/resources/examples/filemgr.properties new file mode 100644 index 0000000..4d8d957 --- /dev/null +++ b/config/src/main/resources/examples/filemgr.properties @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Configuration properties for the File Manager + +# repository factory +filemgr.repository.factory=org.apache.oodt.cas.filemgr.repository.XMLRepositoryManagerFactory + +# Lucene catalog factory +filemgr.catalog.factory=org.apache.oodt.cas.filemgr.catalog.LuceneCatalogFactory + +# data transfer factory +filemgr.datatransfer.factory=org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory + +# validation layer factory +filemgr.validationLayer.factory=org.apache.oodt.cas.filemgr.validation.XMLValidationLayerFactory + +# xml rpc client configuration +org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes=20 +org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes=60 +#org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries=0 +#org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds=3 + +# mapped data source catalog configuration +#org.apache.oodt.cas.filemgr.catalog.mappeddatasource.mapFile=/path/to/ops.catalog.typemap.properties + +# lucene catalog configuration +org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=[OODT_HOME]/data/catalog +org.apache.oodt.cas.filemgr.catalog.lucene.pageSize=20 +org.apache.oodt.cas.filemgr.catalog.lucene.commitLockTimeout.seconds=60 +org.apache.oodt.cas.filemgr.catalog.lucene.writeLockTimeout.seconds=60 +org.apache.oodt.cas.filemgr.catalog.lucene.mergeFactor=20 + +# XML repository manager configuration +org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:[FILEMGR_HOME]/policy/oodt + +# XML validation layer configuration +org.apache.oodt.cas.filemgr.validation.dirs=file:[FILEMGR_HOME]/policy/oodt + +# remote data transfer configuration +org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024 + +# location of Mime-Type repository +org.apache.oodt.cas.filemgr.mime.type.repository=[FILEMGR_HOME]/etc/mime-types.xml + + +############ data source configuration ################################## +# +# These 3 blocks of config properties can be used to setup a catalog, +# repository manager, and validation layer based on a RDBMS such as +# Oracle, MySQL, PostgreSQL, or any others that support a jdbc connection. +# Just un-comment the following blocks of properties and configure as +# needed. +# +######################################################################### + +# datasource catalog config +#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.url=some_jdbc_url +#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.user=user +#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.pass=pass +#org.apache.oodt.cas.filemgr.catalog.datasource.jdbc.driver=driver.class.name +#org.apache.oodt.cas.filemgr.catalog.datasource.quoteFields=false +#org.apache.oodt.cas.filemgr.catalog.datasource.pageSize=20 +#org.apache.oodt.cas.filemgr.catalog.datasource.cacheUpdateMinutes=5 + +# data source repository manager configuration +#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.url=some_jdbc_url +#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.user=user +#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.pass=pass +#org.apache.oodt.cas.filemgr.repositorymgr.datasource.jdbc.driver=driver.class.name + +# data source validation layer configuration +#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.url=some_jdbc_url +#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.user=user +#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.pass=pass +#org.apache.oodt.cas.filemgr.validation.datasource.jdbc.driver=driver.class.name +#org.apache.oodt.cas.filemgr.validation.datasource.quoteFields=false + + +# tells the file manager system layer to include product instance metadata +# NOTE: here are the expected field mappings +# +# product.getProductId() -> ProductId +# product.getProductName() -> ProductName +# product.getProductStructure() -> ProductStructure +# product.getTransferStatus() -> ProductTransferStatus +# product.getRootRef() -> ProductRootReference + +# for the references returned by product.getProductReferences() the following +# metadata fields will be added (order will be maintained, such that data store +# ref at index 0 will map to orig ref at index 0, etc.) +# +# ProductDataStoreReferences (list of all data store references: +# note already translated into path, not URI) +# ProductOrigReferences (list of all orig references: +# note already translated into path, not URI) +# ProductMimeType (list of all references' mime-types) +# ProductFileSize (list of all references' file sizes) + +org.apache.oodt.cas.filemgr.metadata.expandProduct=false \ No newline at end of file
