KNOX-1107 - Remote Configuration Registry Client Service (Phil Zampino via lmccay)
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/5af2413c Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/5af2413c Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/5af2413c Branch: refs/heads/KNOX-998-Package_Restructuring Commit: 5af2413c4ea4593a081e4f5ad8ba6b2d3cf78e12 Parents: 24d51ad Author: Larry McCay <[email protected]> Authored: Mon Nov 27 12:46:35 2017 -0500 Committer: Larry McCay <[email protected]> Committed: Mon Nov 27 12:46:35 2017 -0500 ---------------------------------------------------------------------- gateway-server/pom.xml | 9 + .../apache/hadoop/gateway/GatewayMessages.java | 40 +- .../gateway/config/impl/GatewayConfigImpl.java | 47 +- .../gateway/services/CLIGatewayServices.java | 10 + .../services/DefaultGatewayServices.java | 14 +- .../topology/impl/DefaultTopologyService.java | 38 +- .../DefaultConfigurationMonitorProvider.java | 31 ++ .../DefaultRemoteConfigurationMonitor.java | 163 +++++++ .../RemoteConfigurationMonitorFactory.java | 74 ++++ .../simple/SimpleDescriptorHandler.java | 1 + ...y.monitor.RemoteConfigurationMonitorProvider | 19 + .../config/impl/GatewayConfigImplTest.java | 28 ++ .../topology/DefaultTopologyServiceTest.java | 10 +- .../ZooKeeperConfigurationMonitorTest.java | 355 ++++++++++++++++ .../apache/hadoop/gateway/util/KnoxCLITest.java | 26 +- .../hadoop/gateway/websockets/BadUrlTest.java | 11 + .../gateway/websockets/WebsocketEchoTest.java | 11 + .../WebsocketMultipleConnectionTest.java | 11 + gateway-service-remoteconfig/pom.xml | 89 ++++ .../remote/RemoteConfigurationMessages.java | 46 ++ ...nfigurationRegistryClientServiceFactory.java | 41 ++ ...figurationRegistryClientServiceProvider.java | 27 ++ .../RemoteConfigurationRegistryConfig.java | 43 ++ .../DefaultRemoteConfigurationRegistries.java | 104 +++++ .../config/RemoteConfigurationRegistries.java | 33 ++ .../RemoteConfigurationRegistriesAccessor.java | 60 +++ .../RemoteConfigurationRegistriesParser.java | 48 +++ .../config/RemoteConfigurationRegistry.java | 139 ++++++ .../config/remote/zk/CuratorClientService.java | 423 ++++++++++++++++++ .../RemoteConfigurationRegistryJAASConfig.java | 169 ++++++++ .../remote/zk/ZooKeeperClientService.java | 25 ++ .../zk/ZooKeeperClientServiceProvider.java | 34 ++ ...teConfigurationRegistryClientServiceProvider | 19 + ...efaultRemoteConfigurationRegistriesTest.java | 184 ++++++++ ...teConfigurationRegistryConfigParserTest.java | 108 +++++ .../util/RemoteRegistryConfigTestUtils.java | 117 +++++ ...eConfigurationRegistryClientServiceTest.java | 424 +++++++++++++++++++ ...moteConfigurationRegistryJAASConfigTest.java | 255 +++++++++++ .../hadoop/gateway/config/GatewayConfig.java | 34 ++ .../gateway/services/GatewayServices.java | 2 + .../RemoteConfigurationRegistryClient.java | 74 ++++ ...emoteConfigurationRegistryClientService.java | 28 ++ .../monitor/RemoteConfigurationMonitor.java | 24 ++ .../RemoteConfigurationMonitorProvider.java | 34 ++ .../hadoop/gateway/GatewayTestConfig.java | 26 ++ .../java/org/apache/hadoop/test/TestUtils.java | 2 +- gateway-test/pom.xml | 6 + .../monitor/RemoteConfigurationMonitorTest.java | 397 +++++++++++++++++ pom.xml | 18 +- 49 files changed, 3918 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml index 0c05625..0a43584 100644 --- a/gateway-server/pom.xml +++ b/gateway-server/pom.xml @@ -188,6 +188,10 @@ <artifactId>gateway-server-xforwarded-filter</artifactId> </dependency> <dependency> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-service-remoteconfig</artifactId> + </dependency> + <dependency> <groupId>net.lingala.zip4j</groupId> <artifactId>zip4j</artifactId> </dependency> @@ -316,6 +320,11 @@ <artifactId>metrics-servlets</artifactId> </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java index 4cb4c40..d78ef71 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java @@ -514,19 +514,57 @@ public interface GatewayMessages { void topologyPortMappingCannotFindTopology(final String topology, final int port); + @Message( level = MessageLevel.WARN, text = "There is no registry client defined for remote configuration monitoring." ) + void missingClientConfigurationForRemoteMonitoring(); + + @Message( level = MessageLevel.WARN, text = "Could not resolve a remote configuration registry client for {0}." ) + void unresolvedClientConfigurationForRemoteMonitoring(final String clientName); + @Message( level = MessageLevel.INFO, text = "Monitoring simple descriptors in directory: {0}" ) void monitoringDescriptorChangesInDirectory(String descriptorsDir); - @Message( level = MessageLevel.INFO, text = "Monitoring shared provider configurations in directory: {0}" ) void monitoringProviderConfigChangesInDirectory(String sharedProviderDir); + @Message( level = MessageLevel.ERROR, text = "Error registering listener for remote configuration path {0} : {1}" ) + void errorAddingRemoteConfigurationListenerForPath(final String path, + @StackTrace( level = MessageLevel.DEBUG ) Exception e); + + @Message( level = MessageLevel.ERROR, text = "Error unregistering listener for remote configuration path {0} : {1}" ) + void errorRemovingRemoteConfigurationListenerForPath(final String path, + @StackTrace( level = MessageLevel.DEBUG ) Exception e); + + @Message( level = MessageLevel.ERROR, text = "Error downloading remote configuration {0} : {1}" ) + void errorDownloadingRemoteConfiguration(final String path, + @StackTrace( level = MessageLevel.DEBUG ) Exception e); + @Message( level = MessageLevel.INFO, text = "Prevented deletion of shared provider configuration because there are referencing descriptors: {0}" ) void preventedDeletionOfSharedProviderConfiguration(String providerConfigurationPath); @Message( level = MessageLevel.INFO, text = "Generated topology {0} because the associated descriptor {1} changed." ) void generatedTopologyForDescriptorChange(String topologyName, String descriptorName); + @Message( level = MessageLevel.WARN, text = "An error occurred while attempting to initialize the remote configuration monitor: {0}" ) + void remoteConfigurationMonitorInitFailure(final String errorMessage, + @StackTrace( level = MessageLevel.DEBUG ) Exception e ); + + @Message( level = MessageLevel.WARN, text = "An error occurred while attempting to start the remote configuration monitor {0} : {1}" ) + void remoteConfigurationMonitorStartFailure(final String monitorType, + final String errorMessage, + @StackTrace( level = MessageLevel.DEBUG ) Exception e ); + + @Message( level = MessageLevel.INFO, text = "Starting remote configuration monitor for source {0} ..." ) + void startingRemoteConfigurationMonitor(final String address); + + @Message( level = MessageLevel.INFO, text = "Monitoring remote configuration source {0}" ) + void monitoringRemoteConfigurationSource(final String address); + + @Message( level = MessageLevel.INFO, text = "Remote configuration monitor downloaded {0} configuration file {1}" ) + void downloadedRemoteConfigFile(final String type, final String configFileName); + + @Message( level = MessageLevel.INFO, text = "Remote configuration monitor deleted {0} configuration file {1} based on remote change." ) + void deletedRemoteConfigFile(final String type, final String configFileName); + @Message( level = MessageLevel.ERROR, text = "An error occurred while processing {0} : {1}" ) void simpleDescriptorHandlingError(final String simpleDesc, @StackTrace(level = MessageLevel.DEBUG) Exception e); http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java index 4202a18..17c2552 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImpl.java @@ -184,6 +184,8 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final String DEFAULT_DEPLOYMENT_DIR = "deployments"; public static final String DEFAULT_SECURITY_DIR = "security"; public static final String DEFAULT_DATA_DIR = "data"; + private static final String PROVIDERCONFIG_DIR_NAME = "shared-providers"; + private static final String DESCRIPTORS_DIR_NAME = "descriptors"; /* Websocket defaults */ public static final boolean DEFAULT_WEBSOCKET_FEATURE_ENABLED = false; @@ -214,6 +216,10 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { private static final String CRYPTO_KEY_LENGTH = GATEWAY_CONFIG_FILE_PREFIX + ".crypto.key.length"; public static final String SERVER_HEADER_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".server.header.enabled"; + /* @since 0.15 Remote configuration monitoring */ + static final String CONFIG_REGISTRY_PREFIX = GATEWAY_CONFIG_FILE_PREFIX + ".remote.config.registry"; + static final String REMOTE_CONFIG_MONITOR_CLIENT_NAME = GATEWAY_CONFIG_FILE_PREFIX + ".remote.config.monitor.client"; + private static List<String> DEFAULT_GLOBAL_RULES_SERVICES; @@ -264,7 +270,7 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { } else { dataDir = get(DATA_DIR, getGatewayHomeDir() + File.separator + DEFAULT_DATA_DIR); } - return dataDir; + return FilenameUtils.normalize(dataDir); } @Override @@ -412,6 +418,16 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { } @Override + public String getGatewayProvidersConfigDir() { + return getGatewayConfDir() + File.separator + PROVIDERCONFIG_DIR_NAME; + } + + @Override + public String getGatewayDescriptorsDir() { + return getGatewayConfDir() + File.separator + DESCRIPTORS_DIR_NAME; + } + + @Override public String getGatewayTopologyDir() { return getGatewayConfDir() + File.separator + "topologies"; } @@ -923,4 +939,33 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public boolean isGatewayServerHeaderEnabled() { return Boolean.parseBoolean(getVar(SERVER_HEADER_ENABLED, "true")); } + + @Override + public List<String> getRemoteRegistryConfigurationNames() { + List<String> result = new ArrayList<>(); + + // Iterate over all the properties in this configuration + for (Map.Entry<String, String> entry : this) { + String propertyName = entry.getKey(); + + // Search for all the remote config registry properties + if (propertyName.startsWith(CONFIG_REGISTRY_PREFIX)) { + String registryName = propertyName.substring(CONFIG_REGISTRY_PREFIX.length() + 1); + result.add(registryName); + } + } + + return result; + } + + @Override + public String getRemoteRegistryConfiguration(String name) { + return get(CONFIG_REGISTRY_PREFIX + "." + name ); + } + + @Override + public String getRemoteConfigurationMonitorClientName() { + return get(REMOTE_CONFIG_MONITOR_CLIENT_NAME); + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/services/CLIGatewayServices.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/CLIGatewayServices.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/CLIGatewayServices.java index 114aa83..74dc4d3 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/CLIGatewayServices.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/CLIGatewayServices.java @@ -23,6 +23,8 @@ import org.apache.hadoop.gateway.deploy.DeploymentContext; import org.apache.hadoop.gateway.descriptor.FilterParamDescriptor; import org.apache.hadoop.gateway.descriptor.ResourceDescriptor; import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; +import org.apache.hadoop.gateway.service.config.remote.RemoteConfigurationRegistryClientServiceFactory; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; import org.apache.hadoop.gateway.services.topology.impl.DefaultTopologyService; import org.apache.hadoop.gateway.services.security.impl.DefaultAliasService; import org.apache.hadoop.gateway.services.security.impl.DefaultCryptoService; @@ -71,6 +73,12 @@ public class CLIGatewayServices implements GatewayServices { DefaultTopologyService tops = new DefaultTopologyService(); tops.init( config, options ); services.put(TOPOLOGY_SERVICE, tops); + + RemoteConfigurationRegistryClientService registryClientService = + RemoteConfigurationRegistryClientServiceFactory.newInstance(config); + registryClientService.setAliasService(alias); + registryClientService.init(config, options); + services.put(REMOTE_REGISTRY_CLIENT_SERVICE, registryClientService); } public void start() throws ServiceLifecycleException { @@ -83,6 +91,8 @@ public class CLIGatewayServices implements GatewayServices { DefaultTopologyService tops = (DefaultTopologyService)services.get(TOPOLOGY_SERVICE); tops.start(); + + (services.get(REMOTE_REGISTRY_CLIENT_SERVICE)).start(); } public void stop() throws ServiceLifecycleException { http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java index 02ac154..9dca344 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java @@ -23,6 +23,8 @@ import org.apache.hadoop.gateway.deploy.DeploymentContext; import org.apache.hadoop.gateway.descriptor.FilterParamDescriptor; import org.apache.hadoop.gateway.descriptor.ResourceDescriptor; import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; +import org.apache.hadoop.gateway.service.config.remote.RemoteConfigurationRegistryClientServiceFactory; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; import org.apache.hadoop.gateway.services.registry.impl.DefaultServiceDefinitionRegistry; import org.apache.hadoop.gateway.services.metrics.impl.DefaultMetricsService; import org.apache.hadoop.gateway.services.topology.impl.DefaultTopologyService; @@ -104,6 +106,12 @@ public class DefaultGatewayServices implements GatewayServices { sis.init( config, options ); services.put( SERVER_INFO_SERVICE, sis ); + RemoteConfigurationRegistryClientService registryClientService = + RemoteConfigurationRegistryClientServiceFactory.newInstance(config); + registryClientService.setAliasService(alias); + registryClientService.init(config, options); + services.put(REMOTE_REGISTRY_CLIENT_SERVICE, registryClientService); + DefaultTopologyService tops = new DefaultTopologyService(); tops.setAliasService(alias); tops.init( config, options ); @@ -117,7 +125,7 @@ public class DefaultGatewayServices implements GatewayServices { metricsService.init( config, options ); services.put( METRICS_SERVICE, metricsService ); } - + public void start() throws ServiceLifecycleException { ms.start(); @@ -132,6 +140,10 @@ public class DefaultGatewayServices implements GatewayServices { ServerInfoService sis = (ServerInfoService) services.get(SERVER_INFO_SERVICE); sis.start(); + RemoteConfigurationRegistryClientService clientService = + (RemoteConfigurationRegistryClientService)services.get(REMOTE_REGISTRY_CLIENT_SERVICE); + clientService.start(); + DefaultTopologyService tops = (DefaultTopologyService)services.get(TOPOLOGY_SERVICE); tops.start(); http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java index 39e8029..5fc3620 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java @@ -46,6 +46,8 @@ import org.apache.hadoop.gateway.topology.TopologyListener; import org.apache.hadoop.gateway.topology.TopologyMonitor; import org.apache.hadoop.gateway.topology.TopologyProvider; import org.apache.hadoop.gateway.topology.builder.TopologyBuilder; +import org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitor; +import org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitorFactory; import org.apache.hadoop.gateway.topology.simple.SimpleDescriptorHandler; import org.apache.hadoop.gateway.topology.validation.TopologyValidator; import org.apache.hadoop.gateway.topology.xml.AmbariFormatXmlTopologyRules; @@ -101,6 +103,7 @@ public class DefaultTopologyService private volatile Map<File, Topology> topologies; private AliasService aliasService; + private RemoteConfigurationMonitor remoteMonitor = null; private Topology loadTopology(File file) throws IOException, SAXException, URISyntaxException, InterruptedException { final long TIMEOUT = 250; //ms @@ -214,6 +217,16 @@ public class DefaultTopologyService return events; } + private File calculateAbsoluteProvidersConfigDir(GatewayConfig config) { + File pcDir = new File(config.getGatewayProvidersConfigDir()); + return pcDir.getAbsoluteFile(); + } + + private File calculateAbsoluteDescriptorsDir(GatewayConfig config) { + File descDir = new File(config.getGatewayDescriptorsDir()); + return descDir.getAbsoluteFile(); + } + private File calculateAbsoluteTopologiesDir(GatewayConfig config) { File topoDir = new File(config.getGatewayTopologyDir()); topoDir = topoDir.getAbsoluteFile(); @@ -221,7 +234,7 @@ public class DefaultTopologyService } private File calculateAbsoluteConfigDir(GatewayConfig config) { - File configDir = null; + File configDir; String path = config.getGatewayConfDir(); configDir = (path != null) ? new File(path) : (new File(config.getGatewayTopologyDir())).getParentFile(); @@ -468,16 +481,32 @@ public class DefaultTopologyService @Override public void startMonitor() throws Exception { + // Start the local configuration monitors for (FileAlterationMonitor monitor : monitors) { monitor.start(); } + + // Start the remote configuration monitor, if it has been initialized + if (remoteMonitor != null) { + try { + remoteMonitor.start(); + } catch (Exception e) { + log.remoteConfigurationMonitorStartFailure(remoteMonitor.getClass().getTypeName(), e.getLocalizedMessage(), e); + } + } } @Override public void stopMonitor() throws Exception { + // Stop the local configuration monitors for (FileAlterationMonitor monitor : monitors) { monitor.stop(); } + + // Stop the remote configuration monitor, if it has been initialized + if (remoteMonitor != null) { + remoteMonitor.stop(); + } } @Override @@ -532,7 +561,7 @@ public class DefaultTopologyService public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException { try { - listeners = new HashSet<>(); + listeners = new HashSet<>(); topologies = new HashMap<>(); topologiesDirectory = calculateAbsoluteTopologiesDir(config); @@ -567,6 +596,9 @@ public class DefaultTopologyService } } + // Initialize the remote configuration monitor, if it has been configured + remoteMonitor = RemoteConfigurationMonitorFactory.get(config); + } catch (IOException | SAXException io) { throw new ServiceLifecycleException(io.getMessage()); } @@ -582,7 +614,7 @@ public class DefaultTopologyService * @return A List of the Files on the directory. */ private static List<File> listFiles(File directory) { - List<File> result = null; + List<File> result; File[] files = directory.listFiles(); if (files != null) { result = Arrays.asList(files); http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java new file mode 100644 index 0000000..7b34e3d --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultConfigurationMonitorProvider.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.monitor; + +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; + + +public class DefaultConfigurationMonitorProvider implements RemoteConfigurationMonitorProvider { + + @Override + public RemoteConfigurationMonitor newInstance(final GatewayConfig config, + final RemoteConfigurationRegistryClientService clientService) { + return new DefaultRemoteConfigurationMonitor(config, clientService); + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java new file mode 100644 index 0000000..1dd71ac --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.monitor; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.gateway.GatewayMessages; +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClient.ChildEntryListener; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClient.EntryListener; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClient; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; + +import java.io.File; +import java.io.IOException; +import java.util.List; + + +class DefaultRemoteConfigurationMonitor implements RemoteConfigurationMonitor { + + private static final String NODE_KNOX = "/knox"; + private static final String NODE_KNOX_CONFIG = NODE_KNOX + "/config"; + private static final String NODE_KNOX_PROVIDERS = NODE_KNOX_CONFIG + "/shared-providers"; + private static final String NODE_KNOX_DESCRIPTORS = NODE_KNOX_CONFIG + "/descriptors"; + + private static GatewayMessages log = MessagesFactory.get(GatewayMessages.class); + + private RemoteConfigurationRegistryClient client = null; + + private File providersDir; + private File descriptorsDir; + + /** + * @param config The gateway configuration + * @param registryClientService The service from which the remote registry client should be acquired. + */ + DefaultRemoteConfigurationMonitor(GatewayConfig config, + RemoteConfigurationRegistryClientService registryClientService) { + this.providersDir = new File(config.getGatewayProvidersConfigDir()); + this.descriptorsDir = new File(config.getGatewayDescriptorsDir()); + + if (registryClientService != null) { + String clientName = config.getRemoteConfigurationMonitorClientName(); + if (clientName != null) { + this.client = registryClientService.get(clientName); + if (this.client == null) { + log.unresolvedClientConfigurationForRemoteMonitoring(clientName); + } + } else { + log.missingClientConfigurationForRemoteMonitoring(); + } + } + } + + @Override + public void start() throws Exception { + if (client == null) { + throw new IllegalStateException("Failed to acquire a remote configuration registry client."); + } + + final String monitorSource = client.getAddress(); + log.startingRemoteConfigurationMonitor(monitorSource); + + // Confirm access to the remote provider configs directory znode + List<String> providerConfigs = client.listChildEntries(NODE_KNOX_PROVIDERS); + if (providerConfigs == null) { + // Either the ZNode does not exist, or there is an authentication problem + throw new IllegalStateException("Unable to access remote path: " + NODE_KNOX_PROVIDERS); + } + + // Confirm access to the remote descriptors directory znode + List<String> descriptors = client.listChildEntries(NODE_KNOX_DESCRIPTORS); + if (descriptors == null) { + // Either the ZNode does not exist, or there is an authentication problem + throw new IllegalStateException("Unable to access remote path: " + NODE_KNOX_DESCRIPTORS); + } + + // Register a listener for provider config znode additions/removals + client.addChildEntryListener(NODE_KNOX_PROVIDERS, new ConfigDirChildEntryListener(providersDir)); + + // Register a listener for descriptor znode additions/removals + client.addChildEntryListener(NODE_KNOX_DESCRIPTORS, new ConfigDirChildEntryListener(descriptorsDir)); + + log.monitoringRemoteConfigurationSource(monitorSource); + } + + + @Override + public void stop() throws Exception { + } + + + private static class ConfigDirChildEntryListener implements ChildEntryListener { + File localDir; + + ConfigDirChildEntryListener(File localDir) { + this.localDir = localDir; + } + + @Override + public void childEvent(RemoteConfigurationRegistryClient client, Type type, String path) { + File localFile = new File(localDir, path.substring(path.lastIndexOf("/") + 1)); + + switch (type) { + case REMOVED: + FileUtils.deleteQuietly(localFile); + log.deletedRemoteConfigFile(localDir.getName(), localFile.getName()); + try { + client.removeEntryListener(path); + } catch (Exception e) { + log.errorRemovingRemoteConfigurationListenerForPath(path, e); + } + break; + case ADDED: + try { + client.addEntryListener(path, new ConfigEntryListener(localDir)); + } catch (Exception e) { + log.errorAddingRemoteConfigurationListenerForPath(path, e); + } + break; + } + } + } + + private static class ConfigEntryListener implements EntryListener { + private File localDir; + + ConfigEntryListener(File localDir) { + this.localDir = localDir; + } + + @Override + public void entryChanged(RemoteConfigurationRegistryClient client, String path, byte[] data) { + File localFile = new File(localDir, path.substring(path.lastIndexOf("/"))); + if (data != null) { + try { + FileUtils.writeByteArrayToFile(localFile, data); + log.downloadedRemoteConfigFile(localDir.getName(), localFile.getName()); + } catch (IOException e) { + log.errorDownloadingRemoteConfiguration(path, e); + } + } else { + FileUtils.deleteQuietly(localFile); + log.deletedRemoteConfigFile(localDir.getName(), localFile.getName()); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java new file mode 100644 index 0000000..4d2df45 --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/RemoteConfigurationMonitorFactory.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.monitor; + +import org.apache.hadoop.gateway.GatewayMessages; +import org.apache.hadoop.gateway.GatewayServer; +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; +import org.apache.hadoop.gateway.services.GatewayServices; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; + +import java.util.ServiceLoader; + +public class RemoteConfigurationMonitorFactory { + + private static final GatewayMessages log = MessagesFactory.get(GatewayMessages.class); + + private static RemoteConfigurationRegistryClientService remoteConfigRegistryClientService = null; + + public static void setClientService(RemoteConfigurationRegistryClientService clientService) { + remoteConfigRegistryClientService = clientService; + } + + private static RemoteConfigurationRegistryClientService getClientService() { + if (remoteConfigRegistryClientService == null) { + GatewayServices services = GatewayServer.getGatewayServices(); + if (services != null) { + remoteConfigRegistryClientService = services.getService(GatewayServices.REMOTE_REGISTRY_CLIENT_SERVICE); + } + } + + return remoteConfigRegistryClientService; + } + + /** + * + * @param config The GatewayConfig + * + * @return The first RemoteConfigurationMonitor extension that is found. + */ + public static RemoteConfigurationMonitor get(GatewayConfig config) { + RemoteConfigurationMonitor rcm = null; + + ServiceLoader<RemoteConfigurationMonitorProvider> providers = + ServiceLoader.load(RemoteConfigurationMonitorProvider.class); + for (RemoteConfigurationMonitorProvider provider : providers) { + try { + rcm = provider.newInstance(config, getClientService()); + if (rcm != null) { + break; + } + } catch (Exception e) { + log.remoteConfigurationMonitorInitFailure(e.getLocalizedMessage(), e); + } + } + + return rcm; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java index 089925d..d1dc11d 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorHandler.java @@ -185,6 +185,7 @@ public class SimpleDescriptorHandler { // Write the service declarations for (String serviceName : serviceNames) { + fw.write("\n"); fw.write(" <service>\n"); fw.write(" <role>" + serviceName + "</role>\n"); http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitorProvider ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitorProvider b/gateway-server/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitorProvider new file mode 100644 index 0000000..bd4023e --- /dev/null +++ b/gateway-server/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.monitor.RemoteConfigurationMonitorProvider @@ -0,0 +1,19 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +org.apache.hadoop.gateway.topology.monitor.DefaultConfigurationMonitorProvider http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImplTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImplTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImplTest.java index a9347f4..cd56f11 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImplTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/config/impl/GatewayConfigImplTest.java @@ -11,6 +11,10 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertTrue; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; /** * Licensed to the Apache Software Foundation (ASF) under one @@ -217,4 +221,28 @@ public class GatewayConfigImplTest { assertThat( serverHeaderEnabled, is(false)); } + + @Test + public void testGetRemoteConfigurationRegistryNames() { + GatewayConfigImpl config = new GatewayConfigImpl(); + + List<String> registryNames = config.getRemoteRegistryConfigurationNames(); + assertNotNull(registryNames); + assertTrue(registryNames.isEmpty()); + + config.set(GatewayConfigImpl.CONFIG_REGISTRY_PREFIX + ".test1", + "type=ZooKeeper;address=host1:2181;authType=digest;principal=itsme;credentialAlias=testAlias"); + registryNames = config.getRemoteRegistryConfigurationNames(); + assertNotNull(registryNames); + assertFalse(registryNames.isEmpty()); + assertEquals(1, registryNames.size()); + + config.set(GatewayConfigImpl.CONFIG_REGISTRY_PREFIX + ".test2", + "type=ZooKeeper;address=host2:2181,host3:2181,host4:2181"); + registryNames = config.getRemoteRegistryConfigurationNames(); + assertNotNull(registryNames); + assertFalse(registryNames.isEmpty()); + assertEquals(registryNames.size(), 2); + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/services/topology/DefaultTopologyServiceTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/services/topology/DefaultTopologyServiceTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/services/topology/DefaultTopologyServiceTest.java index 2357ad6..e4a0eba 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/services/topology/DefaultTopologyServiceTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/services/topology/DefaultTopologyServiceTest.java @@ -96,6 +96,12 @@ public class DefaultTopologyServiceTest { File dir = createDir(); File topologyDir = new File(dir, "topologies"); + File descriptorsDir = new File(dir, "descriptors"); + descriptorsDir.mkdirs(); + + File sharedProvidersDir = new File(dir, "shared-providers"); + sharedProvidersDir.mkdirs(); + long time = topologyDir.lastModified(); try { createFile(topologyDir, "one.xml", "org/apache/hadoop/gateway/topology/file/topology-one.xml", time); @@ -108,7 +114,9 @@ public class DefaultTopologyServiceTest { GatewayConfig config = EasyMock.createNiceMock(GatewayConfig.class); EasyMock.expect(config.getGatewayTopologyDir()).andReturn(topologyDir.getAbsolutePath()).anyTimes(); - EasyMock.expect(config.getGatewayConfDir()).andReturn(topologyDir.getParentFile().getAbsolutePath()).anyTimes(); + EasyMock.expect(config.getGatewayConfDir()).andReturn(descriptorsDir.getParentFile().getAbsolutePath()).anyTimes(); + EasyMock.expect(config.getGatewayProvidersConfigDir()).andReturn(sharedProvidersDir.getAbsolutePath()).anyTimes(); + EasyMock.expect(config.getGatewayDescriptorsDir()).andReturn(descriptorsDir.getAbsolutePath()).anyTimes(); EasyMock.replay(config); provider.init(config, c); http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java new file mode 100644 index 0000000..1c4ed6e --- /dev/null +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/topology/monitor/ZooKeeperConfigurationMonitorTest.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.monitor; + +import org.apache.commons.io.FileUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.InstanceSpec; +import org.apache.curator.test.TestingCluster; +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.service.config.remote.zk.ZooKeeperClientService; +import org.apache.hadoop.gateway.service.config.remote.zk.ZooKeeperClientServiceProvider; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; +import org.apache.hadoop.gateway.services.security.AliasService; +import org.apache.hadoop.test.TestUtils; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.easymock.EasyMock; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test the ZooKeeperConfigMonitor WITHOUT SASL configured or znode ACLs applied. + * The implementation of the monitor is the same regardless, since the ACLs are defined by the ZooKeeper znode + * creator, and the SASL config is purely JAAS (and external to the implementation). + */ +public class ZooKeeperConfigurationMonitorTest { + + private static final String PATH_KNOX = "/knox"; + private static final String PATH_KNOX_CONFIG = PATH_KNOX + "/config"; + private static final String PATH_KNOX_PROVIDERS = PATH_KNOX_CONFIG + "/shared-providers"; + private static final String PATH_KNOX_DESCRIPTORS = PATH_KNOX_CONFIG + "/descriptors"; + + private static File testTmp; + private static File providersDir; + private static File descriptorsDir; + + private static TestingCluster zkCluster; + + private static CuratorFramework client; + + private GatewayConfig gc; + + + @BeforeClass + public static void setupSuite() throws Exception { + testTmp = TestUtils.createTempDir(ZooKeeperConfigurationMonitorTest.class.getName()); + File confDir = TestUtils.createTempDir(testTmp + "/conf"); + providersDir = TestUtils.createTempDir(confDir + "/shared-providers"); + descriptorsDir = TestUtils.createTempDir(confDir + "/descriptors"); + + configureAndStartZKCluster(); + } + + private static void configureAndStartZKCluster() throws Exception { + // Configure security for the ZK cluster instances + Map<String, Object> customInstanceSpecProps = new HashMap<>(); + customInstanceSpecProps.put("authProvider.1", "org.apache.zookeeper.server.auth.SASLAuthenticationProvider"); + customInstanceSpecProps.put("requireClientAuthScheme", "sasl"); + + // Define the test cluster + List<InstanceSpec> instanceSpecs = new ArrayList<>(); + for (int i = 0 ; i < 3 ; i++) { + InstanceSpec is = new InstanceSpec(null, -1, -1, -1, false, (i+1), -1, -1, customInstanceSpecProps); + instanceSpecs.add(is); + } + zkCluster = new TestingCluster(instanceSpecs); + + // Start the cluster + zkCluster.start(); + + // Create the client for the test cluster + client = CuratorFrameworkFactory.builder() + .connectString(zkCluster.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(100, 3)) + .build(); + assertNotNull(client); + client.start(); + + // Create the knox config paths with an ACL for the sasl user configured for the client + List<ACL> acls = new ArrayList<>(); + acls.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.ANYONE_ID_UNSAFE)); + + client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(acls).forPath(PATH_KNOX_DESCRIPTORS); + assertNotNull("Failed to create node:" + PATH_KNOX_DESCRIPTORS, + client.checkExists().forPath(PATH_KNOX_DESCRIPTORS)); + client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).withACL(acls).forPath(PATH_KNOX_PROVIDERS); + assertNotNull("Failed to create node:" + PATH_KNOX_PROVIDERS, + client.checkExists().forPath(PATH_KNOX_PROVIDERS)); + } + + @AfterClass + public static void tearDownSuite() throws Exception { + // Clean up the ZK nodes, and close the client + if (client != null) { + client.delete().deletingChildrenIfNeeded().forPath(PATH_KNOX); + client.close(); + } + + // Shutdown the ZK cluster + zkCluster.close(); + + // Delete the working dir + testTmp.delete(); + } + + @Test + public void testZooKeeperConfigMonitor() throws Exception { + String configMonitorName = "remoteConfigMonitorClient"; + + // Setup the base GatewayConfig mock + gc = EasyMock.createNiceMock(GatewayConfig.class); + EasyMock.expect(gc.getGatewayProvidersConfigDir()).andReturn(providersDir.getAbsolutePath()).anyTimes(); + EasyMock.expect(gc.getGatewayDescriptorsDir()).andReturn(descriptorsDir.getAbsolutePath()).anyTimes(); + EasyMock.expect(gc.getRemoteRegistryConfigurationNames()) + .andReturn(Collections.singletonList(configMonitorName)) + .anyTimes(); + final String registryConfig = + GatewayConfig.REMOTE_CONFIG_REGISTRY_TYPE + "=" + ZooKeeperClientService.TYPE + ";" + + GatewayConfig.REMOTE_CONFIG_REGISTRY_ADDRESS + "=" + zkCluster.getConnectString(); + EasyMock.expect(gc.getRemoteRegistryConfiguration(configMonitorName)) + .andReturn(registryConfig) + .anyTimes(); + EasyMock.expect(gc.getRemoteConfigurationMonitorClientName()).andReturn(configMonitorName).anyTimes(); + EasyMock.replay(gc); + + AliasService aliasService = EasyMock.createNiceMock(AliasService.class); + EasyMock.replay(aliasService); + + RemoteConfigurationRegistryClientService clientService = (new ZooKeeperClientServiceProvider()).newInstance(); + clientService.setAliasService(aliasService); + clientService.init(gc, Collections.emptyMap()); + clientService.start(); + + DefaultRemoteConfigurationMonitor cm = new DefaultRemoteConfigurationMonitor(gc, clientService); + + try { + cm.start(); + } catch (Exception e) { + fail("Failed to start monitor: " + e.getMessage()); + } + + try { + final String pc_one_znode = getProviderPath("providers-config1.xml"); + final File pc_one = new File(providersDir, "providers-config1.xml"); + final String pc_two_znode = getProviderPath("providers-config2.xml"); + final File pc_two = new File(providersDir, "providers-config2.xml"); + + client.create().withMode(CreateMode.PERSISTENT).forPath(pc_one_znode, TEST_PROVIDERS_CONFIG_1.getBytes()); + Thread.sleep(100); + assertTrue(pc_one.exists()); + assertEquals(TEST_PROVIDERS_CONFIG_1, FileUtils.readFileToString(pc_one)); + + client.create().withMode(CreateMode.PERSISTENT).forPath(getProviderPath("providers-config2.xml"), TEST_PROVIDERS_CONFIG_2.getBytes()); + Thread.sleep(100); + assertTrue(pc_two.exists()); + assertEquals(TEST_PROVIDERS_CONFIG_2, FileUtils.readFileToString(pc_two)); + + client.setData().forPath(pc_two_znode, TEST_PROVIDERS_CONFIG_1.getBytes()); + Thread.sleep(100); + assertTrue(pc_two.exists()); + assertEquals(TEST_PROVIDERS_CONFIG_1, FileUtils.readFileToString(pc_two)); + + client.delete().forPath(pc_two_znode); + Thread.sleep(100); + assertFalse(pc_two.exists()); + + client.delete().forPath(pc_one_znode); + Thread.sleep(100); + assertFalse(pc_one.exists()); + + final String desc_one_znode = getDescriptorPath("test1.json"); + final String desc_two_znode = getDescriptorPath("test2.json"); + final String desc_three_znode = getDescriptorPath("test3.json"); + final File desc_one = new File(descriptorsDir, "test1.json"); + final File desc_two = new File(descriptorsDir, "test2.json"); + final File desc_three = new File(descriptorsDir, "test3.json"); + + client.create().withMode(CreateMode.PERSISTENT).forPath(desc_one_znode, TEST_DESCRIPTOR_1.getBytes()); + Thread.sleep(100); + assertTrue(desc_one.exists()); + assertEquals(TEST_DESCRIPTOR_1, FileUtils.readFileToString(desc_one)); + + client.create().withMode(CreateMode.PERSISTENT).forPath(desc_two_znode, TEST_DESCRIPTOR_1.getBytes()); + Thread.sleep(100); + assertTrue(desc_two.exists()); + assertEquals(TEST_DESCRIPTOR_1, FileUtils.readFileToString(desc_two)); + + client.setData().forPath(desc_two_znode, TEST_DESCRIPTOR_2.getBytes()); + Thread.sleep(100); + assertTrue(desc_two.exists()); + assertEquals(TEST_DESCRIPTOR_2, FileUtils.readFileToString(desc_two)); + + client.create().withMode(CreateMode.PERSISTENT).forPath(desc_three_znode, TEST_DESCRIPTOR_1.getBytes()); + Thread.sleep(100); + assertTrue(desc_three.exists()); + assertEquals(TEST_DESCRIPTOR_1, FileUtils.readFileToString(desc_three)); + + client.delete().forPath(desc_two_znode); + Thread.sleep(100); + assertFalse("Expected test2.json to have been deleted.", desc_two.exists()); + + client.delete().forPath(desc_three_znode); + Thread.sleep(100); + assertFalse(desc_three.exists()); + + client.delete().forPath(desc_one_znode); + Thread.sleep(100); + assertFalse(desc_one.exists()); + } finally { + cm.stop(); + } + } + + private static String getDescriptorPath(String descriptorName) { + return PATH_KNOX_DESCRIPTORS + "/" + descriptorName; + } + + private static String getProviderPath(String providerConfigName) { + return PATH_KNOX_PROVIDERS + "/" + providerConfigName; + } + + + private static final String TEST_PROVIDERS_CONFIG_1 = + "<gateway>\n" + + " <provider>\n" + + " <role>identity-assertion</role>\n" + + " <name>Default</name>\n" + + " <enabled>true</enabled>\n" + + " </provider>\n" + + " <provider>\n" + + " <role>hostmap</role>\n" + + " <name>static</name>\n" + + " <enabled>true</enabled>\n" + + " <param><name>localhost</name><value>sandbox,sandbox.hortonworks.com</value></param>\n" + + " </provider>\n" + + "</gateway>\n"; + + private static final String TEST_PROVIDERS_CONFIG_2 = + "<gateway>\n" + + " <provider>\n" + + " <role>authentication</role>\n" + + " <name>ShiroProvider</name>\n" + + " <enabled>true</enabled>\n" + + " <param>\n" + + " <name>sessionTimeout</name>\n" + + " <value>30</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>main.ldapRealm</name>\n" + + " <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>main.ldapContextFactory</name>\n" + + " <value>org.apache.hadoop.gateway.shirorealm.KnoxLdapContextFactory</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>main.ldapRealm.contextFactory</name>\n" + + " <value>$ldapContextFactory</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>main.ldapRealm.userDnTemplate</name>\n" + + " <value>uid={0},ou=people,dc=hadoop,dc=apache,dc=org</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>main.ldapRealm.contextFactory.url</name>\n" + + " <value>ldap://localhost:33389</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>main.ldapRealm.contextFactory.authenticationMechanism</name>\n" + + " <value>simple</value>\n" + + " </param>\n" + + " <param>\n" + + " <name>urls./**</name>\n" + + " <value>authcBasic</value>\n" + + " </param>\n" + + " </provider>\n" + + "</gateway>\n"; + + private static final String TEST_DESCRIPTOR_1 = + "{\n" + + " \"discovery-type\":\"AMBARI\",\n" + + " \"discovery-address\":\"http://sandbox.hortonworks.com:8080\",\n"; + + " \"discovery-user\":\"maria_dev\",\n" + + " \"discovery-pwd-alias\":\"sandbox.ambari.discovery.password\",\n" + + " \"provider-config-ref\":\"sandbox-providers.xml\",\n" + + " \"cluster\":\"Sandbox\",\n" + + " \"services\":[\n" + + " {\"name\":\"NODEUI\"},\n" + + " {\"name\":\"YARNUI\"},\n" + + " {\"name\":\"HDFSUI\"},\n" + + " {\"name\":\"OOZIEUI\"},\n" + + " {\"name\":\"HBASEUI\"},\n" + + " {\"name\":\"NAMENODE\"},\n" + + " {\"name\":\"JOBTRACKER\"},\n" + + " {\"name\":\"WEBHDFS\"},\n" + + " {\"name\":\"WEBHCAT\"},\n" + + " {\"name\":\"OOZIE\"},\n" + + " {\"name\":\"WEBHBASE\"},\n" + + " {\"name\":\"RESOURCEMANAGER\"},\n" + + " {\"name\":\"AMBARI\", \"urls\":[\"http://c6401.ambari.apache.org:8080\"]},\n"; + + " {\"name\":\"AMBARIUI\", \"urls\":[\"http://c6401.ambari.apache.org:8080\"]}\n"; + + " ]\n" + + "}\n"; + + private static final String TEST_DESCRIPTOR_2 = + "{\n" + + " \"discovery-type\":\"AMBARI\",\n" + + " \"discovery-address\":\"http://sandbox.hortonworks.com:8080\",\n"; + + " \"discovery-user\":\"maria_dev\",\n" + + " \"discovery-pwd-alias\":\"sandbox.ambari.discovery.password\",\n" + + " \"provider-config-ref\":\"sandbox-providers.xml\",\n" + + " \"cluster\":\"Sandbox\",\n" + + " \"services\":[\n" + + " {\"name\":\"NAMENODE\"},\n" + + " {\"name\":\"JOBTRACKER\"},\n" + + " {\"name\":\"WEBHDFS\"},\n" + + " {\"name\":\"WEBHCAT\"},\n" + + " {\"name\":\"OOZIE\"},\n" + + " {\"name\":\"WEBHBASE\"},\n" + + " {\"name\":\"RESOURCEMANAGER\"}\n" + + " ]\n" + + "}\n"; + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java index 0352fa3..838f114 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java @@ -20,8 +20,11 @@ package org.apache.hadoop.gateway.util; import com.mycila.xmltool.XMLDoc; import com.mycila.xmltool.XMLTag; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.gateway.config.impl.GatewayConfigImpl; import org.apache.hadoop.gateway.services.GatewayServices; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClient; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; import org.apache.hadoop.gateway.services.security.AliasService; import org.apache.hadoop.gateway.services.security.MasterService; import org.junit.Before; @@ -42,6 +45,7 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -60,7 +64,27 @@ public class KnoxCLITest { } @Test - public void testSuccessfulAlaisLifecycle() throws Exception { + public void testRemoteConfigurationRegistryClientService() throws Exception { + outContent.reset(); + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + config.set("gateway.remote.config.registry.test_client", "type=ZooKeeper;address=localhost:2181"); + cli.setConf(config); + + // This is only to get the gateway services initialized + cli.run(new String[]{"version"}); + + RemoteConfigurationRegistryClientService service = + cli.getGatewayServices().getService(GatewayServices.REMOTE_REGISTRY_CLIENT_SERVICE); + assertNotNull(service); + RemoteConfigurationRegistryClient client = service.get("test_client"); + assertNotNull(client); + + assertNull(service.get("bogus")); + } + + @Test + public void testSuccessfulAliasLifecycle() throws Exception { outContent.reset(); String[] args1 = {"create-alias", "alias1", "--value", "testvalue1", "--master", "master"}; int rc = 0; http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/BadUrlTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/BadUrlTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/BadUrlTest.java index 559b2a1..c465585 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/BadUrlTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/BadUrlTest.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.URI; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -205,6 +206,12 @@ public class BadUrlTest { EasyMock.expect(gatewayConfig.getGatewayTopologyDir()) .andReturn(topoDir.toString()).anyTimes(); + EasyMock.expect(gatewayConfig.getGatewayProvidersConfigDir()) + .andReturn(topoDir.getAbsolutePath() + "/shared-providers").anyTimes(); + + EasyMock.expect(gatewayConfig.getGatewayDescriptorsDir()) + .andReturn(topoDir.getAbsolutePath() + "/descriptors").anyTimes(); + EasyMock.expect(gatewayConfig.getGatewayServicesDir()) .andReturn(serviceUrl.getFile()).anyTimes(); @@ -247,6 +254,10 @@ public class BadUrlTest { EasyMock.expect(gatewayConfig.getWebsocketIdleTimeout()) .andReturn(GatewayConfigImpl.DEFAULT_WEBSOCKET_IDLE_TIMEOUT).anyTimes(); + EasyMock.expect(gatewayConfig.getRemoteRegistryConfigurationNames()) + .andReturn(Collections.emptyList()) + .anyTimes(); + EasyMock.replay(gatewayConfig); try { http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketEchoTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketEchoTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketEchoTest.java index 4b0fe08..5d5f280 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketEchoTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketEchoTest.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.URI; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -290,6 +291,12 @@ public class WebsocketEchoTest { EasyMock.expect(gatewayConfig.getGatewayTopologyDir()) .andReturn(topoDir.toString()).anyTimes(); + EasyMock.expect(gatewayConfig.getGatewayProvidersConfigDir()) + .andReturn(topoDir.getAbsolutePath() + "/shared-providers").anyTimes(); + + EasyMock.expect(gatewayConfig.getGatewayDescriptorsDir()) + .andReturn(topoDir.getAbsolutePath() + "/descriptors").anyTimes(); + EasyMock.expect(gatewayConfig.getGatewayServicesDir()) .andReturn(serviceUrl.getFile()).anyTimes(); @@ -332,6 +339,10 @@ public class WebsocketEchoTest { EasyMock.expect(gatewayConfig.getWebsocketIdleTimeout()) .andReturn(GatewayConfigImpl.DEFAULT_WEBSOCKET_IDLE_TIMEOUT).anyTimes(); + EasyMock.expect(gatewayConfig.getRemoteRegistryConfigurationNames()) + .andReturn(Collections.emptyList()) + .anyTimes(); + EasyMock.replay(gatewayConfig); try { http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketMultipleConnectionTest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketMultipleConnectionTest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketMultipleConnectionTest.java index 676c98c..7ddada2 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketMultipleConnectionTest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/websockets/WebsocketMultipleConnectionTest.java @@ -25,6 +25,7 @@ import java.lang.management.MemoryMXBean; import java.net.URI; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -283,6 +284,12 @@ public class WebsocketMultipleConnectionTest { EasyMock.expect(gatewayConfig.getGatewayTopologyDir()) .andReturn(topoDir.toString()).anyTimes(); + EasyMock.expect(gatewayConfig.getGatewayProvidersConfigDir()) + .andReturn(topoDir.getAbsolutePath() + "/shared-providers").anyTimes(); + + EasyMock.expect(gatewayConfig.getGatewayDescriptorsDir()) + .andReturn(topoDir.getAbsolutePath() + "/descriptors").anyTimes(); + EasyMock.expect(gatewayConfig.getGatewayServicesDir()) .andReturn(serviceUrl.getFile()).anyTimes(); @@ -325,6 +332,10 @@ public class WebsocketMultipleConnectionTest { EasyMock.expect(gatewayConfig.getWebsocketIdleTimeout()) .andReturn(GatewayConfigImpl.DEFAULT_WEBSOCKET_IDLE_TIMEOUT).anyTimes(); + EasyMock.expect(gatewayConfig.getRemoteRegistryConfigurationNames()) + .andReturn(Collections.emptyList()) + .anyTimes(); + EasyMock.replay(gatewayConfig); try { http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/pom.xml b/gateway-service-remoteconfig/pom.xml new file mode 100644 index 0000000..8d06360 --- /dev/null +++ b/gateway-service-remoteconfig/pom.xml @@ -0,0 +1,89 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.knox</groupId> + <artifactId>gateway</artifactId> + <version>1.0.0-SNAPSHOT</version> + </parent> + <artifactId>gateway-service-remoteconfig</artifactId> + + <name>gateway-service-remoteconfig</name> + <description>The gateway service for interacting with remote configuration registries.</description> + + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + + <dependencies> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-spi</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-client</artifactId> + </dependency> + + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-test-utils</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <scope>test</scope> + </dependency> + + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationMessages.java ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationMessages.java b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationMessages.java new file mode 100644 index 0000000..22e622d --- /dev/null +++ b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationMessages.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote; + +import org.apache.hadoop.gateway.i18n.messages.Message; +import org.apache.hadoop.gateway.i18n.messages.MessageLevel; +import org.apache.hadoop.gateway.i18n.messages.Messages; +import org.apache.hadoop.gateway.i18n.messages.StackTrace; + + +/** + * + */ +@Messages(logger="org.apache.hadoop.gateway.service.config.remote") +public interface RemoteConfigurationMessages { + + @Message(level = MessageLevel.WARN, + text = "Multiple remote configuration registries are not currently supported if any of them requires authentication.") + void multipleRemoteRegistryConfigurations(); + + @Message(level = MessageLevel.ERROR, text = "Failed to resolve the credential alias {0}") + void unresolvedCredentialAlias(final String alias); + + @Message(level = MessageLevel.ERROR, text = "An error occurred interacting with the remote configuration registry : {0}") + void errorInteractingWithRemoteConfigRegistry(@StackTrace(level = MessageLevel.DEBUG) Exception e); + + @Message(level = MessageLevel.ERROR, text = "An error occurred handling the ACL for remote configuration {0} : {1}") + void errorHandlingRemoteConfigACL(final String path, + @StackTrace(level = MessageLevel.DEBUG) Exception e); + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceFactory.java ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceFactory.java b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceFactory.java new file mode 100644 index 0000000..cd58e22 --- /dev/null +++ b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceFactory.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote; + +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; + +import java.util.ServiceLoader; + +public class RemoteConfigurationRegistryClientServiceFactory { + + public static RemoteConfigurationRegistryClientService newInstance(GatewayConfig config) { + RemoteConfigurationRegistryClientService rcs = null; + + ServiceLoader<RemoteConfigurationRegistryClientServiceProvider> providers = + ServiceLoader.load(RemoteConfigurationRegistryClientServiceProvider.class); + for (RemoteConfigurationRegistryClientServiceProvider provider : providers) { + rcs = provider.newInstance(); + if (rcs != null) { + break; + } + } + + return rcs; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceProvider.java ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceProvider.java b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceProvider.java new file mode 100644 index 0000000..ddfc392 --- /dev/null +++ b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryClientServiceProvider.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote; + +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; + +public interface RemoteConfigurationRegistryClientServiceProvider { + + String getType(); + + RemoteConfigurationRegistryClientService newInstance(); + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryConfig.java ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryConfig.java b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryConfig.java new file mode 100644 index 0000000..6409250 --- /dev/null +++ b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/RemoteConfigurationRegistryConfig.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote; + +public interface RemoteConfigurationRegistryConfig { + + String getName(); + + String getRegistryType(); + + String getConnectionString(); + + String getNamespace(); + + boolean isSecureRegistry(); + + String getAuthType(); // digest, kerberos, etc... + + String getPrincipal(); + + String getCredentialAlias(); + + String getKeytab(); + + boolean isUseTicketCache(); + + boolean isUseKeyTab(); + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/DefaultRemoteConfigurationRegistries.java ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/DefaultRemoteConfigurationRegistries.java b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/DefaultRemoteConfigurationRegistries.java new file mode 100644 index 0000000..ebcae1b --- /dev/null +++ b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/DefaultRemoteConfigurationRegistries.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote.config; + +import org.apache.hadoop.gateway.config.GatewayConfig; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A set of RemoteConfigurationRegistry configurations based on a set of property name-value pairs. + */ +class DefaultRemoteConfigurationRegistries extends RemoteConfigurationRegistries { + + private static final String PROPERTY_DELIM = ";"; + private static final String PROPERTY_VALUE_DELIM = "="; + + private List<RemoteConfigurationRegistry> configuredRegistries = new ArrayList<>(); + + /** + * Derive the remote registry configurations from the specified GatewayConfig. + * + * @param gc The source GatewayConfig + */ + DefaultRemoteConfigurationRegistries(GatewayConfig gc) { + List<String> configRegistryNames = gc.getRemoteRegistryConfigurationNames(); + for (String configRegistryName : configRegistryNames) { + configuredRegistries.add(extractConfigForRegistry(gc, configRegistryName)); + } + } + + /** + * Extract the configuration for the specified registry configuration name. + * + * @param gc The GatewayConfig from which to extract the registry config. + * @param registryName The name of the registry config. + * + * @return The resulting RemoteConfigurationRegistry object, or null. + */ + private static RemoteConfigurationRegistry extractConfigForRegistry(GatewayConfig gc, String registryName) { + RemoteConfigurationRegistry result = new RemoteConfigurationRegistry(); + + result.setName(registryName); + + Map<String, String> properties = parsePropertyValue(gc.getRemoteRegistryConfiguration(registryName)); + + result.setRegistryType(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_TYPE)); + result.setConnectionString(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_ADDRESS)); + result.setNamespace(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_NAMESPACE)); + result.setAuthType(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_AUTH_TYPE)); + result.setPrincipal(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_PRINCIPAL)); + result.setCredentialAlias(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_CREDENTIAL_ALIAS)); + result.setKeytab(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_KEYTAB)); + result.setUseKeytab(Boolean.valueOf(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_USE_KEYTAB))); + result.setUseTicketCache(Boolean.valueOf(properties.get(GatewayConfig.REMOTE_CONFIG_REGISTRY_USE_TICKET_CACHE))); + + return result; + } + + /** + * Parse the specified registry config properties String. + * + * @param value The property value content from GatewayConfig. + * + * @return A Map of the parsed properties and their respective values. + */ + private static Map<String, String> parsePropertyValue(final String value) { + Map<String, String> result = new HashMap<>(); + + if (value != null) { + String[] props = value.split(PROPERTY_DELIM); + for (String prop : props) { + String[] split = prop.split(PROPERTY_VALUE_DELIM); + String propName = split[0]; + String propValue = (split.length > 1) ? split[1] : null; + result.put(propName, propValue); + } + } + + return result; + } + + @Override + List<RemoteConfigurationRegistry> getRegistryConfigurations() { + return configuredRegistries; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/5af2413c/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/RemoteConfigurationRegistries.java ---------------------------------------------------------------------- diff --git a/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/RemoteConfigurationRegistries.java b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/RemoteConfigurationRegistries.java new file mode 100644 index 0000000..fa045c0 --- /dev/null +++ b/gateway-service-remoteconfig/src/main/java/org/apache/hadoop/gateway/service/config/remote/config/RemoteConfigurationRegistries.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote.config; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +@XmlRootElement(name="remote-configuration-registries") +class RemoteConfigurationRegistries { + + private List<RemoteConfigurationRegistry> registryConfigurations = new ArrayList<>(); + + @XmlElement(name="remote-configuration-registry") + List<RemoteConfigurationRegistry> getRegistryConfigurations() { + return registryConfigurations; + } +}
