KNOX-1125 - KNOXCLI Additions to Support Management of Knox config in remote registry (Phil Zampino via Sandeep More)
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/828ea38f Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/828ea38f Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/828ea38f Branch: refs/heads/KNOX-998-Package_Restructuring Commit: 828ea38fcfd4a4edee2813ae9d357ee0f555afc8 Parents: a09e751 Author: Sandeep More <[email protected]> Authored: Mon Dec 4 13:44:04 2017 -0500 Committer: Sandeep More <[email protected]> Committed: Mon Dec 4 13:44:04 2017 -0500 ---------------------------------------------------------------------- .../DefaultRemoteConfigurationMonitor.java | 2 + .../org/apache/hadoop/gateway/util/KnoxCLI.java | 391 ++++++++++++++++++- ...emoteConfigurationRegistryClientService.java | 243 ++++++++++++ ...figurationRegistryClientServiceProvider.java | 32 ++ .../apache/hadoop/gateway/util/KnoxCLITest.java | 361 ++++++++++++++++- ...teConfigurationRegistryClientServiceProvider | 19 + 6 files changed, 1039 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/828ea38f/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java index 1dd71ac..03bbf16 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/monitor/DefaultRemoteConfigurationMonitor.java @@ -101,6 +101,8 @@ class DefaultRemoteConfigurationMonitor implements RemoteConfigurationMonitor { @Override public void stop() throws Exception { + client.removeEntryListener(NODE_KNOX_PROVIDERS); + client.removeEntryListener(NODE_KNOX_DESCRIPTORS); } http://git-wip-us.apache.org/repos/asf/knox/blob/828ea38f/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java index afc6ee0..5576df7 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java @@ -20,7 +20,6 @@ package org.apache.hadoop.gateway.util; import java.io.BufferedReader; import java.io.Console; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -28,7 +27,6 @@ import java.io.PrintStream; import java.net.InetAddress; import java.net.UnknownHostException; import java.security.cert.Certificate; -import java.security.cert.CertificateEncodingException; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -51,6 +49,8 @@ import org.apache.hadoop.gateway.services.CLIGatewayServices; import org.apache.hadoop.gateway.services.GatewayServices; import org.apache.hadoop.gateway.services.Service; import org.apache.hadoop.gateway.services.ServiceLifecycleException; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClient; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; import org.apache.hadoop.gateway.services.security.AliasService; import org.apache.hadoop.gateway.services.security.KeystoreService; import org.apache.hadoop.gateway.services.security.KeystoreServiceException; @@ -82,6 +82,7 @@ import org.apache.shiro.util.ThreadContext; import org.eclipse.persistence.oxm.MediaType; import org.jboss.shrinkwrap.api.exporter.ExplodedExporter; import org.jboss.shrinkwrap.api.spec.EnterpriseArchive; + /** * */ @@ -102,7 +103,13 @@ public class KnoxCLI extends Configured implements Tool { " [" + ValidateTopologyCommand.USAGE + "]\n" + " [" + LDAPAuthCommand.USAGE + "]\n" + " [" + LDAPSysBindCommand.USAGE + "]\n" + - " [" + ServiceTestCommand.USAGE + "]\n"; + " [" + ServiceTestCommand.USAGE + "]\n" + + " [" + RemoteRegistryClientsListCommand.USAGE + "]\n" + + " [" + RemoteRegistryUploadProviderConfigCommand.USAGE + "]\n" + + " [" + RemoteRegistryUploadDescriptorCommand.USAGE + "]\n" + + " [" + RemoteRegistryDeleteProviderConfigCommand.USAGE + "]\n" + + " [" + RemoteRegistryDeleteDescriptorCommand.USAGE + "]\n" + + " [" + RemoteRegistryGetACLCommand.USAGE + "]\n"; /** allows stdout to be captured if necessary */ public PrintStream out = System.out; @@ -123,6 +130,9 @@ public class KnoxCLI extends Configured implements Tool { private String pass = null; private boolean groups = false; + private String remoteRegistryClient = null; + private String remoteRegistryEntryName = null; + // For testing only private String master = null; private String type = null; @@ -187,7 +197,12 @@ public class KnoxCLI extends Configured implements Tool { * % knoxcli user-auth-test [--cluster clustername] [--u username] [--p password] * % knoxcli system-user-auth-test [--cluster clustername] [--d] * % knoxcli service-test [--u user] [--p password] [--cluster clustername] [--hostname name] [--port port] - * + * % knoxcli list-registry-clients + * % knoxcli get-registry-acl entryName --registry-client name + * % knoxcli upload-provider-config filePath --registry-client name [--entry-name entryName] + * % knoxcli upload-descriptor filePath --registry-client name [--entry-name entryName] + * % knoxcli delete-provider-config providerConfig --registry-client name + * % knoxcli delete-descriptor descriptor --registry-client name * </pre> * @param args * @return @@ -282,7 +297,7 @@ public class KnoxCLI extends Configured implements Tool { } this.cluster = args[++i]; } else if (args[i].equals("service-test")) { - if( i + 1 >= args[i].length()) { + if( i + 1 >= args.length) { printKnoxShellUsage(); return -1; } else { @@ -348,6 +363,63 @@ public class KnoxCLI extends Configured implements Tool { } } else if (args[i].equals("--g")) { this.groups = true; + } else if (args[i].equals("list-registry-clients")) { + command = new RemoteRegistryClientsListCommand(); + } else if (args[i].equals("--registry-client")) { + if (i + 1 >= args.length || args[i + 1].startsWith("-")) { + printKnoxShellUsage(); + return -1; + } + this.remoteRegistryClient = args[++i]; + } else if (args[i].equalsIgnoreCase("upload-provider-config")) { + String fileName; + if (i <= (args.length - 1)) { + fileName = args[++i]; + command = new RemoteRegistryUploadProviderConfigCommand(fileName); + } else { + printKnoxShellUsage(); + return -1; + } + } else if (args[i].equals("upload-descriptor")) { + String fileName; + if (i <= (args.length - 1)) { + fileName = args[++i]; + command = new RemoteRegistryUploadDescriptorCommand(fileName); + } else { + printKnoxShellUsage(); + return -1; + } + } else if (args[i].equals("--entry-name")) { + if (i <= (args.length - 1)) { + remoteRegistryEntryName = args[++i]; + } else { + printKnoxShellUsage(); + return -1; + } + } else if (args[i].equals("delete-descriptor")) { + if (i <= (args.length - 1)) { + String entry = args[++i]; + command = new RemoteRegistryDeleteDescriptorCommand(entry); + } else { + printKnoxShellUsage(); + return -1; + } + } else if (args[i].equals("delete-provider-config")) { + if (i <= (args.length - 1)) { + String entry = args[++i]; + command = new RemoteRegistryDeleteProviderConfigCommand(entry); + } else { + printKnoxShellUsage(); + return -1; + } + } else if (args[i].equalsIgnoreCase("get-registry-acl")) { + if (i <= (args.length - 1)) { + String entry = args[++i]; + command = new RemoteRegistryGetACLCommand(entry); + } else { + printKnoxShellUsage(); + return -1; + } } else { printKnoxShellUsage(); //ToolRunner.printGenericCommandUsage(System.err); @@ -406,6 +478,24 @@ public class KnoxCLI extends Configured implements Tool { out.println(ServiceTestCommand.USAGE + "\n\n" + ServiceTestCommand.DESC); out.println(); out.println( div ); + out.println(RemoteRegistryClientsListCommand.USAGE + "\n\n" + RemoteRegistryClientsListCommand.DESC); + out.println(); + out.println( div ); + out.println(RemoteRegistryGetACLCommand.USAGE + "\n\n" + RemoteRegistryGetACLCommand.DESC); + out.println(); + out.println( div ); + out.println(RemoteRegistryUploadProviderConfigCommand.USAGE + "\n\n" + RemoteRegistryUploadProviderConfigCommand.DESC); + out.println(); + out.println( div ); + out.println(RemoteRegistryUploadDescriptorCommand.USAGE + "\n\n" + RemoteRegistryUploadDescriptorCommand.DESC); + out.println(); + out.println( div ); + out.println(RemoteRegistryDeleteProviderConfigCommand.USAGE + "\n\n" + RemoteRegistryDeleteProviderConfigCommand.DESC); + out.println(); + out.println( div ); + out.println(RemoteRegistryDeleteDescriptorCommand.USAGE + "\n\n" + RemoteRegistryDeleteDescriptorCommand.DESC); + out.println(); + out.println( div ); } } @@ -439,6 +529,11 @@ public class KnoxCLI extends Configured implements Tool { TopologyService ts = services.getService(GatewayServices.TOPOLOGY_SERVICE); return ts; } + + protected RemoteConfigurationRegistryClientService getRemoteConfigRegistryClientService() { + return services.getService(GatewayServices.REMOTE_REGISTRY_CLIENT_SERVICE); + } + } private class AliasListCommand extends Command { @@ -1598,9 +1693,10 @@ public class KnoxCLI extends Configured implements Tool { public class ServiceTestCommand extends Command { public static final String USAGE = "service-test [--u username] [--p password] [--cluster clustername] [--hostname name] " + "[--port port]"; - public static final String DESC = "This command requires a running instance of Knox to be present on the same " + - "machine. It will execute a test to make sure all services are accessible through the gateway URLs. Errors are " + - "reported and suggestions to resolve any problems are returned. JSON formatted."; + public static final String DESC = + "This command requires a running instance of Knox to be present on the same machine.\n" + + "It will execute a test to make sure all services are accessible through the gateway URLs.\n" + + "Errors are reported and suggestions to resolve any problems are returned. JSON formatted.\n"; private boolean ssl = true; private int attempts = 0; @@ -1753,6 +1849,285 @@ public class KnoxCLI extends Configured implements Tool { } + public class RemoteRegistryClientsListCommand extends Command { + + static final String USAGE = "list-registry-clients"; + static final String DESC = "Lists all of the remote configuration registry clients defined in gateway-site.xml.\n"; + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#execute() + */ + @Override + public void execute() throws Exception { + GatewayConfig config = getGatewayConfig(); + List<String> remoteConfigRegistryClientNames = config.getRemoteRegistryConfigurationNames(); + if (!remoteConfigRegistryClientNames.isEmpty()) { + out.println("Listing remote configuration registry clients:"); + for (String name : remoteConfigRegistryClientNames) { + out.println(name); + } + } + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#getUsage() + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + + /** + * Base class for remote config registry upload commands + */ + public abstract class RemoteRegistryUploadCommand extends Command { + protected static final String ROOT_ENTRY = "/knox"; + protected static final String CONFIG_ENTRY = ROOT_ENTRY + "/config"; + protected static final String PROVIDER_CONFIG_ENTRY = CONFIG_ENTRY + "/shared-providers"; + protected static final String DESCRIPTORS__ENTRY = CONFIG_ENTRY + "/descriptors"; + + private File sourceFile = null; + protected String filename = null; + + protected RemoteRegistryUploadCommand(String sourceFileName) { + this.filename = sourceFileName; + } + + private void upload(RemoteConfigurationRegistryClient client, String entryPath, File source) throws Exception { + String content = FileUtils.readFileToString(source); + if (client.entryExists(entryPath)) { + // If it exists, then we're going to set the data + client.setEntryData(entryPath, content); + } else { + // If it does not exist, then create it and set the data + client.createEntry(entryPath, content); + } + } + + File getSourceFile() { + if (sourceFile == null) { + sourceFile = new File(filename); + } + return sourceFile; + } + + String getEntryName(String prefixPath) { + String entryName = remoteRegistryEntryName; + if (entryName == null) { + File sourceFile = getSourceFile(); + if (sourceFile.exists()) { + String path = sourceFile.getAbsolutePath(); + entryName = path.substring(path.lastIndexOf(File.separator) + 1); + } else { + out.println("Could not locate source file: " + filename); + } + } + return prefixPath + "/" + entryName; + } + + protected void execute(String entryName, File sourceFile) throws Exception { + if (remoteRegistryClient != null) { + RemoteConfigurationRegistryClientService cs = getRemoteConfigRegistryClientService(); + RemoteConfigurationRegistryClient client = cs.get(remoteRegistryClient); + if (client != null) { + if (entryName != null) { + upload(client, entryName, sourceFile); + } + } else { + out.println("No remote configuration registry identified by '" + remoteRegistryClient + "' could be found."); + } + } else { + out.println("Missing required argument : --registry-client\n"); + } + } + + } + + + public class RemoteRegistryUploadProviderConfigCommand extends RemoteRegistryUploadCommand { + + static final String USAGE = "upload-provider-config providerConfigFile --registry-client name [--entry-name entryName]"; + static final String DESC = "Uploads a provider configuration to the specified remote registry client, optionally " + + "renaming the entry.\nIf the entry name is not specified, the name of the uploaded " + + "file is used.\n"; + + RemoteRegistryUploadProviderConfigCommand(String fileName) { + super(fileName); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#execute() + */ + @Override + public void execute() throws Exception { + super.execute(getEntryName(PROVIDER_CONFIG_ENTRY), getSourceFile()); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#getUsage() + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + + public class RemoteRegistryUploadDescriptorCommand extends RemoteRegistryUploadCommand { + + static final String USAGE = "upload-descriptor descriptorFile --registry-client name [--entry-name entryName]"; + static final String DESC = "Uploads a simple descriptor using the specified remote registry client, optionally " + + "renaming the entry.\nIf the entry name is not specified, the name of the uploaded " + + "file is used.\n"; + + RemoteRegistryUploadDescriptorCommand(String fileName) { + super(fileName); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#execute() + */ + @Override + public void execute() throws Exception { + super.execute(getEntryName(DESCRIPTORS__ENTRY), getSourceFile()); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#getUsage() + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + + public class RemoteRegistryGetACLCommand extends Command { + + static final String USAGE = "get-registry-acl entry --registry-client name"; + static final String DESC = "Presents the ACL settings for the specified remote registry entry.\n"; + + private String entry = null; + + RemoteRegistryGetACLCommand(String entry) { + this.entry = entry; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#execute() + */ + @Override + public void execute() throws Exception { + if (remoteRegistryClient != null) { + RemoteConfigurationRegistryClientService cs = getRemoteConfigRegistryClientService(); + RemoteConfigurationRegistryClient client = cs.get(remoteRegistryClient); + if (client != null) { + if (entry != null) { + List<RemoteConfigurationRegistryClient.EntryACL> acls = client.getACL(entry); + for (RemoteConfigurationRegistryClient.EntryACL acl : acls) { + out.println(acl.getType() + ":" + acl.getId() + ":" + acl.getPermissions()); + } + } + } else { + out.println("No remote configuration registry identified by '" + remoteRegistryClient + "' could be found."); + } + } else { + out.println("Missing required argument : --registry-client\n"); + } + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.util.KnoxCLI.Command#getUsage() + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + + /** + * Base class for remote config registry delete commands + */ + public abstract class RemoteRegistryDeleteCommand extends Command { + protected static final String ROOT_ENTRY = "/knox"; + protected static final String CONFIG_ENTRY = ROOT_ENTRY + "/config"; + protected static final String PROVIDER_CONFIG_ENTRY = CONFIG_ENTRY + "/shared-providers"; + protected static final String DESCRIPTORS__ENTRY = CONFIG_ENTRY + "/descriptors"; + + protected String entryName = null; + + protected RemoteRegistryDeleteCommand(String entryName) { + this.entryName = entryName; + } + + private void delete(RemoteConfigurationRegistryClient client, String entryPath) throws Exception { + if (client.entryExists(entryPath)) { + // If it exists, then delete it + client.deleteEntry(entryPath); + } + } + + protected void execute(String entryName) throws Exception { + if (remoteRegistryClient != null) { + RemoteConfigurationRegistryClientService cs = getRemoteConfigRegistryClientService(); + RemoteConfigurationRegistryClient client = cs.get(remoteRegistryClient); + if (client != null) { + if (entryName != null) { + delete(client, entryName); + } + } else { + out.println("No remote configuration registry identified by '" + remoteRegistryClient + "' could be found."); + } + } else { + out.println("Missing required argument : --registry-client\n"); + } + } + } + + + public class RemoteRegistryDeleteProviderConfigCommand extends RemoteRegistryDeleteCommand { + static final String USAGE = "delete-provider-config providerConfig --registry-client name"; + static final String DESC = "Deletes a shared provider configuration from the specified remote registry.\n"; + + public RemoteRegistryDeleteProviderConfigCommand(String entryName) { + super(entryName); + } + + @Override + public void execute() throws Exception { + execute(PROVIDER_CONFIG_ENTRY + "/" + entryName); + } + + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + + public class RemoteRegistryDeleteDescriptorCommand extends RemoteRegistryDeleteCommand { + static final String USAGE = "delete-descriptor descriptor --registry-client name"; + static final String DESC = "Deletes a simple descriptor from the specified remote registry.\n"; + + public RemoteRegistryDeleteDescriptorCommand(String entryName) { + super(entryName); + } + + @Override + public void execute() throws Exception { + execute(DESCRIPTORS__ENTRY + "/" + entryName); + } + + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + private static Properties loadBuildProperties() { Properties properties = new Properties(); InputStream inputStream = KnoxCLI.class.getClassLoader().getResourceAsStream( "build.properties" ); http://git-wip-us.apache.org/repos/asf/knox/blob/828ea38f/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientService.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientService.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientService.java new file mode 100644 index 0000000..161c201 --- /dev/null +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientService.java @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.service.config.remote.config.RemoteConfigurationRegistriesAccessor; +import org.apache.hadoop.gateway.services.ServiceLifecycleException; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClient; +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; +import org.apache.hadoop.gateway.services.security.AliasService; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** + * An implementation of RemoteConfigurationRegistryClientService intended to be used for testing without having to + * connect to an actual remote configuration registry. + */ +public class LocalFileSystemRemoteConfigurationRegistryClientService implements RemoteConfigurationRegistryClientService { + + public static final String TYPE = "LocalFileSystem"; + + private Map<String, RemoteConfigurationRegistryClient> clients = new HashMap<>(); + + + @Override + public void setAliasService(AliasService aliasService) { + // N/A + } + + @Override + public RemoteConfigurationRegistryClient get(String name) { + return clients.get(name); + } + + @Override + public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException { + List<RemoteConfigurationRegistryConfig> registryConfigurations = + RemoteConfigurationRegistriesAccessor.getRemoteRegistryConfigurations(config); + for (RemoteConfigurationRegistryConfig registryConfig : registryConfigurations) { + if (TYPE.equalsIgnoreCase(registryConfig.getRegistryType())) { + RemoteConfigurationRegistryClient registryClient = createClient(registryConfig); + clients.put(registryConfig.getName(), registryClient); + } + } + } + + @Override + public void start() throws ServiceLifecycleException { + + } + + @Override + public void stop() throws ServiceLifecycleException { + + } + + + private RemoteConfigurationRegistryClient createClient(RemoteConfigurationRegistryConfig config) { + String rootDir = config.getConnectionString(); + + return new RemoteConfigurationRegistryClient() { + private File root = new File(rootDir); + + @Override + public String getAddress() { + return root.getAbsolutePath(); + } + + @Override + public boolean entryExists(String path) { + return (new File(root, path)).exists(); + } + + @Override + public List<EntryACL> getACL(String path) { + List<EntryACL> result = new ArrayList<>(); + + Path resolved = Paths.get(rootDir, path); + try { + Map<String, List<String>> collected = new HashMap<>(); + + Set<PosixFilePermission> perms = Files.getPosixFilePermissions(resolved); + for (PosixFilePermission perm : perms) { + String[] parsed = perm.toString().split("_"); + collected.computeIfAbsent(parsed[0].toLowerCase(), s -> new ArrayList<>()).add(parsed[1].toLowerCase()); + } + + for (String id : collected.keySet()) { + EntryACL acl = new EntryACL() { + @Override + public String getId() { + return id; + } + + @Override + public String getType() { + return "fs"; + } + + @Override + public Object getPermissions() { + return collected.get(id).toString(); + } + }; + result.add(acl); + } + } catch (IOException e) { + e.printStackTrace(); + } + return result; + } + + @Override + public List<String> listChildEntries(String path) { + List<String> result = new ArrayList<>(); + + File entry = new File(root, path); + if (entry.exists() && entry.isDirectory()) { + String[] list = entry.list(); + if (list != null) { + result.addAll(Arrays.asList(entry.list())); + } + } + + return result; + } + + @Override + public String getEntryData(String path) { + return getEntryData(path, "UTF-8"); + } + + @Override + public String getEntryData(String path, String encoding) { + String result = null; + File entry = new File(root, path); + if (entry.isFile() && entry.exists()) { + try { + result = FileUtils.readFileToString(entry, encoding); + } catch (IOException e) { + e.printStackTrace(); + } + } + return result; + } + + @Override + public void createEntry(String path) { + createEntry(path, ""); + } + + @Override + public void createEntry(String path, String data) { + createEntry(path, data, "UTF-8"); + } + + @Override + public void createEntry(String path, String data, String encoding) { + File entry = new File(root, path); + if (!entry.exists()) { + if (data != null) { + try { + FileUtils.writeStringToFile(entry, data, encoding); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + @Override + public int setEntryData(String path, String data) { + setEntryData(path, data, "UTF-8"); + return 0; + } + + @Override + public int setEntryData(String path, String data, String encoding) { + File entry = new File(root, path); + if (entry.exists()) { + try { + FileUtils.writeStringToFile(entry, data, encoding); + } catch (IOException e) { + e.printStackTrace(); + } + } + return 0; + } + + @Override + public void deleteEntry(String path) { + File entry = new File(root, path); + if (entry.exists()) { + entry.delete(); + } + } + + @Override + public void addChildEntryListener(String path, ChildEntryListener listener) throws Exception { + // N/A + } + + @Override + public void addEntryListener(String path, EntryListener listener) throws Exception { + // N/A + } + + @Override + public void removeEntryListener(String path) throws Exception { + // N/A + } + }; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/828ea38f/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientServiceProvider.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientServiceProvider.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientServiceProvider.java new file mode 100644 index 0000000..42e79c1 --- /dev/null +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/service/config/remote/LocalFileSystemRemoteConfigurationRegistryClientServiceProvider.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.service.config.remote; + +import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; + +public class LocalFileSystemRemoteConfigurationRegistryClientServiceProvider implements RemoteConfigurationRegistryClientServiceProvider { + + @Override + public String getType() { + return LocalFileSystemRemoteConfigurationRegistryClientService.TYPE; + } + + @Override + public RemoteConfigurationRegistryClientService newInstance() { + return new LocalFileSystemRemoteConfigurationRegistryClientService(); + } +} http://git-wip-us.apache.org/repos/asf/knox/blob/828ea38f/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java b/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java index 838f114..2d4586f 100644 --- a/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java +++ b/gateway-server/src/test/java/org/apache/hadoop/gateway/util/KnoxCLITest.java @@ -27,6 +27,7 @@ import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegis import org.apache.hadoop.gateway.services.config.client.RemoteConfigurationRegistryClientService; import org.apache.hadoop.gateway.services.security.AliasService; import org.apache.hadoop.gateway.services.security.MasterService; +import org.apache.hadoop.test.TestUtils; import org.junit.Before; import org.junit.Test; @@ -66,9 +67,11 @@ public class KnoxCLITest { @Test public void testRemoteConfigurationRegistryClientService() throws Exception { outContent.reset(); + KnoxCLI cli = new KnoxCLI(); Configuration config = new GatewayConfigImpl(); - config.set("gateway.remote.config.registry.test_client", "type=ZooKeeper;address=localhost:2181"); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=/test"); cli.setConf(config); // This is only to get the gateway services initialized @@ -84,6 +87,345 @@ public class KnoxCLITest { } @Test + public void testListRemoteConfigurationRegistryClients() throws Exception { + outContent.reset(); + + KnoxCLI cli = new KnoxCLI(); + String[] args = { "list-registry-clients", "--master","master" }; + + Configuration config = new GatewayConfigImpl(); + cli.setConf(config); + + // Test with no registry clients configured + int rc = cli.run(args); + assertEquals(0, rc); + assertTrue(outContent.toString(), outContent.toString().isEmpty()); + + // Test with a single client configured + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=/test1"); + cli.setConf(config); + outContent.reset(); + rc = cli.run(args); + assertEquals(0, rc); + assertTrue(outContent.toString(), outContent.toString().contains("test_client")); + + // Configure another client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.another_client", "type=LocalFileSystem;address=/test2"); + cli.setConf(config); + outContent.reset(); + rc = cli.run(args); + assertEquals(0, rc); + assertTrue(outContent.toString(), outContent.toString().contains("test_client")); + assertTrue(outContent.toString(), outContent.toString().contains("another_client")); + } + + @Test + public void testRemoteConfigurationRegistryGetACLs() throws Exception { + outContent.reset(); + + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + + final String providerConfigName = "my-provider-config.xml"; + final String providerConfigContent = "<gateway/>\n"; + final File testProviderConfig = new File(testRoot, providerConfigName); + final String[] uploadArgs = {"upload-provider-config", testProviderConfig.getAbsolutePath(), + "--registry-client", "test_client", + "--master", "master"}; + FileUtils.writeStringToFile(testProviderConfig, providerConfigContent); + + + final String[] args = {"get-registry-acl", "/knox/config/shared-providers", + "--registry-client", "test_client", + "--master", "master"}; + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + int rc = cli.run(uploadArgs); + assertEquals(0, rc); + + // Run the test command + rc = cli.run(args); + + // Validate the result + assertEquals(0, rc); + String result = outContent.toString(); + assertEquals(result, 3, result.split("\n").length); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + + @Test + public void testRemoteConfigurationRegistryUploadProviderConfig() throws Exception { + outContent.reset(); + + final String providerConfigName = "my-provider-config.xml"; + final String providerConfigContent = "<gateway/>\n"; + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + final File testProviderConfig = new File(testRoot, providerConfigName); + + final String[] args = {"upload-provider-config", testProviderConfig.getAbsolutePath(), + "--registry-client", "test_client", + "--master", "master"}; + + FileUtils.writeStringToFile(testProviderConfig, providerConfigContent); + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + // Run the test command + int rc = cli.run(args); + + // Validate the result + assertEquals(0, rc); + File registryFile = new File(testRegistry, "knox/config/shared-providers/" + providerConfigName); + assertTrue(registryFile.exists()); + assertEquals(FileUtils.readFileToString(registryFile), providerConfigContent); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + + @Test + public void testRemoteConfigurationRegistryUploadProviderConfigWithDestinationOverride() throws Exception { + outContent.reset(); + + final String providerConfigName = "my-provider-config.xml"; + final String entryName = "my-providers.xml"; + final String providerConfigContent = "<gateway/>\n"; + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + final File testProviderConfig = new File(testRoot, providerConfigName); + + final String[] args = {"upload-provider-config", testProviderConfig.getAbsolutePath(), + "--entry-name", entryName, + "--registry-client", "test_client", + "--master", "master"}; + + FileUtils.writeStringToFile(testProviderConfig, providerConfigContent); + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + // Run the test command + int rc = cli.run(args); + + // Validate the result + assertEquals(0, rc); + assertFalse((new File(testRegistry, "knox/config/shared-providers/" + providerConfigName)).exists()); + File registryFile = new File(testRegistry, "knox/config/shared-providers/" + entryName); + assertTrue(registryFile.exists()); + assertEquals(FileUtils.readFileToString(registryFile), providerConfigContent); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + + @Test + public void testRemoteConfigurationRegistryUploadDescriptor() throws Exception { + outContent.reset(); + + final String descriptorName = "my-topology.json"; + final String descriptorContent = testDescriptorContentJSON; + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + final File testDescriptor = new File(testRoot, descriptorName); + + final String[] args = {"upload-descriptor", testDescriptor.getAbsolutePath(), + "--registry-client", "test_client", + "--master", "master"}; + + FileUtils.writeStringToFile(testDescriptor, descriptorContent); + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + // Run the test command + int rc = cli.run(args); + + // Validate the result + assertEquals(0, rc); + File registryFile = new File(testRegistry, "knox/config/descriptors/" + descriptorName); + assertTrue(registryFile.exists()); + assertEquals(FileUtils.readFileToString(registryFile), descriptorContent); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + @Test + public void testRemoteConfigurationRegistryUploadDescriptorWithDestinationOverride() throws Exception { + outContent.reset(); + + final String descriptorName = "my-topology.json"; + final String entryName = "different-topology.json"; + final String descriptorContent = testDescriptorContentJSON; + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + final File testDescriptor = new File(testRoot, descriptorName); + + final String[] args = {"upload-descriptor", testDescriptor.getAbsolutePath(), + "--entry-name", entryName, + "--registry-client", "test_client", + "--master", "master"}; + + FileUtils.writeStringToFile(testDescriptor, descriptorContent); + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + // Run the test command + int rc = cli.run(args); + + // Validate the result + assertEquals(0, rc); + assertFalse((new File(testRegistry, "knox/config/descriptors/" + descriptorName)).exists()); + File registryFile = new File(testRegistry, "knox/config/descriptors/" + entryName); + assertTrue(registryFile.exists()); + assertEquals(FileUtils.readFileToString(registryFile), descriptorContent); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + @Test + public void testRemoteConfigurationRegistryDeleteProviderConfig() throws Exception { + outContent.reset(); + + // Create a provider config + final String providerConfigName = "my-provider-config.xml"; + final String providerConfigContent = "<gateway/>\n"; + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + final File testProviderConfig = new File(testRoot, providerConfigName); + + final String[] createArgs = {"upload-provider-config", testProviderConfig.getAbsolutePath(), + "--registry-client", "test_client", + "--master", "master"}; + + FileUtils.writeStringToFile(testProviderConfig, providerConfigContent); + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + // Run the test command + int rc = cli.run(createArgs); + + // Validate the result + assertEquals(0, rc); + File registryFile = new File(testRegistry, "knox/config/shared-providers/" + providerConfigName); + assertTrue(registryFile.exists()); + + outContent.reset(); + + // Delete the created provider config + final String[] deleteArgs = {"delete-provider-config", providerConfigName, + "--registry-client", "test_client", + "--master", "master"}; + rc = cli.run(deleteArgs); + assertEquals(0, rc); + assertFalse(registryFile.exists()); + + // Try to delete a provider config that does not exist + rc = cli.run(new String[]{"delete-provider-config", "imaginary-providers.xml", + "--registry-client", "test_client", + "--master", "master"}); + assertEquals(0, rc); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + @Test + public void testRemoteConfigurationRegistryDeleteDescriptor() throws Exception { + outContent.reset(); + + final String descriptorName = "my-topology.json"; + final String descriptorContent = testDescriptorContentJSON; + + final File testRoot = TestUtils.createTempDir(this.getClass().getName()); + try { + final File testRegistry = new File(testRoot, "registryRoot"); + final File testDescriptor = new File(testRoot, descriptorName); + + final String[] createArgs = {"upload-descriptor", testDescriptor.getAbsolutePath(), + "--registry-client", "test_client", + "--master", "master"}; + + FileUtils.writeStringToFile(testDescriptor, descriptorContent); + + KnoxCLI cli = new KnoxCLI(); + Configuration config = new GatewayConfigImpl(); + // Configure a client for the test local filesystem registry implementation + config.set("gateway.remote.config.registry.test_client", "type=LocalFileSystem;address=" + testRegistry); + cli.setConf(config); + + // Run the test command + int rc = cli.run(createArgs); + + // Validate the result + assertEquals(0, rc); + File registryFile = new File(testRegistry, "knox/config/descriptors/" + descriptorName); + assertTrue(registryFile.exists()); + + outContent.reset(); + + // Delete the created provider config + final String[] deleteArgs = {"delete-descriptor", descriptorName, + "--registry-client", "test_client", + "--master", "master"}; + rc = cli.run(deleteArgs); + assertEquals(0, rc); + assertFalse(registryFile.exists()); + + // Try to delete a descriptor that does not exist + rc = cli.run(new String[]{"delete-descriptor", "bogus.json", + "--registry-client", "test_client", + "--master", "master"}); + assertEquals(0, rc); + } finally { + FileUtils.forceDelete(testRoot); + } + } + + @Test public void testSuccessfulAliasLifecycle() throws Exception { outContent.reset(); String[] args1 = {"create-alias", "alias1", "--value", "testvalue1", "--master", "master"}; @@ -670,4 +1012,21 @@ public class KnoxCLITest { } + private static final String testDescriptorContentJSON = "{\n" + + " \"discovery-address\":\"http://localhost:8080\",\n"; + + " \"discovery-user\":\"maria_dev\",\n" + + " \"discovery-pwd-alias\":\"sandbox.discovery.password\",\n" + + " \"provider-config-ref\":\"my-provider-config\",\n" + + " \"cluster\":\"Sandbox\",\n" + + " \"services\":[\n" + + " {\"name\":\"NAMENODE\"},\n" + + " {\"name\":\"JOBTRACKER\"},\n" + + " {\"name\":\"WEBHDFS\"},\n" + + " {\"name\":\"WEBHCAT\"},\n" + + " {\"name\":\"OOZIE\"},\n" + + " {\"name\":\"WEBHBASE\"},\n" + + " {\"name\":\"HIVE\"},\n" + + " {\"name\":\"RESOURCEMANAGER\"}\n" + + " ]\n" + + "}"; } http://git-wip-us.apache.org/repos/asf/knox/blob/828ea38f/gateway-server/src/test/resources/META-INF/services/org.apache.hadoop.gateway.service.config.remote.RemoteConfigurationRegistryClientServiceProvider ---------------------------------------------------------------------- diff --git a/gateway-server/src/test/resources/META-INF/services/org.apache.hadoop.gateway.service.config.remote.RemoteConfigurationRegistryClientServiceProvider b/gateway-server/src/test/resources/META-INF/services/org.apache.hadoop.gateway.service.config.remote.RemoteConfigurationRegistryClientServiceProvider new file mode 100644 index 0000000..ffd9284 --- /dev/null +++ b/gateway-server/src/test/resources/META-INF/services/org.apache.hadoop.gateway.service.config.remote.RemoteConfigurationRegistryClientServiceProvider @@ -0,0 +1,19 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +org.apache.hadoop.gateway.service.config.remote.LocalFileSystemRemoteConfigurationRegistryClientServiceProvider
