Repository: hadoop Updated Branches: refs/heads/trunk 404eab4dc -> d30d57828
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java index e18bcae..173001b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.net.DNS; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; @@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; @@ -43,9 +45,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; -import java.net.ServerSocket; -import java.net.URL; -import java.net.URLDecoder; +import java.net.*; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -57,6 +57,11 @@ import java.util.Map; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic + .HADOOP_SECURITY_DNS_INTERFACE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic + .HADOOP_SECURITY_DNS_NAMESERVER_KEY; + /** * These are slider-specific Util methods */ @@ -542,4 +547,24 @@ public final class ServiceUtils { public static String createDescriptionTag(String description) { return "Description: " + description; } + + // Copied from SecurityUtil because it is not public + public static String getLocalHostName(@Nullable Configuration conf) + throws UnknownHostException { + if (conf != null) { + String dnsInterface = conf.get(HADOOP_SECURITY_DNS_INTERFACE_KEY); + String nameServer = conf.get(HADOOP_SECURITY_DNS_NAMESERVER_KEY); + + if (dnsInterface != null) { + return DNS.getDefaultHost(dnsInterface, nameServer, true); + } else if (nameServer != null) { + throw new IllegalArgumentException(HADOOP_SECURITY_DNS_NAMESERVER_KEY + + " requires " + HADOOP_SECURITY_DNS_INTERFACE_KEY + ". Check your" + + "configuration."); + } + } + + // Fallback to querying the default hostname as we did before. + return InetAddress.getLocalHost().getCanonicalHostName(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo new file mode 100644 index 0000000..14cdf68 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.yarn.service.ClientAMSecurityInfo http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java index 704b097..e74ca81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java @@ -111,6 +111,27 @@ public final class RegistryOperationsFactory { } /** + * Create a kerberos registry service client + * @param conf configuration + * @param jaasClientEntry the name of the login config entry + * @param principal principal of the client. + * @param keytab location to the keytab file + * @return a registry service client instance + */ + public static RegistryOperations createKerberosInstance(Configuration conf, + String jaasClientEntry, String principal, String keytab) { + Preconditions.checkArgument(conf != null, "Null configuration"); + conf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_KERBEROS); + conf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, jaasClientEntry); + RegistryOperationsClient operations = + new RegistryOperationsClient("KerberosRegistryOperations"); + operations.setKerberosPrincipalAndKeytab(principal, keytab); + operations.init(conf); + return operations; + } + + + /** * Create and initialize an operations instance authenticated with write * access via an <code>id:password</code> pair. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java index 8713920..c81a0ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java @@ -127,6 +127,7 @@ public class CuratorService extends CompositeService } else { this.bindingSource = this; } + registrySecurity = new RegistrySecurity("registry security"); } /** @@ -152,8 +153,7 @@ public class CuratorService extends CompositeService registryRoot = conf.getTrimmed(KEY_REGISTRY_ZK_ROOT, DEFAULT_ZK_REGISTRY_ROOT); - // create and add the registy service - registrySecurity = new RegistrySecurity("registry security"); + // add the registy service addService(registrySecurity); if (LOG.isDebugEnabled()) { @@ -163,6 +163,10 @@ public class CuratorService extends CompositeService super.serviceInit(conf); } + public void setKerberosPrincipalAndKeytab(String principal, String keytab) { + registrySecurity.setKerberosPrincipalAndKeytab(principal, keytab); + } + /** * Start the service. * This is where the curator instance is started. http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java index 23fadb5..ff6e8aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java @@ -23,6 +23,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.util.KerberosUtil; @@ -31,6 +32,7 @@ import org.apache.hadoop.service.ServiceStateException; import org.apache.hadoop.util.ZKUtil; import org.apache.zookeeper.Environment; import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.client.ZooKeeperSaslClient; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Id; import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; @@ -44,9 +46,11 @@ import java.lang.reflect.InvocationTargetException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.ListIterator; import java.util.Locale; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import static org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions.*; @@ -170,13 +174,17 @@ public class RegistrySecurity extends AbstractService { /** * Client context */ - private String jaasClientContext; + private String jaasClientEntry; /** * Client identity */ private String jaasClientIdentity; + private String principal; + + private String keytab; + /** * Create an instance * @param name service name @@ -238,6 +246,8 @@ public class RegistrySecurity extends AbstractService { systemACLs.addAll(buildACLs(system, kerberosRealm, ZooDefs.Perms.ALL)); + LOG.info("Registry default system acls: " + System.lineSeparator() + + systemACLs); // user accounts (may be empty, but for digest one user AC must // be built up String user = getConfig().get(KEY_REGISTRY_USER_ACCOUNTS, @@ -252,6 +262,7 @@ public class RegistrySecurity extends AbstractService { userACLs.add(self); } } + LOG.info("Registry User ACLs " + System.lineSeparator()+ userACLs); // here check for UGI having secure on or digest + ID switch (access) { @@ -262,13 +273,12 @@ public class RegistrySecurity extends AbstractService { } UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - jaasClientContext = getOrFail(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, + jaasClientEntry = getOrFail(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, DEFAULT_REGISTRY_CLIENT_JAAS_CONTEXT); jaasClientIdentity = currentUser.getShortUserName(); if (LOG.isDebugEnabled()) { LOG.debug("Auth is SASL user=\"{}\" JAAS context=\"{}\"", - jaasClientIdentity, - jaasClientContext); + jaasClientIdentity, jaasClientEntry); } break; @@ -738,9 +748,81 @@ public class RegistrySecurity extends AbstractService { break; case sasl: - // bind to the current identity and context within the JAAS file - setZKSaslClientProperties(jaasClientIdentity, jaasClientContext); + JaasConfiguration jconf = + new JaasConfiguration(jaasClientEntry, principal, keytab); + javax.security.auth.login.Configuration.setConfiguration(jconf); + setSystemPropertyIfUnset(ZooKeeperSaslClient.ENABLE_CLIENT_SASL_KEY, + "true"); + setSystemPropertyIfUnset(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, + jaasClientEntry); + LOG.info( + "Enabling ZK sasl client: jaasClientEntry = " + jaasClientEntry + + ", principal = " + principal + ", keytab = " + keytab); + } + } + } + + public void setKerberosPrincipalAndKeytab(String principal, String keytab) { + this.principal = principal; + this.keytab = keytab; + } + + /** + * Creates a programmatic version of a jaas.conf file. This can be used + * instead of writing a jaas.conf file and setting the system property, + * "java.security.auth.login.config", to point to that file. It is meant to be + * used for connecting to ZooKeeper. + */ + @InterfaceAudience.Private + public static class JaasConfiguration extends + javax.security.auth.login.Configuration { + + private final javax.security.auth.login.Configuration baseConfig = + javax.security.auth.login.Configuration.getConfiguration(); + private static AppConfigurationEntry[] entry; + private String entryName; + + /** + * Add an entry to the jaas configuration with the passed in name, + * principal, and keytab. The other necessary options will be set for you. + * + * @param entryName The name of the entry (e.g. "Client") + * @param principal The principal of the user + * @param keytab The location of the keytab + */ + public JaasConfiguration(String entryName, String principal, String keytab) { + this.entryName = entryName; + Map<String, String> options = new HashMap<String, String>(); + options.put("keyTab", keytab); + options.put("principal", principal); + options.put("useKeyTab", "true"); + options.put("storeKey", "true"); + options.put("useTicketCache", "false"); + options.put("refreshKrb5Config", "true"); + String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG"); + if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) { + options.put("debug", "true"); + } + entry = new AppConfigurationEntry[]{ + new AppConfigurationEntry(getKrb5LoginModuleName(), + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, + options)}; + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + return (entryName.equals(name)) ? entry : ((baseConfig != null) + ? baseConfig.getAppConfigurationEntry(name) : null); + } + + private String getKrb5LoginModuleName() { + String krb5LoginModuleName; + if (System.getProperty("java.vendor").contains("IBM")) { + krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule"; + } else { + krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule"; } + return krb5LoginModuleName; } } @@ -899,7 +981,7 @@ public class RegistrySecurity extends AbstractService { .append("; "); builder.append(KEY_REGISTRY_CLIENT_JAAS_CONTEXT) .append("=") - .append(jaasClientContext) + .append(jaasClientEntry) .append("; "); builder.append(describeProperty(PROP_ZK_SASL_CLIENT_USERNAME)); builder.append(describeProperty(PROP_ZK_SASL_CLIENT_CONTEXT)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index d7ebece..358a963 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -412,6 +412,10 @@ public class RegistryDNS extends AbstractService implements DNSOperations, // Single reverse zone } else { Name reverseLookupZoneName = getReverseZoneName(conf); + if (reverseLookupZoneName == null) { + // reverse lookup disabled + return; + } Zone reverseLookupZone = configureZone(reverseLookupZoneName, conf); zones.put(reverseLookupZone.getOrigin(), reverseLookupZone); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java deleted file mode 100644 index e11890f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/integration/RMRegistryOperationsService.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.registry.server.integration; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.curator.framework.api.BackgroundCallback; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.registry.client.impl.zk.RegistryBindingSource; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.server.services.DeleteCompletionCallback; -import org.apache.hadoop.registry.server.services.RegistryAdminService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.Future; - -/** - * Handle RM events by updating the registry - * <p> - * These actions are all implemented as event handlers to operations - * which come from the RM. - * <p> - * This service is expected to be executed by a user with the permissions - * to manipulate the entire registry, - */ [email protected]("YARN") [email protected] -public class RMRegistryOperationsService extends RegistryAdminService { - private static final Logger LOG = - LoggerFactory.getLogger(RMRegistryOperationsService.class); - - private PurgePolicy purgeOnCompletionPolicy = PurgePolicy.PurgeAll; - - public RMRegistryOperationsService(String name) { - this(name, null); - } - - public RMRegistryOperationsService(String name, - RegistryBindingSource bindingSource) { - super(name, bindingSource); - } - - - /** - * Extend the parent service initialization by verifying that the - * service knows âin a secure clusterâ the realm in which it is executing. - * It needs this to properly build up the user names and hence their - * access rights. - * - * @param conf configuration of the service - * @throws Exception - */ - @Override - protected void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - - verifyRealmValidity(); - } - - public PurgePolicy getPurgeOnCompletionPolicy() { - return purgeOnCompletionPolicy; - } - - public void setPurgeOnCompletionPolicy(PurgePolicy purgeOnCompletionPolicy) { - this.purgeOnCompletionPolicy = purgeOnCompletionPolicy; - } - - public void onApplicationAttemptRegistered(ApplicationAttemptId attemptId, - String host, int rpcport, String trackingurl) throws IOException { - - } - - public void onApplicationLaunched(ApplicationId id) throws IOException { - - } - - /** - * Actions to take as an AM registers itself with the RM. - * @param attemptId attempt ID - * @throws IOException problems - */ - public void onApplicationMasterRegistered(ApplicationAttemptId attemptId) throws - IOException { - } - - /** - * Actions to take when the AM container is completed - * @param containerId container ID - * @throws IOException problems - */ - public void onAMContainerFinished(ContainerId containerId) throws - IOException { - LOG.info("AM Container {} finished, purging application attempt records", - containerId); - - // remove all application attempt entries - purgeAppAttemptRecords(containerId.getApplicationAttemptId()); - - // also treat as a container finish to remove container - // level records for the AM container - onContainerFinished(containerId); - } - - /** - * remove all application attempt entries - * @param attemptId attempt ID - */ - protected void purgeAppAttemptRecords(ApplicationAttemptId attemptId) { - purgeRecordsAsync("/", - attemptId.toString(), - PersistencePolicies.APPLICATION_ATTEMPT); - } - - /** - * Actions to take when an application attempt is completed - * @param attemptId application ID - * @throws IOException problems - */ - public void onApplicationAttemptUnregistered(ApplicationAttemptId attemptId) - throws IOException { - LOG.info("Application attempt {} unregistered, purging app attempt records", - attemptId); - purgeAppAttemptRecords(attemptId); - } - - /** - * Actions to take when an application is completed - * @param id application ID - * @throws IOException problems - */ - public void onApplicationCompleted(ApplicationId id) - throws IOException { - LOG.info("Application {} completed, purging application-level records", - id); - purgeRecordsAsync("/", - id.toString(), - PersistencePolicies.APPLICATION); - } - - public void onApplicationAttemptAdded(ApplicationAttemptId appAttemptId) { - } - - /** - * This is the event where the user is known, so the user directory - * can be created - * @param applicationId application ID - * @param user username - * @throws IOException problems - */ - public void onStateStoreEvent(ApplicationId applicationId, String user) throws - IOException { - initUserRegistryAsync(user); - } - - /** - * Actions to take when the AM container is completed - * @param id container ID - * @throws IOException problems - */ - public void onContainerFinished(ContainerId id) throws IOException { - LOG.info("Container {} finished, purging container-level records", - id); - purgeRecordsAsync("/", - id.toString(), - PersistencePolicies.CONTAINER); - } - - /** - * Queue an async operation to purge all matching records under a base path. - * <ol> - * <li>Uses a depth first search</li> - * <li>A match is on ID and persistence policy, or, if policy==-1, any match</li> - * <li>If a record matches then it is deleted without any child searches</li> - * <li>Deletions will be asynchronous if a callback is provided</li> - * </ol> - * @param path base path - * @param id ID for service record.id - * @param persistencePolicyMatch ID for the persistence policy to match: - * no match, no delete. - * @return a future that returns the #of records deleted - */ - @VisibleForTesting - public Future<Integer> purgeRecordsAsync(String path, - String id, - String persistencePolicyMatch) { - - return purgeRecordsAsync(path, - id, persistencePolicyMatch, - purgeOnCompletionPolicy, - new DeleteCompletionCallback()); - } - - /** - * Queue an async operation to purge all matching records under a base path. - * <ol> - * <li>Uses a depth first search</li> - * <li>A match is on ID and persistence policy, or, if policy==-1, any match</li> - * <li>If a record matches then it is deleted without any child searches</li> - * <li>Deletions will be asynchronous if a callback is provided</li> - * </ol> - * @param path base path - * @param id ID for service record.id - * @param persistencePolicyMatch ID for the persistence policy to match: - * no match, no delete. - * @param purgePolicy how to react to children under the entry - * @param callback an optional callback - * @return a future that returns the #of records deleted - */ - @VisibleForTesting - public Future<Integer> purgeRecordsAsync(String path, - String id, - String persistencePolicyMatch, - PurgePolicy purgePolicy, - BackgroundCallback callback) { - LOG.info(" records under {} with ID {} and policy {}: {}", - path, id, persistencePolicyMatch); - return submit( - new AsyncPurge(path, - new SelectByYarnPersistence(id, persistencePolicyMatch), - purgePolicy, - callback)); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java index e160d4a..829ef68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/DeleteCompletionCallback.java @@ -21,7 +21,6 @@ package org.apache.hadoop.registry.server.services; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.BackgroundCallback; import org.apache.curator.framework.api.CuratorEvent; -import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class DeleteCompletionCallback implements BackgroundCallback { private static final Logger LOG = - LoggerFactory.getLogger(RMRegistryOperationsService.class); + LoggerFactory.getLogger(DeleteCompletionCallback.class); private AtomicInteger events = new AtomicInteger(0); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java index 5b34f60..0d4a467 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/AbstractRegistryTest.java @@ -23,7 +23,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; + +import org.apache.hadoop.registry.server.services.RegistryAdminService; import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,22 +32,16 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URISyntaxException; -/** - * Abstract registry tests .. inits the field {@link #registry} - * before the test with an instance of {@link RMRegistryOperationsService}; - * and {@link #operations} with the same instance cast purely - * to the type {@link RegistryOperations}. - * - */ + public class AbstractRegistryTest extends AbstractZKRegistryTest { private static final Logger LOG = LoggerFactory.getLogger(AbstractRegistryTest.class); - protected RMRegistryOperationsService registry; + protected RegistryAdminService registry; protected RegistryOperations operations; @Before public void setupRegistry() throws IOException { - registry = new RMRegistryOperationsService("yarnRegistry"); + registry = new RegistryAdminService("yarnRegistry"); operations = registry; registry.init(createRegistryConfiguration()); registry.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java deleted file mode 100644 index 451a69b..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java +++ /dev/null @@ -1,369 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.registry.integration; - -import org.apache.curator.framework.api.BackgroundCallback; -import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; -import org.apache.hadoop.registry.AbstractRegistryTest; -import org.apache.hadoop.registry.client.api.BindFlags; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper; -import org.apache.hadoop.registry.client.impl.CuratorEventCatcher; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.RegistryPathStatus; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.registry.server.services.DeleteCompletionCallback; -import org.apache.hadoop.registry.server.services.RegistryAdminService; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; - -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.inetAddrEndpoint; -import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.restEndpoint; - -public class TestRegistryRMOperations extends AbstractRegistryTest { - protected static final Logger LOG = - LoggerFactory.getLogger(TestRegistryRMOperations.class); - - /** - * trigger a purge operation - * @param path path - * @param id yarn ID - * @param policyMatch policy to match ID on - * @param purgePolicy policy when there are children under a match - * @return the number purged - * @throws IOException - */ - public int purge(String path, - String id, - String policyMatch, - RegistryAdminService.PurgePolicy purgePolicy) throws - IOException, - ExecutionException, - InterruptedException { - return purge(path, id, policyMatch, purgePolicy, null); - } - - /** - * - * trigger a purge operation - * @param path pathn - * @param id yarn ID - * @param policyMatch policy to match ID on - * @param purgePolicy policy when there are children under a match - * @param callback optional callback - * @return the number purged - * @throws IOException - */ - public int purge(String path, - String id, - String policyMatch, - RegistryAdminService.PurgePolicy purgePolicy, - BackgroundCallback callback) throws - IOException, - ExecutionException, - InterruptedException { - - Future<Integer> future = registry.purgeRecordsAsync(path, - id, policyMatch, purgePolicy, callback); - try { - return future.get(); - } catch (ExecutionException e) { - if (e.getCause() instanceof IOException) { - throw (IOException) e.getCause(); - } else { - throw e; - } - } - } - - @Test - public void testPurgeEntryCuratorCallback() throws Throwable { - - String path = "/users/example/hbase/hbase1/"; - ServiceRecord written = buildExampleServiceEntry( - PersistencePolicies.APPLICATION_ATTEMPT); - written.set(YarnRegistryAttributes.YARN_ID, - "testAsyncPurgeEntry_attempt_001"); - - operations.mknode(RegistryPathUtils.parentOf(path), true); - operations.bind(path, written, 0); - - ZKPathDumper dump = registry.dumpPath(false); - CuratorEventCatcher events = new CuratorEventCatcher(); - - LOG.info("Initial state {}", dump); - - // container query - String id = written.get(YarnRegistryAttributes.YARN_ID, ""); - int opcount = purge("/", - id, - PersistencePolicies.CONTAINER, - RegistryAdminService.PurgePolicy.PurgeAll, - events); - assertPathExists(path); - assertEquals(0, opcount); - assertEquals("Event counter", 0, events.getCount()); - - // now the application attempt - opcount = purge("/", - id, - PersistencePolicies.APPLICATION_ATTEMPT, - RegistryAdminService.PurgePolicy.PurgeAll, - events); - - LOG.info("Final state {}", dump); - - assertPathNotFound(path); - assertEquals("wrong no of delete operations in " + dump, 1, opcount); - // and validate the callback event - assertEquals("Event counter", 1, events.getCount()); - } - - @Test - public void testAsyncPurgeEntry() throws Throwable { - - String path = "/users/example/hbase/hbase1/"; - ServiceRecord written = buildExampleServiceEntry( - PersistencePolicies.APPLICATION_ATTEMPT); - written.set(YarnRegistryAttributes.YARN_ID, - "testAsyncPurgeEntry_attempt_001"); - - operations.mknode(RegistryPathUtils.parentOf(path), true); - operations.bind(path, written, 0); - - ZKPathDumper dump = registry.dumpPath(false); - - LOG.info("Initial state {}", dump); - - DeleteCompletionCallback deletions = new DeleteCompletionCallback(); - int opcount = purge("/", - written.get(YarnRegistryAttributes.YARN_ID, ""), - PersistencePolicies.CONTAINER, - RegistryAdminService.PurgePolicy.PurgeAll, - deletions); - assertPathExists(path); - - dump = registry.dumpPath(false); - - assertEquals("wrong no of delete operations in " + dump, 0, - deletions.getEventCount()); - assertEquals("wrong no of delete operations in " + dump, 0, opcount); - - - // now app attempt - deletions = new DeleteCompletionCallback(); - opcount = purge("/", - written.get(YarnRegistryAttributes.YARN_ID, ""), - PersistencePolicies.APPLICATION_ATTEMPT, - RegistryAdminService.PurgePolicy.PurgeAll, - deletions); - - dump = registry.dumpPath(false); - LOG.info("Final state {}", dump); - - assertPathNotFound(path); - assertEquals("wrong no of delete operations in " + dump, 1, - deletions.getEventCount()); - assertEquals("wrong no of delete operations in " + dump, 1, opcount); - // and validate the callback event - - } - - @Test - public void testPutGetContainerPersistenceServiceEntry() throws Throwable { - - String path = ENTRY_PATH; - ServiceRecord written = buildExampleServiceEntry( - PersistencePolicies.CONTAINER); - - operations.mknode(RegistryPathUtils.parentOf(path), true); - operations.bind(path, written, BindFlags.CREATE); - ServiceRecord resolved = operations.resolve(path); - validateEntry(resolved); - assertMatches(written, resolved); - } - - /** - * Create a complex example app - * @throws Throwable - */ - @Test - public void testCreateComplexApplication() throws Throwable { - String appId = "application_1408631738011_0001"; - String cid = "container_1408631738011_0001_01_"; - String cid1 = cid + "000001"; - String cid2 = cid + "000002"; - String appPath = USERPATH + "tomcat"; - - ServiceRecord webapp = createRecord(appId, - PersistencePolicies.APPLICATION, "tomcat-based web application", - null); - webapp.addExternalEndpoint(restEndpoint("www", - new URI("http", "//loadbalancer/", null))); - - ServiceRecord comp1 = createRecord(cid1, PersistencePolicies.CONTAINER, - null, - null); - comp1.addExternalEndpoint(restEndpoint("www", - new URI("http", "//rack4server3:43572", null))); - comp1.addInternalEndpoint( - inetAddrEndpoint("jmx", "JMX", "rack4server3", 43573)); - - // Component 2 has a container lifespan - ServiceRecord comp2 = createRecord(cid2, PersistencePolicies.CONTAINER, - null, - null); - comp2.addExternalEndpoint(restEndpoint("www", - new URI("http", "//rack1server28:35881", null))); - comp2.addInternalEndpoint( - inetAddrEndpoint("jmx", "JMX", "rack1server28", 35882)); - - operations.mknode(USERPATH, false); - operations.bind(appPath, webapp, BindFlags.OVERWRITE); - String componentsPath = appPath + RegistryConstants.SUBPATH_COMPONENTS; - operations.mknode(componentsPath, false); - String dns1 = RegistryPathUtils.encodeYarnID(cid1); - String dns1path = componentsPath + dns1; - operations.bind(dns1path, comp1, BindFlags.CREATE); - String dns2 = RegistryPathUtils.encodeYarnID(cid2); - String dns2path = componentsPath + dns2; - operations.bind(dns2path, comp2, BindFlags.CREATE); - - ZKPathDumper pathDumper = registry.dumpPath(false); - LOG.info(pathDumper.toString()); - - logRecord("tomcat", webapp); - logRecord(dns1, comp1); - logRecord(dns2, comp2); - - ServiceRecord dns1resolved = operations.resolve(dns1path); - assertEquals("Persistence policies on resolved entry", - PersistencePolicies.CONTAINER, - dns1resolved.get(YarnRegistryAttributes.YARN_PERSISTENCE, "")); - - Map<String, RegistryPathStatus> children = - RegistryUtils.statChildren(operations, componentsPath); - assertEquals(2, children.size()); - Collection<RegistryPathStatus> - componentStats = children.values(); - Map<String, ServiceRecord> records = - RegistryUtils.extractServiceRecords(operations, - componentsPath, componentStats); - assertEquals(2, records.size()); - ServiceRecord retrieved1 = records.get(dns1path); - logRecord(retrieved1.get(YarnRegistryAttributes.YARN_ID, ""), retrieved1); - assertMatches(dns1resolved, retrieved1); - assertEquals(PersistencePolicies.CONTAINER, - retrieved1.get(YarnRegistryAttributes.YARN_PERSISTENCE, "")); - - // create a listing under components/ - operations.mknode(componentsPath + "subdir", false); - - // this shows up in the listing of child entries - Map<String, RegistryPathStatus> childrenUpdated = - RegistryUtils.statChildren(operations, componentsPath); - assertEquals(3, childrenUpdated.size()); - - // the non-record child this is not picked up in the record listing - Map<String, ServiceRecord> recordsUpdated = - - RegistryUtils.extractServiceRecords(operations, - componentsPath, - childrenUpdated); - assertEquals(2, recordsUpdated.size()); - - // now do some deletions. - - // synchronous delete container ID 2 - - // fail if the app policy is chosen - assertEquals(0, purge("/", cid2, PersistencePolicies.APPLICATION, - RegistryAdminService.PurgePolicy.FailOnChildren)); - // succeed for container - assertEquals(1, purge("/", cid2, PersistencePolicies.CONTAINER, - RegistryAdminService.PurgePolicy.FailOnChildren)); - assertPathNotFound(dns2path); - assertPathExists(dns1path); - - // expect a skip on children to skip - assertEquals(0, - purge("/", appId, PersistencePolicies.APPLICATION, - RegistryAdminService.PurgePolicy.SkipOnChildren)); - assertPathExists(appPath); - assertPathExists(dns1path); - - // attempt to delete app with policy of fail on children - try { - int p = purge("/", - appId, - PersistencePolicies.APPLICATION, - RegistryAdminService.PurgePolicy.FailOnChildren); - fail("expected a failure, got a purge count of " + p); - } catch (PathIsNotEmptyDirectoryException expected) { - // expected - } - assertPathExists(appPath); - assertPathExists(dns1path); - - - // now trigger recursive delete - assertEquals(1, - purge("/", appId, PersistencePolicies.APPLICATION, - RegistryAdminService.PurgePolicy.PurgeAll)); - assertPathNotFound(appPath); - assertPathNotFound(dns1path); - - } - - @Test - public void testChildDeletion() throws Throwable { - ServiceRecord app = createRecord("app1", - PersistencePolicies.APPLICATION, "app", - null); - ServiceRecord container = createRecord("container1", - PersistencePolicies.CONTAINER, "container", - null); - - operations.bind("/app", app, BindFlags.OVERWRITE); - operations.bind("/app/container", container, BindFlags.OVERWRITE); - - try { - int p = purge("/", - "app1", - PersistencePolicies.APPLICATION, - RegistryAdminService.PurgePolicy.FailOnChildren); - fail("expected a failure, got a purge count of " + p); - } catch (PathIsNotEmptyDirectoryException expected) { - // expected - } - - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java deleted file mode 100644 index 41760d6..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRMRegistryOperations.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.registry.secure; - - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.PathPermissionException; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.service.ServiceStateException; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException; -import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper; -import org.apache.hadoop.registry.client.impl.RegistryOperationsClient; -import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity; -import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions; -import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; -import org.apache.hadoop.registry.server.services.RegistryAdminService; -import org.apache.zookeeper.client.ZooKeeperSaslClient; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.security.auth.login.LoginException; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; -import java.util.List; - -import static org.apache.hadoop.registry.client.api.RegistryConstants.*; - -/** - * Verify that the {@link RMRegistryOperationsService} works securely - */ -public class TestSecureRMRegistryOperations extends AbstractSecureRegistryTest { - private static final Logger LOG = - LoggerFactory.getLogger(TestSecureRMRegistryOperations.class); - private Configuration secureConf; - private Configuration zkClientConf; - private UserGroupInformation zookeeperUGI; - - @Before - public void setupTestSecureRMRegistryOperations() throws Exception { - startSecureZK(); - secureConf = new Configuration(); - secureConf.setBoolean(KEY_REGISTRY_SECURE, true); - - // create client conf containing the ZK quorum - zkClientConf = new Configuration(secureZK.getConfig()); - zkClientConf.setBoolean(KEY_REGISTRY_SECURE, true); - assertNotEmpty(zkClientConf.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)); - - // ZK is in charge - secureConf.set(KEY_REGISTRY_SYSTEM_ACCOUNTS, "sasl:zookeeper@"); - zookeeperUGI = loginUGI(ZOOKEEPER, keytab_zk); - } - - @After - public void teardownTestSecureRMRegistryOperations() { - } - - /** - * Create the RM registry operations as the current user - * @return the service - * @throws LoginException - * @throws FileNotFoundException - */ - public RMRegistryOperationsService startRMRegistryOperations() throws - LoginException, IOException, InterruptedException { - // kerberos - secureConf.set(KEY_REGISTRY_CLIENT_AUTH, - REGISTRY_CLIENT_AUTH_KERBEROS); - secureConf.set(KEY_REGISTRY_CLIENT_JAAS_CONTEXT, ZOOKEEPER_CLIENT_CONTEXT); - - RMRegistryOperationsService registryOperations = zookeeperUGI.doAs( - new PrivilegedExceptionAction<RMRegistryOperationsService>() { - @Override - public RMRegistryOperationsService run() throws Exception { - RMRegistryOperationsService operations - = new RMRegistryOperationsService("rmregistry", secureZK); - addToTeardown(operations); - operations.init(secureConf); - LOG.info(operations.bindingDiagnosticDetails()); - operations.start(); - return operations; - } - }); - - return registryOperations; - } - - /** - * test that ZK can write as itself - * @throws Throwable - */ - @Test - public void testZookeeperCanWriteUnderSystem() throws Throwable { - - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - RegistryOperations operations = rmRegistryOperations; - operations.mknode(PATH_SYSTEM_SERVICES + "hdfs", - false); - ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true); - LOG.info(pathDumper.toString()); - } - - @Test - public void testAnonReadAccess() throws Throwable { - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - describe(LOG, "testAnonReadAccess"); - RegistryOperations operations = - RegistryOperationsFactory.createAnonymousInstance(zkClientConf); - addToTeardown(operations); - operations.start(); - - assertFalse("RegistrySecurity.isClientSASLEnabled()==true", - RegistrySecurity.isClientSASLEnabled()); - operations.list(PATH_SYSTEM_SERVICES); - } - - @Test - public void testAnonNoWriteAccess() throws Throwable { - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - describe(LOG, "testAnonNoWriteAccess"); - RegistryOperations operations = - RegistryOperationsFactory.createAnonymousInstance(zkClientConf); - addToTeardown(operations); - operations.start(); - - String servicePath = PATH_SYSTEM_SERVICES + "hdfs"; - expectMkNodeFailure(operations, servicePath); - } - - @Test - public void testAnonNoWriteAccessOffRoot() throws Throwable { - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - describe(LOG, "testAnonNoWriteAccessOffRoot"); - RegistryOperations operations = - RegistryOperationsFactory.createAnonymousInstance(zkClientConf); - addToTeardown(operations); - operations.start(); - assertFalse("mknode(/)", operations.mknode("/", false)); - expectMkNodeFailure(operations, "/sub"); - expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true); - } - - /** - * Expect a mknode operation to fail - * @param operations operations instance - * @param path path - * @throws IOException An IO failure other than those permitted - */ - public void expectMkNodeFailure(RegistryOperations operations, - String path) throws IOException { - try { - operations.mknode(path, false); - fail("should have failed to create a node under " + path); - } catch (PathPermissionException expected) { - // expected - } catch (NoPathPermissionsException expected) { - // expected - } - } - - /** - * Expect a delete operation to fail - * @param operations operations instance - * @param path path - * @param recursive - * @throws IOException An IO failure other than those permitted - */ - public void expectDeleteFailure(RegistryOperations operations, - String path, boolean recursive) throws IOException { - try { - operations.delete(path, recursive); - fail("should have failed to delete the node " + path); - } catch (PathPermissionException expected) { - // expected - } catch (NoPathPermissionsException expected) { - // expected - } - } - - @Test - public void testAlicePathRestrictedAnonAccess() throws Throwable { - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - String aliceHome = rmRegistryOperations.initUserRegistry(ALICE); - describe(LOG, "Creating anonymous accessor"); - RegistryOperations anonOperations = - RegistryOperationsFactory.createAnonymousInstance(zkClientConf); - addToTeardown(anonOperations); - anonOperations.start(); - anonOperations.list(aliceHome); - expectMkNodeFailure(anonOperations, aliceHome + "/anon"); - expectDeleteFailure(anonOperations, aliceHome, true); - } - - @Test - public void testUserZookeeperHomePathAccess() throws Throwable { - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - final String home = rmRegistryOperations.initUserRegistry(ZOOKEEPER); - describe(LOG, "Creating ZK client"); - - RegistryOperations operations = zookeeperUGI.doAs( - new PrivilegedExceptionAction<RegistryOperations>() { - @Override - public RegistryOperations run() throws Exception { - RegistryOperations operations = - RegistryOperationsFactory.createKerberosInstance(zkClientConf, - ZOOKEEPER_CLIENT_CONTEXT); - addToTeardown(operations); - operations.start(); - - return operations; - } - }); - operations.list(home); - String path = home + "/subpath"; - operations.mknode(path, false); - operations.delete(path, true); - } - - @Test - public void testUserHomedirsPermissionsRestricted() throws Throwable { - // test that the /users/$user permissions are restricted - RMRegistryOperationsService rmRegistryOperations = - startRMRegistryOperations(); - // create Alice's dir, so it should have an ACL for Alice - final String home = rmRegistryOperations.initUserRegistry(ALICE); - List<ACL> acls = rmRegistryOperations.zkGetACLS(home); - ACL aliceACL = null; - for (ACL acl : acls) { - LOG.info(RegistrySecurity.aclToString(acl)); - Id id = acl.getId(); - if (id.getScheme().equals(ZookeeperConfigOptions.SCHEME_SASL) - && id.getId().startsWith(ALICE)) { - - aliceACL = acl; - break; - } - } - assertNotNull(aliceACL); - assertEquals(RegistryAdminService.USER_HOMEDIR_ACL_PERMISSIONS, - aliceACL.getPerms()); - } - - @Test - public void testDigestAccess() throws Throwable { - RMRegistryOperationsService registryAdmin = - startRMRegistryOperations(); - String id = "username"; - String pass = "password"; - registryAdmin.addWriteAccessor(id, pass); - List<ACL> clientAcls = registryAdmin.getClientAcls(); - LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls)); - - String base = "/digested"; - registryAdmin.mknode(base, false); - List<ACL> baseACLs = registryAdmin.zkGetACLS(base); - String aclset = RegistrySecurity.aclsToString(baseACLs); - LOG.info("Base ACLs=\n{}", aclset); - ACL found = null; - for (ACL acl : baseACLs) { - if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) { - found = acl; - break; - } - } - assertNotNull("Did not find digest entry in ACLs " + aclset, found); - zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS, - "sasl:[email protected], sasl:other"); - RegistryOperations operations = - RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf, - id, - pass); - addToTeardown(operations); - operations.start(); - RegistryOperationsClient operationsClient = - (RegistryOperationsClient) operations; - List<ACL> digestClientACLs = operationsClient.getClientAcls(); - LOG.info("digest client ACLs=\n{}", - RegistrySecurity.aclsToString(digestClientACLs)); - operations.stat(base); - operations.mknode(base + "/subdir", false); - ZKPathDumper pathDumper = registryAdmin.dumpPath(true); - LOG.info(pathDumper.toString()); - } - - @Test(expected = IllegalArgumentException.class) - public void testNoDigestAuthMissingId() throws Throwable { - RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf, - "", - "pass"); - } - - @Test(expected = ServiceStateException.class) - public void testNoDigestAuthMissingId2() throws Throwable { - zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST); - zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, ""); - zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, "pass"); - RegistryOperationsFactory.createInstance("DigestRegistryOperations", - zkClientConf); - } - - @Test(expected = IllegalArgumentException.class) - public void testNoDigestAuthMissingPass() throws Throwable { - RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf, - "id", - ""); - } - - @Test(expected = ServiceStateException.class) - public void testNoDigestAuthMissingPass2() throws Throwable { - zkClientConf.set(KEY_REGISTRY_CLIENT_AUTH, REGISTRY_CLIENT_AUTH_DIGEST); - zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_ID, "id"); - zkClientConf.set(KEY_REGISTRY_CLIENT_AUTHENTICATION_PASSWORD, ""); - RegistryOperationsFactory.createInstance("DigestRegistryOperations", - zkClientConf); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md index 512c011..f13d7d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md @@ -208,7 +208,25 @@ If you are building from source code, make sure you use `-Pyarn-ui` in the `mvn` </property> ``` -# Try with Docker +# Run with security +YARN service framework supports running in a secure(kerberized) environment. User needs to specify the kerberos principal name and keytab when they launch the service. +E.g. A typical configuration looks like below: +``` +{ + "name": "sample-service", + ... + ... + "kerberos_principal" : { + "principal_name" : "[email protected]", + "keytab" : "hdfs:///etc/security/keytabs/hdfs.headless.keytab" + } +} +``` +* principal_name : the principal name of the user who launches the service +* keytab : URI of the keytab. It supports two modes: + * URI starts with `hdfs://`: The URI where the keytab is stored on hdfs. The keytab will be localized to each node by YARN. + * URI starts with `file://`: The URI where the keytab is stored on local host. It is assumed that admin pre-installs the keytabs on the local host before AM launches. +# Run with Docker The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services. Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image. For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.md). http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30d5782/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index c0e12c7..e224e5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -291,6 +291,15 @@ The current state of the container of a service. |----|----|----|----|----| |state|enum of the state of the container|false|enum (INIT, STARTED, READY)|| +### KerberosPrincipal + +The kerberos principal info of the user who launches the service. + +|Name|Description|Required|Schema|Default| +|----|----|----|----|----| +|principal_name|The principal name of the user who launches the service.|false|string|| +|keytab|The URI of the kerberos keytab. It supports two modes, URI starts with "hdfs://": A path on hdfs where the keytab is stored. The keytab will be localized by YARN to each host; URI starts with "file://": A path on the local host where the keytab is stored. It is assumed that the keytabs are pre-installed by admins before AM launches.|false|string|| + ### PlacementPolicy @@ -342,7 +351,7 @@ a service resource has the following attributes. |state|State of the service. Specifying a value for this attribute for the PUT payload means update the service to this desired state.|false|ServiceState|| |quicklinks|A blob of key-value pairs of quicklinks to be exported for a service.|false|object|| |queue|The YARN queue that this service should be submitted to.|false|string|| - +|kerberos_principal | The principal info of the user who launches the service|false|| ### ServiceState --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
