HADOOP-15821. Move YARN Registry to Hadoop Registry.
Contributed by Ãñigo Goiri
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2a9fa84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2a9fa84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2a9fa84
Branch: refs/heads/HDFS-12943
Commit: e2a9fa8448e2aac34c318260e425786a6c8ca2ae
Parents: 00254d7
Author: Eric Yang <[email protected]>
Authored: Fri Oct 19 19:46:48 2018 -0400
Committer: Eric Yang <[email protected]>
Committed: Fri Oct 19 19:46:48 2018 -0400
----------------------------------------------------------------------
dev-support/bin/dist-layout-stitching | 1 +
.../assemblies/hadoop-registry-dist.xml | 41 +
.../hadoop-client-minicluster/pom.xml | 2 +-
.../hadoop-common/src/main/bin/hadoop | 6 +
.../hadoop-common/src/main/conf/hadoop-env.sh | 13 +
.../site/markdown/registry/hadoop-registry.md | 1018 ++++++++++
.../src/site/markdown/registry/index.md | 31 +
.../markdown/registry/registry-configuration.md | 397 ++++
.../src/site/markdown/registry/registry-dns.md | 224 +++
.../site/markdown/registry/registry-security.md | 120 ++
.../using-the-hadoop-service-registry.md | 273 +++
hadoop-common-project/hadoop-registry/pom.xml | 298 +++
.../apache/hadoop/registry/cli/RegistryCli.java | 497 +++++
.../hadoop/registry/client/api/BindFlags.java | 41 +
.../registry/client/api/DNSOperations.java | 60 +
.../client/api/DNSOperationsFactory.java | 78 +
.../registry/client/api/RegistryConstants.java | 388 ++++
.../registry/client/api/RegistryOperations.java | 182 ++
.../client/api/RegistryOperationsFactory.java | 160 ++
.../registry/client/api/package-info.java | 35 +
.../registry/client/binding/JsonSerDeser.java | 117 ++
.../client/binding/RegistryPathUtils.java | 238 +++
.../client/binding/RegistryTypeUtils.java | 291 +++
.../registry/client/binding/RegistryUtils.java | 399 ++++
.../registry/client/binding/package-info.java | 22 +
.../AuthenticationFailedException.java | 39 +
.../exceptions/InvalidPathnameException.java | 40 +
.../exceptions/InvalidRecordException.java | 41 +
.../NoChildrenForEphemeralsException.java | 48 +
.../exceptions/NoPathPermissionsException.java | 45 +
.../client/exceptions/NoRecordException.java | 45 +
.../client/exceptions/RegistryIOException.java | 58 +
.../client/exceptions/package-info.java | 33 +
.../impl/FSRegistryOperationsService.java | 248 +++
.../client/impl/RegistryOperationsClient.java | 55 +
.../registry/client/impl/package-info.java | 26 +
.../client/impl/zk/BindingInformation.java | 41 +
.../registry/client/impl/zk/CuratorService.java | 896 +++++++++
.../registry/client/impl/zk/ListenerHandle.java | 25 +
.../registry/client/impl/zk/PathListener.java | 30 +
.../client/impl/zk/RegistryBindingSource.java | 36 +
.../impl/zk/RegistryInternalConstants.java | 81 +
.../impl/zk/RegistryOperationsService.java | 165 ++
.../client/impl/zk/RegistrySecurity.java | 1143 +++++++++++
.../registry/client/impl/zk/ZKPathDumper.java | 133 ++
.../client/impl/zk/ZookeeperConfigOptions.java | 118 ++
.../registry/client/impl/zk/package-info.java | 39 +
.../registry/client/types/AddressTypes.java | 94 +
.../hadoop/registry/client/types/Endpoint.java | 247 +++
.../registry/client/types/ProtocolTypes.java | 99 +
.../client/types/RegistryPathStatus.java | 123 ++
.../registry/client/types/ServiceRecord.java | 312 +++
.../registry/client/types/package-info.java | 41 +
.../client/types/yarn/PersistencePolicies.java | 50 +
.../types/yarn/YarnRegistryAttributes.java | 42 +
.../registry/conf/RegistryConfiguration.java | 41 +
.../hadoop/registry/conf/package-info.java | 22 +
.../dns/ApplicationServiceRecordProcessor.java | 366 ++++
.../server/dns/BaseServiceRecordProcessor.java | 470 +++++
.../dns/ContainerServiceRecordProcessor.java | 284 +++
.../hadoop/registry/server/dns/LookupTask.java | 39 +
.../dns/PrivilegedRegistryDNSStarter.java | 81 +
.../server/dns/RecordCreatorFactory.java | 275 +++
.../hadoop/registry/server/dns/RegistryDNS.java | 1800 ++++++++++++++++++
.../registry/server/dns/RegistryDNSServer.java | 267 +++
.../registry/server/dns/ReverseZoneUtils.java | 171 ++
.../registry/server/dns/SecureableZone.java | 151 ++
.../server/dns/ServiceRecordProcessor.java | 53 +
.../registry/server/dns/ZoneSelector.java | 33 +
.../registry/server/dns/package-info.java | 26 +
.../integration/SelectByYarnPersistence.java | 60 +
.../server/integration/package-info.java | 23 +
.../hadoop/registry/server/package-info.java | 27 +
.../server/services/AddingCompositeService.java | 56 +
.../services/DeleteCompletionCallback.java | 57 +
.../server/services/MicroZookeeperService.java | 282 +++
.../services/MicroZookeeperServiceKeys.java | 69 +
.../server/services/RegistryAdminService.java | 529 +++++
.../registry/server/services/package-info.java | 39 +
.../src/main/tla/hadoopregistry.tla | 582 ++++++
.../hadoop/registry/AbstractRegistryTest.java | 118 ++
.../hadoop/registry/AbstractZKRegistryTest.java | 113 ++
.../hadoop/registry/RegistryTestHelper.java | 353 ++++
.../hadoop/registry/cli/TestRegistryCli.java | 197 ++
.../client/binding/TestMarshalling.java | 143 ++
.../binding/TestRegistryOperationUtils.java | 56 +
.../client/binding/TestRegistryPathUtils.java | 186 ++
.../client/impl/CuratorEventCatcher.java | 68 +
.../client/impl/TestCuratorService.java | 249 +++
.../impl/TestFSRegistryOperationsService.java | 298 +++
.../client/impl/TestMicroZookeeperService.java | 61 +
.../integration/TestYarnPolicySelector.java | 65 +
.../operations/TestRegistryOperations.java | 331 ++++
.../secure/AbstractSecureRegistryTest.java | 369 ++++
.../registry/secure/KerberosConfiguration.java | 100 +
.../secure/TestRegistrySecurityHelper.java | 211 ++
.../registry/secure/TestSecureLogins.java | 234 +++
.../registry/secure/TestSecureRegistry.java | 213 +++
.../registry/server/dns/TestRegistryDNS.java | 725 +++++++
.../server/dns/TestReverseZoneUtils.java | 89 +
.../server/dns/TestSecureRegistryDNS.java | 44 +
.../test/resources/0.17.172.in-addr.arpa.zone | 36 +
.../src/test/resources/log4j.properties | 63 +
.../src/test/resources/test.private | 32 +
hadoop-common-project/pom.xml | 1 +
hadoop-project/pom.xml | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 +
.../hadoop-yarn/conf/yarn-env.sh | 1 +
.../hadoop-yarn-services-api/pom.xml | 2 +-
.../hadoop-yarn-services-core/pom.xml | 2 +-
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 235 +--
.../apache/hadoop/registry/cli/RegistryCli.java | 497 -----
.../hadoop/registry/client/api/BindFlags.java | 41 -
.../registry/client/api/DNSOperations.java | 60 -
.../client/api/DNSOperationsFactory.java | 78 -
.../registry/client/api/RegistryConstants.java | 390 ----
.../registry/client/api/RegistryOperations.java | 182 --
.../client/api/RegistryOperationsFactory.java | 160 --
.../registry/client/api/package-info.java | 35 -
.../registry/client/binding/JsonSerDeser.java | 117 --
.../client/binding/RegistryPathUtils.java | 238 ---
.../client/binding/RegistryTypeUtils.java | 291 ---
.../registry/client/binding/RegistryUtils.java | 399 ----
.../registry/client/binding/package-info.java | 22 -
.../AuthenticationFailedException.java | 39 -
.../exceptions/InvalidPathnameException.java | 40 -
.../exceptions/InvalidRecordException.java | 41 -
.../NoChildrenForEphemeralsException.java | 48 -
.../exceptions/NoPathPermissionsException.java | 45 -
.../client/exceptions/NoRecordException.java | 45 -
.../client/exceptions/RegistryIOException.java | 58 -
.../client/exceptions/package-info.java | 33 -
.../impl/FSRegistryOperationsService.java | 249 ---
.../client/impl/RegistryOperationsClient.java | 55 -
.../registry/client/impl/package-info.java | 26 -
.../client/impl/zk/BindingInformation.java | 41 -
.../registry/client/impl/zk/CuratorService.java | 896 ---------
.../registry/client/impl/zk/ListenerHandle.java | 25 -
.../registry/client/impl/zk/PathListener.java | 30 -
.../client/impl/zk/RegistryBindingSource.java | 36 -
.../impl/zk/RegistryInternalConstants.java | 81 -
.../impl/zk/RegistryOperationsService.java | 165 --
.../client/impl/zk/RegistrySecurity.java | 1143 -----------
.../registry/client/impl/zk/ZKPathDumper.java | 133 --
.../client/impl/zk/ZookeeperConfigOptions.java | 118 --
.../registry/client/impl/zk/package-info.java | 39 -
.../registry/client/types/AddressTypes.java | 94 -
.../hadoop/registry/client/types/Endpoint.java | 247 ---
.../registry/client/types/ProtocolTypes.java | 99 -
.../client/types/RegistryPathStatus.java | 123 --
.../registry/client/types/ServiceRecord.java | 312 ---
.../registry/client/types/package-info.java | 41 -
.../client/types/yarn/PersistencePolicies.java | 50 -
.../types/yarn/YarnRegistryAttributes.java | 42 -
.../dns/ApplicationServiceRecordProcessor.java | 366 ----
.../server/dns/BaseServiceRecordProcessor.java | 470 -----
.../dns/ContainerServiceRecordProcessor.java | 284 ---
.../hadoop/registry/server/dns/LookupTask.java | 39 -
.../dns/PrivilegedRegistryDNSStarter.java | 80 -
.../server/dns/RecordCreatorFactory.java | 275 ---
.../hadoop/registry/server/dns/RegistryDNS.java | 1800 ------------------
.../registry/server/dns/RegistryDNSServer.java | 267 ---
.../registry/server/dns/ReverseZoneUtils.java | 171 --
.../registry/server/dns/SecureableZone.java | 151 --
.../server/dns/ServiceRecordProcessor.java | 53 -
.../registry/server/dns/ZoneSelector.java | 33 -
.../registry/server/dns/package-info.java | 26 -
.../integration/SelectByYarnPersistence.java | 60 -
.../server/integration/package-info.java | 23 -
.../hadoop/registry/server/package-info.java | 27 -
.../server/services/AddingCompositeService.java | 56 -
.../services/DeleteCompletionCallback.java | 57 -
.../server/services/MicroZookeeperService.java | 282 ---
.../services/MicroZookeeperServiceKeys.java | 69 -
.../server/services/RegistryAdminService.java | 529 -----
.../registry/server/services/package-info.java | 39 -
.../src/main/resources/.keep | 0
.../src/main/tla/yarnregistry.tla | 582 ------
.../hadoop/registry/AbstractRegistryTest.java | 118 --
.../hadoop/registry/AbstractZKRegistryTest.java | 113 --
.../hadoop/registry/RegistryTestHelper.java | 353 ----
.../hadoop/registry/cli/TestRegistryCli.java | 197 --
.../client/binding/TestMarshalling.java | 143 --
.../binding/TestRegistryOperationUtils.java | 56 -
.../client/binding/TestRegistryPathUtils.java | 186 --
.../client/impl/CuratorEventCatcher.java | 68 -
.../client/impl/TestCuratorService.java | 249 ---
.../impl/TestFSRegistryOperationsService.java | 298 ---
.../client/impl/TestMicroZookeeperService.java | 60 -
.../integration/TestYarnPolicySelector.java | 65 -
.../operations/TestRegistryOperations.java | 331 ----
.../secure/AbstractSecureRegistryTest.java | 368 ----
.../registry/secure/KerberosConfiguration.java | 100 -
.../secure/TestRegistrySecurityHelper.java | 211 --
.../registry/secure/TestSecureLogins.java | 231 ---
.../registry/secure/TestSecureRegistry.java | 213 ---
.../registry/server/dns/TestRegistryDNS.java | 725 -------
.../server/dns/TestReverseZoneUtils.java | 89 -
.../server/dns/TestSecureRegistryDNS.java | 44 -
.../test/resources/0.17.172.in-addr.arpa.zone | 36 -
.../src/test/resources/log4j.properties | 63 -
.../src/test/resources/test.private | 32 -
.../hadoop-yarn-server-common/pom.xml | 2 +-
.../hadoop-yarn-server-nodemanager/pom.xml | 2 +-
.../src/site/markdown/registry/index.md | 30 -
.../markdown/registry/registry-configuration.md | 397 ----
.../site/markdown/registry/registry-security.md | 120 --
.../registry/using-the-yarn-service-registry.md | 273 ---
.../src/site/markdown/registry/yarn-registry.md | 1018 ----------
.../src/site/markdown/yarn-service/Overview.md | 2 +-
.../site/markdown/yarn-service/RegistryDNS.md | 200 --
.../markdown/yarn-service/ServiceDiscovery.md | 83 +-
212 files changed, 19496 insertions(+), 19349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-layout-stitching
b/dev-support/bin/dist-layout-stitching
index 584821a..119ee60 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -128,6 +128,7 @@ run cp -p "${ROOT}/README.txt" .
# Remaining projects will copy only libraries which are not present already in
'share' directory.
run copy
"${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
run copy
"${ROOT}/hadoop-common-project/hadoop-nfs/target/hadoop-nfs-${VERSION}" .
+run copy
"${ROOT}/hadoop-common-project/hadoop-registry/target/hadoop-registry-${VERSION}"
.
run copy
"${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" .
run copy
"${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}"
.
run copy
"${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}"
.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-assemblies/src/main/resources/assemblies/hadoop-registry-dist.xml
----------------------------------------------------------------------
diff --git
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-registry-dist.xml
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-registry-dist.xml
new file mode 100644
index 0000000..9221907
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-registry-dist.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3
http://maven.apache.org/xsd/assembly-1.1.3.xsd";>
+ <id>hadoop-registry-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>/share/hadoop/common</outputDirectory>
+ <includes>
+ <include>${project.artifactId}-${project.version}.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>/share/hadoop/common/lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+
+</assembly>
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index 70af1ed..ddebfa9 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -477,7 +477,7 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-registry</artifactId>
+ <artifactId>hadoop-registry</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index fa78ec3..750dca3 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -43,6 +43,7 @@ function hadoop_usage
hadoop_add_subcommand "jnipath" client "prints the java.library.path"
hadoop_add_subcommand "kerbname" client "show auth_to_local principal
conversion"
hadoop_add_subcommand "key" client "manage keys via the KeyProvider"
+ hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
hadoop_add_subcommand "trace" client "view and modify Hadoop tracing
settings"
hadoop_add_subcommand "version" client "print the version"
hadoop_add_subcommand "kdiag" client "Diagnose Kerberos Problems"
@@ -155,6 +156,11 @@ function hadoopcmd_case
key)
HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell
;;
+ registrydns)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+
HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter'
+
HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer'
+ ;;
trace)
HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index 6db085a..64d6bcb 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -437,3 +437,16 @@ esac
#
# For example, to limit who can execute the namenode command,
# export HDFS_NAMENODE_USER=hdfs
+
+
+###
+# Registry DNS specific parameters
+###
+# For privileged registry DNS, user to run as after dropping privileges
+# This will replace the hadoop.id.str Java property in secure mode.
+# export HADOOP_REGISTRYDNS_SECURE_USER=yarn
+
+# Supplemental options for privileged registry DNS
+# By default, Hadoop uses jsvc which needs to know to launch a
+# server jvm.
+# export HADOOP_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-common-project/hadoop-common/src/site/markdown/registry/hadoop-registry.md
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/registry/hadoop-registry.md
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/hadoop-registry.md
new file mode 100644
index 0000000..b5bf9cd
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/hadoop-registry.md
@@ -0,0 +1,1018 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+## The YARN Service Registry
+
+# Introduction and concepts
+
+This document describes a Hadoop service registry built to address two
problems:
+
+1. How can clients talk to YARN-deployed services and the components which form
+such services?
+1. Allow Hadoop core services to be registered and discovered thereby
+reducing configuration parameters and to allow core services to be more
+easily moved.
+
+Service registration and discovery is a long-standing problem in distributed
+computing, dating back to Xerox's Grapevine Service. This proposal is for a
+registry for locating distributed applications deployed by YARN, and
+determining the binding information needed to communicate with these
+applications.
+
+### Definitions
+**Service**: a potentially-distributed application deployed in âor reachable
+fromâ a Hadoop YARN cluster. Examples: Apache HBase, Apache hcatalog, Apache
+Storm. Services may be short-lived or long-lived.
+
+**Service Class:** the name of a type of service, used as a path in a registry
+and matching the DNS-compatible path naming scheme. Examples:
+`org-apache-hbase`, `org-apache-hcatalog`
+
+**Component**: a distributed element of a service. Examples: HBase master
+nodes, HBase region servers and HBase REST servers.
+
+**Service Instance:** A single instance of an application. Example, an HBase
+cluster `demo1`. A service instance is running if the instances the components
+which for the service are running. This does not imply "live" in the
+distributed computing sense, merely that the process are running.
+
+**Component Instance**: a single instance of a component within a service
+instance. Examples: an HBase master node on host `rack1server6` or a region
+server on host `rack3server40`.
+
+**Endpoint**: one means of binding with a service instance or a component
+instance. Examples: HBase's Apache Zookeeper binding, a Java JMX port on a
+region server, a Web UI on an HBase Master, and the REST API of an HBase REST
+component instance. Endpoints may be *internal* âfor use within the service
+instance, or *external*: for use by clients of a service instance.
+
+**Service Record**: a record in the registry describing a service instance or a
+component instance, including listing its endpoints.
+
+**YARN Resource Manager, "RM":** the YARN component which allows client
+applications to submit work (including requests to deploy service instances) to
+a YARN cluster. The RM retains state on all running applications.
+
+**YARN Application**: An application deployed via YARN. Every application
+instance has a unique application ID.
+
+**YARN Application Master, "AM":** the application-specific component which is
+scheduled and deployed by the RM. It has the duty of maintaining the internal
+state of the application, including requesting and managing all other component
+instances of this application instance. The YARN RM will detect the failure of
+the AM, and respond by rescheduling it.
+
+**YARN Container:** An allocation of resources, including CPU and RAM, for a
+component instance. The AM has the responsibility of requesting the containers
+its components need, and building the commands to instantiate its component
+instances onto allocated containers. Every allocated container has a unique
+container ID.
+
+## The binding problem
+Hadoop YARN allows applications to run on the Hadoop cluster. Some of these are
+batch jobs or queries that can managed via YARNâs existing API using its
+application ID. In addition YARN can deploy long-lived services instances such
a
+pool of Apache Tomcat web servers or an Apache HBase cluster. YARN will deploy
+them across the cluster depending on the individual each component requirements
+and server availability. These service instances need to be discovered by
+clients; traditionally their IP added is registered in DNS or in some
+configuration file âbut that is not feasible in YARN-deployed applications
when
+neither the hostname nor network ports can be known in advance.
+
+As a result there is no easy way for clients to interact with dynamically
+deployed applications.
+
+YARN supports a rudimentary registry which allows YARN Application Masters to
+register a web URL and an IPC address. but is not sufficient for our purposes
+since it It does not allow any other *endpoints* to be registered âsuch as
REST
+URLs, or zookeeper path or the endpoints of the tasks that the Application
+Master executes. Further, information that can be registered is mapped to the
+YARN application instance âa unique instance ID that changes whenever a YARN
+application is started. This makes it impossible to resolve binding information
+via a static reference to a named service, or to even probe for the existence
+of a service instance which is not currently live.
+
+# Use Cases
+
+## Service Name Examples
+
+
+Core Hadoop services.
+
+These may be deployed statically, dynamically via an account with the
+permissions to write to the `/services` path, or even registrations of remote
+services accessible from within the Hadoop cluster
+
+ /services/hdfs
+ /services/yarn
+ /services/oozie
+
+YARN-deployed services belonging to individual users.
+
+ /users/joe/org-apache-hbase/demo1
+ /users/joe/org-apache-hbase/demo1/components/regionserver1
+
+## Registration Use Cases
+
+1. A Hadoop core service that is not running under YARN example: HDFS) can be
+registered in for discovery. This could be done by the service or by management
+tools..
+
+2. A long-lived application deployed by YARN registers itself for discovery by
+clients. The registration data is intended to outlive the application master,
+and perhaps the lifespan of a single deployment of the service instance.
+
+3. Component instances of a service register themselves, publishing internal
+binding information, such as JMX ports.
+
+4. A YARN-deployed application can bind to dependent service instances both
+static and dynamic. Example: a Tomcat web pool binding to the dynamic HBase
+service instance "/users/joe/services/hbase/demo1".
+
+5. Component Instances use the registry to bind to an internal endpoint of
+their application master, to which they heartbeat regularly.
+
+## Unsupported Registration use cases:
+
+1. A short-lived YARN application is registered automatically in the registry,
+including all its containers. and unregistered when the job terminates.
+Short-lived applications with many containers will place excessive load on a
+registry. All YARN applications will be given the option of registering, but it
+will not be automatic âand application authors must be advised against
+registering short-lived containers.
+
+## Lookup Use Cases
+
+1. A client application looks up a dynamically deployed service instance whose
+user, service class and instance name is known, e.g.
+`/users/joe/services/hbase/demo1`, and retrieves the information needed to
+connect to the service
+
+2. A client application looks up a statically deployed Hadoop service Example:
+`/services/hdfs`.
+
+3. An Application Master enumerates all registered component instances,
+discovers their listed JMX ports, and, initializes own web UI, offers links to
these
+endpoints.
+
+4. A user connects to a private HBase service instance at
+`/users/joe/services/hbase/demo1`.
+
+5. A user connects to the clusterâs HBase service at `/services/hbase`.
+
+6. A user looks up the binding information to a remote Hadoop cluster's
+filesystem at `/net/cluster4/services/hdfs`. The registration information
+includes the `webhdfs://` URL for the remote filesystem.
+
+7. A user lists their HBase service instances:
+
+ ls /users/joe/services/hbase
+
+8. User finds all Hbase services in the cluster:
+
+ find -endpointField.api=org.apache.hbase
+
+9. Possibly in future: looking up a service via DNS.
+
+This registry proposal is intended to support these use cases by providing a
+means for applications to register their service endpoints, and for clients to
+locate them.
+
+# Key Requirements of a Service Registry
+
+Allow dynamic registration of service instances
+
+ * YARN deployed services instances must be able register their bindings and
be
+ discovered by clients.
+
+ * Core Hadoop service instances must be able to register their service
+ endpoints.
+
+ * The binding must be upgradable if the service moves or in case if HA fails
+ over.
+
+ * A service instance must be able to publish a variety of endpoints for a
+ service: Web UI, RPC, REST, Zookeeper, others. Furthermore one must also be
+ able register certificates and other public security information may be
+ published as part of a binding.
+
+Registry service properties:
+
+ * The registry must be highly available.
+
+ * Scale: many services and many clients in a large cluster. This will limit
+ how much data a service can publish.
+
+ * Ubiquity: we need this in every YARN cluster, whether physical, virtual or
+ in-cloud.
+
+ * Must support hierarchical namespace and names. The name convention must
+ match that of DNS so that we have the option of accessing the namespace via
+ DNS protocol at a later phase of the project.
+
+ * Registry API Language/Protocols
+
+ * Cross-language: independent of any language; client language != service
+
+ * REST API for reading registry data
+
+Access Control:
+
+ * Read access for all
+
+ * Write is restricted so that squatting and impersonation can be avoided.
+
+Remote accessibility: supports remote access even on clusters which are
+ only reachable via Apache Knox, or hosted in cloud environments.
+
+## Non-Requirements
+
+* The registry is not intended for liveness detection, leader-election or
+ perform other "shared consensual state" actions for an application itself,
+ other than potentially sharing binding information between component
+ instances.
+
+* The registry is not intended to be a store for arbitrary application state,
+ or for publishing configuration data other than binding information to
+ endpoints offered by a service and its components. Such use would overload
+ the registry and rapidly reach limits of what Zookeeper permits.
+
+# Architecture
+
+We propose a base registry service that binds string-names to records
+describing service and component instances. We plan to use ZK as the base name
+service since it supports many of the properties, We pick a part of the ZK
+namespace to be the root of the service registry ( default: `yarnRegistry`).
+
+On top this base implementation we build our registry service API and the
+naming conventions that YARN will use for its services. The registry will be
+accessed by the registry API, not directly via ZK - ZK is just an
+implementation choice (although unlikely to change in the future).
+
+1. Services are registered by binding a **_path_** to a value called a
+**_Service Record_**. Paths are hierarchical and use `/` as the root as well as
+the separator.
+
+2. Service records are registered as persistent znodes. This ensures that the
+record remains present during planned and unplanned outages of the service, on
+the assumption that client code is resilient to transient outages.
+
+3. Each service instanceâs service record lists the endpoints for its various
+protocols exported by that service instance.
+
+4. For each protocol endpoint it must contain
+
+ 1. The *protocol* name including: Web, REST, IPC, zookeeper. (type:string)
+
+ 2. Its *address*: the specific details used to locate this endpoint
+
+ 3. Its *addressType*. This is the format of the binding string. (URL, ZK
path,
+ hostname:port pair). For the predefined protocols, we will define what
+ format the binding string MUST be. Example: `protocol==REST` means
binding
+ type is `URL`, `protocol==IPC` binding uses the addresstype
`host/port`.
+
+ 4. The *api*. This is the API offered by the endpoint, and is application
+ specific. examples: `org.apache.hadoop.namenode`,
+ `org.apache.hadoop.webhdfs`
+
+5. Endpoints may be *external* âfor use by programs other than the service
+itself, and *internal* âfor connecting components within the service
instance.
+They will be listed in different sections of the Service Record to distinguish
+them.
+
+6. Core services will be registered using the following convention:
+`/services/{servicename}` e.g. `/services/hdfs`.
+
+7. YARN services SHOULD be registered using the following convention:
+
+ /users/{username}/{serviceclass}/{instancename}
+
+6. Component instances SHOULD be registered under
+
+
/users/{username}/{serviceclass}/{instancename}/components/{componentname}
+
+
+8. Each of the user's services which follows this convention must have unique
+service class names,
+
+9. Each component instance must have a name that is unique for that service
+instance. For a YARN-deployed application, this can be trivially
+derived from the container ID.
+
+The requirements for unique names ensures that the path to a service instance
+or component instance is guaranteed to be unique, and that all instances of a
+specific service class can be enumerated by listing all children of the service
+class path.
+
+
+# Registry Model
+
+Service entries MUST be persistent âit is the responsibility of YARN and
other
+tools to determine when a service entry is to be deleted.
+
+## Path Elements
+
+All path elements MUST match that of a lower-case entry in a hostname path as
+defined in RFC1123; the regular expression is:
+
+ ([a-z0-9]|([a-z0-9][a-z0-9\-]*[a-z0-9]))
+
+This policy will ensure that were the registry hierarchy ever to exported by a
+DNS service, all service classes and names would be valid.
+
+A complication arises with user names, as platforms may allow user names with
+spaces, high unicode and other characters in them. Such paths must be converted
+to valid DNS hostname entries using the punycode convention used for
+internationalized DNS.
+
+## Service Record
+
+A Service Record has some basic information and possibly empty lists of
+internal and external endpoints.
+
+### Service Record:
+
+A Service Record contains some basic informations and two lists of endpoints:
+one list for users of a service, one list for internal use within the
+application.
+
+<table>
+ <tr>
+ <td>Name</td>
+ <td>Description</td>
+ </tr>
+ <tr>
+ <td>type: String</td>
+ <td>Always: "JSONServiceRecord"</td>
+ </tr>
+ <tr>
+ <td>description: String</td>
+ <td>Human-readable description.</td>
+ </tr>
+ <tr>
+ <td>external: List[Endpoint]</td>
+ <td>A list of service endpoints for external callers.</td>
+ </tr>
+ <tr>
+ <td>internal: List[Endpoint]</td>
+ <td>A list of service endpoints for internal use within the service
instance.</td>
+ </tr>
+</table>
+
+The type field MUST be `"JSONServiceRecord"`. Mandating this string allows
+future record types *and* permits rapid rejection of byte arrays that
+lack this string before attempting to parse the data with a JSON parser.
+
+### YARN Persistence policies
+
+The attributes, `yarn:id` and `yarn:persistence` specify which records
+*and any child entries* may be deleted as the associated YARN components
complete.
+
+The `yarn:id` field defines the application, attempt or container ID to match;
+the `yarn:persistence` attribute defines the trigger for record cleanup, and
+implicitly the type of the contents of the `yarn:id` field.
+
+These attributes use the prefix "`yarn:`" to indicate that their reliance on
+the YARN layer of the Hadoop cluster to implement the policy. If the registry
+were to run standalone âwhich is entirely possibleâ all records would be
+implicitly persistent.
+
+<table>
+ <tr>
+ <td>Name</td>
+ <td>Description</td>
+ <td>contents of `yarn:id` field</td>
+ </tr>
+ <tr>
+ <td>permanent</td>
+ <td>The record persists until removed manually.</td>
+ <td>(unused)</td>
+ </tr>
+ <tr>
+ <td>application</td>
+ <td>Remove when the YARN application defined in the id field
terminates.</td>
+ <td>application ID</td>
+ </tr>
+ <tr>
+ <td>application-attempt</td>
+ <td>Remove when the current YARN application attempt finishes.</td>
+ <td>application attempt ID</td>
+ </tr>
+ <tr>
+ <td>container</td>
+ <td>Remove when the YARN container in the ID field finishes</td>
+ <td>container ID</td>
+ </tr>
+</table>
+
+
+The policies which clean up when an application, application attempt or
+container terminates require the `yarn:id` field to match that of the
+application, attempt or container. If the wrong ID is set, the cleanup does not
+take place âand if set to a different application or container, will be
cleaned
+up according the lifecycle of that application.
+
+### Endpoint:
+
+<table>
+ <tr>
+ <td>Name</td>
+ <td>Description</td>
+ </tr>
+ <tr>
+ <td>api: URI as String</td>
+ <td>API implemented at the end of the binding</td>
+ <tr>
+ <td>protocol: String</td>
+ <td>Protocol. Examples:
+`http`, `https`, `hadoop-rpc`, `zookeeper`, `web`, `REST`, `SOAP`, ...</td>
+ </tr>
+ <tr>
+ <td>addressType: String</td>
+ <td>format of the binding</td>
+ </tr>
+ </tr>
+ <tr>
+ <td>addresses: List[Map[String, String]]</td>
+ <td>a list of address maps</td>
+ </tr>
+
+</table>
+
+
+All string fields have a limit on size, to dissuade services from hiding
+complex JSON structures in the text description.
+
+#### Field `addressType`: Address Type
+
+The `addressType` field defines the string format of entries.
+
+Having separate types is that tools (such as a web viewer) can process binding
+strings without having to recognize the protocol.
+
+<table>
+ <tr>
+ <td>Format</td>
+ <td>binding format</td>
+ </tr>
+ <tr>
+ <td>uri</td>
+ <td>uri:URI of endpoint</td>
+ </tr>
+ <tr>
+ <td>hostname</td>
+ <td>hostname: service host</td>
+ </tr>
+ <tr>
+ <td>inetaddress</td>
+ <td>hostname: service host, port: service port</td>
+ </tr>
+ <tr>
+ <td>path</td>
+ <td>path: generic unix filesystem path</td>
+ </tr>
+ <tr>
+ <td>zookeeper</td>
+ <td>hostname: service host, port: service port, path: ZK path</td>
+ </tr>
+</table>
+
+
+In the zookeeper binding, every entry represents a single node in quorum,
+the `hostname` and `port` fields defining the hostname of the ZK instance
+and the port on which it is listening. The `path` field lists zookeeper path
+for applications to use. For example, for HBase this would refer to the znode
+containing information about the HBase cluster.
+
+The path MUST be identical across all address elements in the `addresses` list.
+This ensures that any single address contains enough information to connect
+to the quorum and connect to the relevant znode.
+
+New Address types may be defined; if not standard please prefix with the
+character sequence `"x-"`.
+
+### Field `api`: API identifier
+
+The API field MUST contain a URI that identifies the specific API of an
endpoint.
+These MUST be unique to an API to avoid confusion.
+
+The following strategies are suggested to provide unique URIs for an API
+
+1. The SOAP/WS-* convention of using the URL to where the WSDL defining the
service
+2. A URL to the svn/git hosted document defining a REST API
+3. the `classpath` schema followed by a path to a class or package in an
application.
+4. The `uuid` schema with a generated UUID.
+
+It is hoped that standard API URIs will be defined for common APIs. Two such
non-normative APIs are used in this document
+
+* `http://` : A web site for humans
+* `classpath:javax.management.jmx`: and endpoint supporting the JMX management
protocol (RMI-based)
+
+
+### Examples of Service Entries
+
+Here is an example of a service entry for a YARN-deployed tomcat application.
+
+After creation and registration of the application, the registry looks as
follows:
+
+ /users
+ /devteam
+ /org-apache-tomcat
+ /test1
+ /components
+ /container-1408631738011-0001-01-000002
+ /container-1408631738011-0001-01-000001
+
+The `/users/devteam/org-apache-tomcat/tomcat-test` service record describes the
+overall application. It exports the URL to a load balancer.
+
+ {
+ "description" : "tomcat-based web application",
+ "external" : [ {
+ "api" : "http://internal.example.org/restapis/scheduler/20141026v1";,
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [
+ { "uri" : "http://loadbalancer/"; },
+ { "uri" : "http://loadbalancer2/"; }
+ ]
+ } ],
+ "internal" : [ ]
+ }
+
+The service instance is built from two component instances, each described with
+their container ID converted into a DNS-compatible hostname. The entries are
+marked as ephemeral. If the entries were set within the container, then when
+that container is released or if the component fails, the entries will be
+automatically removed. Accordingly, it's persistence policy is declared to be
+"3", container. The `yarn:id` field identifies the container whose completion
+will trigger the deletion of this entry
+
+
/users/devteam/org-apache-tomcat/test1/components/container-1408631738011-0001-01-000001
+
+ {
+ "yarn:id" : "container_1408631738011_0001_01_000001",
+ "yarn:persistence" : "container",
+ "description" : "",
+ "external" : [ {
+ "api" : "http://internal.example.org/restapis/scheduler/20141026v1";,
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [{ "uri" : "rack4server3:43572" } ]
+ } ],
+ "internal" : [ {
+ "api" : "classpath:javax.management.jmx",
+ "addressType" : "host/port",
+ "protocol" : "rmi",
+ "addresses" : [ {
+ "host" : "rack4server3",
+ "port" : "48551"
+ } ]
+ } ]
+ }
+
+The component instances list their endpoints: the public REST API as an
+external endpoint, the JMX addresses as internal.
+
+
/users/devteam/org-apache-tomcat/test1/components/container-1408631738011-0001-01-000002
+
+ {
+ "registrationTime" : 1408638082445,
+ "yarn:id" : "container_1408631738011_0001_01_000002",
+ "yarn:persistence" : "container",
+ "description" : null,
+ "external" : [ {
+ "api" : "http://internal.example.org/restapis/scheduler/20141026v1";,
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ [ "http://rack1server28:35881"; ] ]
+ } ],
+ "internal" : [ {
+ "api" : "classpath:javax.management.jmx",
+ "addressType" : "host/port",
+ "protocol" : "rmi",
+ "addresses" : [ {
+ "host" : "rack1server28",
+ "port" : "48551"
+ } ]
+ } ]
+ }
+
+This information could be used by the (hypothetical) load balancer to enumerate
+the components and build a list of component instances to dispatch requests to.
+Similarly, a management application could enumerate all available component
+instances and their JMX ports, then connect to each to collect performance
+metrics.
+
+# Registry API
+
+Here is the registry API as seen from a Java application. The API is a thin
+layer above the ZK operations, essentially building up paths, reading, writing
+and updating entries, and enumerating children. The REST API is implemented
+inside a server and use this same API to implement its REST API.
+
+The exceptions that are listed are only a subset of possible exception âthe
+interface merely lists those that have special meaning.
+
+All write operations must assume that they are communicating with a registry
+service with the consistency view of a Zookeeper client; read-only clients must
+assume that their view may be somewhat out of date.
+
+All clients must recognize that the registry is a shared resource and that
+it may change during a sequence of actions.
+
+### Registry Operations
+
+ public interface RegistryOperations extends Service {
+
+ /**
+ * Create a path.
+ *
+ * It is not an error if the path exists already, be it empty or not.
+ *
+ * The createParents flag also requests creating the parents.
+ * As entries in the registry can hold data while still having
+ * child entries, it is not an error if any of the parent path
+ * elements have service records.
+ *
+ * @param path path to create
+ * @param createParents also create the parents.
+ * @throws PathNotFoundException parent path is not in the registry.
+ * @throws InvalidPathnameException path name is invalid.
+ * @throws IOException Any other IO Exception.
+ * @return true if the path was created, false if it existed.
+ */
+ boolean mknode(String path, boolean createParents)
+ throws PathNotFoundException,
+ InvalidPathnameException,
+ IOException;
+
+ /**
+ * Set a service record to an entry
+ * @param path path to service record
+ * @param record service record service record to create/update
+ * @param createFlags creation flags
+ * @throws PathNotFoundException the parent path does not exist
+ * @throws FileAlreadyExistsException path exists but create flags
+ * do not include "overwrite"
+ * @throws InvalidPathnameException path name is invalid.
+ * @throws IOException Any other IO Exception.
+ */
+ void bind(String path, ServiceRecord record, int createFlags)
+ throws PathNotFoundException,
+ FileAlreadyExistsException,
+ InvalidPathnameException,
+ IOException;
+
+ /**
+ * Resolve the record at a path
+ * @param path path to service record
+ * @return the record
+ * @throws PathNotFoundException path is not in the registry.
+ * @throws InvalidPathnameException the path is invalid.
+ * @throws IOException Any other IO Exception
+ */
+
+ ServiceRecord resolve(String path) throws PathNotFoundException,
+ InvalidPathnameException,
+ IOException;
+
+ /**
+ * Get the status of a path
+ * @param path path to query
+ * @return the status of the path
+ * @throws PathNotFoundException path is not in the registry.
+ * @throws InvalidPathnameException the path is invalid.
+ * @throws IOException Any other IO Exception
+ */
+ RegistryPathStatus stat(String path)
+ throws PathNotFoundException,
+ InvalidPathnameException,
+ IOException;
+
+ /**
+ * Probe for a path existing.
+ * This is equivalent to {@link #stat(String)} with
+ * any failure downgraded to a
+ * @param path path to query
+ * @return true if the path was found
+ * @throws IOException
+ */
+ boolean exists(String path) throws IOException;
+
+ /**
+ * List all entries under a registry path
+ * @param path path to query
+ * @return a possibly empty list of the full path names of
+ * child entries
+ * @throws PathNotFoundException
+ * @throws InvalidPathnameException
+ * @throws IOException
+ */
+ List<String> list(String path) throws
+ PathNotFoundException,
+ InvalidPathnameException,
+ IOException;
+
+ /**
+ * Delete a path.
+ *
+ * If the operation returns without an error then the entry has been
+ * deleted.
+ * @param path path delete recursively
+ * @param recursive recursive flag
+ * @throws PathNotFoundException path is not in the registry.
+ * @throws InvalidPathnameException the path is invalid.
+ * @throws PathIsNotEmptyDirectoryException path has child entries, but
+ * recursive is false.
+ * @throws IOException Any other IO Exception
+ *
+ */
+ void delete(String path, boolean recursive)
+ throws PathNotFoundException,
+ PathIsNotEmptyDirectoryException,
+ InvalidPathnameException,
+ IOException;
+
+ /**
+ * Add a new write access entry to be added to node permissions in all
+ * future write operations of a session connected to a secure registry.
+ *
+ * This does not grant the session any more rights: if it lacked any
write
+ * access, it will still be unable to manipulate the registry.
+ *
+ * In an insecure cluster, this operation has no effect.
+ * @param id ID to use
+ * @param pass password
+ * @return true if the accessor was added: that is, the registry
connection
+ * uses permissions to manage access
+ * @throws IOException on any failure to build the digest
+ */
+ boolean addWriteAccessor(String id, String pass) throws IOException;
+
+ /**
+ * Clear all write accessors.
+ *
+ * At this point all standard permissions/ACLs are retained,
+ * including any set on behalf of the user
+ * Only accessors added via {@link #addWriteAccessor(String, String)}
+ * are removed.
+ */
+ public void clearWriteAccessors();
+ }
+
+
+### `RegistryPathStatus`
+
+The `RegistryPathStatus` class summarizes the contents of a node in the
registry.
+
+ public class RegistryPathStatus {
+
+ /**
+ * Short path in the registry to this entry
+ */
+ public String path;
+
+ /**
+ * Timestamp
+ */
+ public long time;
+
+ /**
+ * Entry size in bytes, as returned by the storage infrastructure.
+ * In zookeeper, even "empty" nodes have a non-zero size.
+ */
+ public long size;
+
+ /**
+ * Number of child nodes
+ */
+ public int children;
+ }
+
+
+## Security
+
+The registry will allow a service instance can only be registered under the
+path where it has permissions. YARN will create directories with appropriate
+permissions for users where YARN deployed services can be registered by a user.
+of the user account of the service instance. The admin will also create
+directories (such as `/services`) with appropriate permissions (where core
Hadoop
+services can register themselves.
+
+There will no attempt to restrict read access to registry information. The
+services will protect inappropriate access by clients by requiring
+authentication and authorization. There is a *scope* field in a service record
+, but this is just a marker to say "internal API only", rather than a direct
+security restriction. (this is why "internal" and "external" are proposed, not
+"public" and "private").
+
+Rationale: the endpoints being registered would be discoverable through port
+scanning anyway. Having everything world-readable allows the REST API to have a
+simpler access model âand is consistent with DNS.
+
+On a secure cluster, ZK token renewal may become an issue for long-lived
+services âif their token expires their session may expire. Renewal of such
+tokens is not part of the API implementation âwe may need to add a means to
+update the tokens of an instance of the registry operations class.
+
+### Security Policy Summary
+
+In an a non-Kerberos Zookeeper Cluster, no security policy is implemented.
+
+The registry is designed to be secured *on a kerberos-managed cluster*.
+
+* The registry root grants full rights to "system accounts":
+`mapred`, `hdfs`, `yarn` : `"rwcda"`; all other accounts, and anonymous access
+is read-only.
+
+* The permissions are similarly restricted for `/users`, and `/services/`
+
+* installations may extend or change these system accounts.
+
+* When an application belonging to a user is scheduled, YARN
+SHALL create an entry for that user `/users/${username}`.
+
+* This node will have full access to the system; the user the access rights:
+`"crd"`. That is, they may create or delete child nodes, but not write to
+their home node, âor alter its permissions.
+
+* Applications wishing to write to the registry must use a SASL connection
+to authenticate via Zookeeper,
+
+* Applications creating nodes in the user path MUST include the site-specified
+system accounts in the ACL list, with full access.
+
+* Applications creating nodes in the user path MUST include an ACL Which
+
+* Applications creating nodes in the user path MUST declare their own
+user identity as a `sasl:user@REALM` entry.
+
+* Applications creating nodes the user path MAY add extra `digest:` ACL tokens
+so as to give their services the ability to manipulate portions of the
+registry *without needing kerberos credentials*.
+
+The digest-driven authentication avoid the problem of credential renewal in
+long-lived applications. An YARN application may be passed the token to
+connect with the ZK service when launched. It can then create or update an
+entry, including a secret digest ACL in the permissions of nodes it creates.
+As a result, even after the credentials expire, it retains *some* access.
+
+Note that for this to be successful, the client will need to fall back
+session to *not* use SASL, instead using authentication id:pass credentials.
+
+
+## Out of cluster and cross-cluster access
+
+1. A client should be able to access the registry of another cluster in order
+to access services of that cluster. Detail of this need to further fleshed out.
+
+2. Firewall services such as Apache Knox can examine the internal set of
+published services, and publish a subset of their endpoints. They MAY implement
+a future REST API.
+
+# Limits
+
+**Entry Size**
+
+Zookeeper has a default limit of 1MB/node. If all endpoints of a service or
+component are stored in JSON attached to that node, then there is a total limit
+of 1MB of all endpoint registration data.
+
+To prevent this becoming a problem, the client API should implement strict
+limits on the maximum length of fields, with low limits on the addressType,
+protocol, and api fields, something longer on the description and addresses
+elements âalong with a limit on the number of elements in the addresses
field.
+
+**Name size**
+
+To support DNS in future, there must be a limit of 63 bytes on all path
+elements. For non-ASCII User names, this restriction implies that a shorter
+path may be a limit.
+
+**Rate of Update**
+
+A rapid rate of entry change is considered antisocial in a ZK cluster.
+Implementations may throttle update operations.
+
+**Rate of Polling**
+
+Clients which poll the registry may be throttled.
+
+# Complete service record example
+
+Below is a (non-normative) example of a service record retrieved
+from a YARN application.
+
+
+ {
+ "type" : "JSONServiceRecord",
+ "description" : "Slider Application Master",
+ "yarn:persistence" : "application",
+ "yarn:id" : "application_1414052463672_0028",
+ "external" : [ {
+ "api" : "classpath:org.apache.slider.appmaster",
+ "addressType" : "host/port",
+ "protocol" : "hadoop/IPC",
+ "addresses" : [ {
+ "port" : "48551",
+ "host" : "nn.example.com"
+ } ]
+ }, {
+ "api" : "http://";,
+ "addressType" : "uri",
+ "protocol" : "web",
+ "addresses" : [ {
+ "uri" : "http://nn.example.com:40743";
+ } ]
+ }, {
+ "api" : "classpath:org.apache.slider.management",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "http://nn.example.com:40743/ws/v1/slider/mgmt";
+ } ]
+ }, {
+ "api" : "classpath:org.apache.slider.publisher",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "http://nn.example.com:40743/ws/v1/slider/publisher";
+ } ]
+ }, {
+ "api" : "classpath:org.apache.slider.registry",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "http://nn.example.com:40743/ws/v1/slider/registry";
+ } ]
+ }, {
+ "api" : "classpath:org.apache.slider.publisher.configurations",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "http://nn.example.com:40743/ws/v1/slider/publisher/slider";
+ } ]
+ }, {
+ "api" : "classpath:org.apache.slider.publisher.exports",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "http://nn.example.com:40743/ws/v1/slider/publisher/exports";
+ } ]
+ } ],
+ "internal" : [ {
+ "api" : "classpath:org.apache.slider.agents.secure",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "https://nn.example.com:52705/ws/v1/slider/agents";
+ } ]
+ }, {
+ "api" : "classpath:org.apache.slider.agents.oneway",
+ "addressType" : "uri",
+ "protocol" : "REST",
+ "addresses" : [ {
+ "uri" : "https://nn.example.com:33425/ws/v1/slider/agents";
+ } ]
+ } ]
+ }
+
+It publishes a number of endpoints, both internal and external.
+
+External:
+
+1. The IPC hostname and port for client-AM communications
+1. URL to the AM's web UI
+1. A series of REST URLs under the web UI for specific application services.
+The details are irrelevant ânote that they use an application-specific API
+value to ensure uniqueness.
+
+Internal:
+
+1. Two URLs to REST APIs offered by the AM for containers deployed by
+ the application itself.
+
+Python agents running in the containers retrieve the internal endpoint
+URLs to communicate with their AM. The record is resolved on container startup
+and cached until communications problems occur. At that point the registry is
+queried for the current record, then an attempt is made to reconnect to the AM.
+
+Here "connectivity" problems means both "low level socket/IO errors" and
+"failures in HTTPS authentication". The agents use two-way HTTPS authentication
+âif the AM fails and another application starts listening on the same ports
+it will trigger an authentication failure and hence service record reread.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-common-project/hadoop-common/src/site/markdown/registry/index.md
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/registry/index.md
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/index.md
new file mode 100644
index 0000000..59ebaa9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/registry/index.md
@@ -0,0 +1,31 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Hadoop Service Registry
+
+The Service registry is a service which can be deployed in a Hadoop cluster
+to allow deployed applications to register themselves and the means of
+communicating with them. Client applications can then locate services
+and use the binding information to connect with the services's
network-accessible
+endpoints, be they REST, IPC, Web UI, Zookeeper quorum+path or some other
protocol.
+Currently, all the registry data is stored in a zookeeper cluster.
+
+* [Architecture](hadoop-registry.html)
+* [Configuration](registry-configuration.html)
+* [Using the Hadoop Service registry](using-the-hadoop-service-registry.html)
+* [Security](registry-security.html)
+* [Registry DNS](registry-dns.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
new file mode 100644
index 0000000..e2646e0
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
@@ -0,0 +1,397 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Registry Configuration
+
+
+The Hadoop service registry is built on top of Apache Zookeeper.
+It is configured by way of a Hadoop `Configuration` class:
+the instance used to create the service controls the behavior of the client.
+
+This document lists the configuration parameters which control the
+registry client.
+
+The default values of all these settings are defined in `core-default.xml`.
+The values in this file may not match those listed in this document.
+If this is the case, the values in `core-default.xml` MUST be considered
normative.
+
+## Changing the configuration values
+
+Changes to the configuration values SHOULD be done in `core-site.xml`.
+This will ensure that client and non-YARN applications will pick up
+the values, so enabling them to read from and potentially write to the
registry.
+
+
+## Core Settings
+
+
+### Setting the Zookeeper Quorum: `hadoop.registry.zk.quorum`
+
+This is an essential setting: it identifies the lists of zookeeper hosts
+and the ports on which the ZK services are listening.
+
+
+```
+ <property>
+ <description>
+ A comma separated list of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+```
+
+It takes a comma-separated list, such as `zk1:2181 ,zk2:2181, zk3:2181`
+
+### Setting the Zookeeper Registry Base path: `hadoop.registry.zk.root`
+
+This path sets the base zookeeper node for the registry
+
+```
+ <property>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ </property>
+```
+
+The default value of `/registry` is normally sufficient. A different value
+may be needed for security reasons or because the `/registry` path is in use.
+
+The root value is prepended to all registry paths so as to create the absolute
+path. For example:
+
+* `/` maps to `/registry`
+* `/services` maps to `/registry/services`
+* `/users/yarn` maps to `/registry/users/yarn`
+
+A different value of `hadoop.registry.zk.root` would result in a different
+mapping to absolute zookeeper paths.
+
+
+## Security Options
+
+Registry security is enabled when the property `hadoop.registry.secure`
+is set to `true`. Once set, nodes are created with permissions, so that
+only a specific user *and the configured cluster "superuser" accounts*
+can write under their home path of `${hadoop.registry.zk.root}/users`.
+Only the superuser accounts
+will be able to manipulate the root path, including
`${hadoop.registry.zk.root}/services`
+and `${hadoop.registry.zk.root}/users`.
+
+All write operations on the registry (including deleting entries and paths)
+must be authenticated. Read operations are still permitted by unauthenticated
+callers.
+
+The key settings for secure registry support are:
+
+* enabling the secure mode: `hadoop.registry.secure`
+* listing the superuser zookeeper ACLs: `hadoop.registry.system.acls`
+* listing the kerberos realm for the principals:
`hadoop.registry.kerberos.realm`
+* identifying the JAAS context within the JAAS configuration which defines
+the user: `hadoop.registry.jaas.context`
+
+
+### Enabling security
+
+```
+ <property>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ </property>
+```
+
+### Identifying the client JAAS context
+
+The registry clients must identify the JAAS context which they use
+to authenticate to the registry.
+
+```
+ <property>
+ <description>
+ Key to define the JAAS context. Used in secure mode
+ </description>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ </property>
+```
+
+*Note* as the Resource Manager is simply another client of the registry, it
+too must have this context defined.
+
+
+### Identifying the system accounts `hadoop.registry.system.acls`
+
+These are the the accounts which are given full access to the base of the
+registry. The Resource Manager needs this option to create the root paths.
+
+Client applications writing to the registry access to the nodes it creates.
+
+1. The property `hadoop.registry.system.acls` takes a comma-separated list
+of zookeeper `ACLs` which are given full access to created nodes; the
permissions
+`READ | WRITE | CREATE | DELETE | ADMIN`.
+2. Any zookeeper ACL scheme may be added to this, such as the `digest:` scheme.
+3. The SASL scheme, `sasl:`, is used to identify which callers identified
+by sasl have full access. These are the superuser accounts.
+4. They may be identified by elements such as `sasl:[email protected]`.
+5. To aid portability of SASL settings, especially the default value,
+any `sasl:` entry without the realm value âthat
+is, any entry that terminates in the `@` symbolâ has the current realm
appended
+to it.
+6. This realm is set to that of the current user.
+7. It may be overridden by the property `hadoop.registry.kerberos.realm`.
+
+
+```
+ <property>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+ These are given full access to all entries.
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:mapred@, sasl:hdfs@</value>
+ </property>
+
+ <property>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+ If empty, the default realm of the running process
+ is used.
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ </property>
+```
+
+Example: an `hadoop.registry.system.acls` entry of
+ `sasl:yarn@, sasl:[email protected], sasl:system@REALM2`,
+would, in a YARN cluster with the realm `EXAMPLE.COM`, add the following
+admin accounts to every node
+
+* `sasl:[email protected]`
+* `sasl:[email protected]`
+* `sasl:system@REALM2`
+
+The identity of a client application creating registry entries will be
+automatically included in the permissions of all entries created.
+If, for example, the account creating an entry was `hbase`, another
+entry would be created
+
+* `sasl:[email protected]`
+
+
+**Important**: when setting the system ACLS, *it is critical to include the
+identity of the YARN Resource Manager*.
+
+The RM needs to be able to create the root and user paths, and delete service
+records during application and container cleanup.
+
+
+## Zookeeper connection management options
+
+Some low level options manage the ZK connection âmore specifically, its
failure
+handling.
+
+The Zookeeper registry clients use Apache Curator to connect to Zookeeper,
+a library which detects timeouts and attempts to reconnect to one of the
+servers which forms the zookeeper quorum. It is only after a timeout is
detected
+that a retry is triggered.
+
+```
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>
+ </description>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff.
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ </property>
+```
+
+The retry strategy used in the registry client is
+[`BoundedExponentialBackoffRetry`](https://curator.apache.org/apidocs/org/apache/curator/retry/BoundedExponentialBackoffRetry.html):
+This backs off exponentially on connection failures before eventually
+concluding that the quorum is unreachable and failing.
+
+## Complete Set of Configuration Options
+
+```
+ <!-- YARN registry -->
+
+ <property>
+ <description>
+ A comma separated list of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+
+ <property>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ </property>
+
+ <property>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+
+ These are given full access to all entries.
+
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:mapred@, sasl:hdfs@</value>
+ </property>
+
+ <property>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+
+ If empty, the default realm of the running process
+ is used.
+
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
+ Key to define the JAAS context. Used in secure
+ mode
+ </description>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ </property>
+
+
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>
+ </description>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff: {@value}
+
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ </property>
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2a9fa84/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-dns.md
----------------------------------------------------------------------
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-dns.md
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-dns.md
new file mode 100644
index 0000000..e02788c
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-dns.md
@@ -0,0 +1,224 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Registry DNS Server
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+The document describes the internals of Registry DNS server. It is based on
the [Hadoop service registry](../registry/index.html) which is backed by a
zookeeper cluster.
+## Introduction
+
+The Registry DNS Server provides a standard DNS interface to the information
posted into the Hadoop Registry by deployed applications. The DNS service
serves the following functions:
+
+1. **Exposing existing service-discovery information via DNS** - Information
provided in
+the current Hadoop service registryâs records will be converted into DNS
entries, thus
+allowing users to discover information about YARN applications using standard
DNS
+client mechanisms (e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of
containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+name-based communication will be facilitated (e.g. `curl
+http://solr-0.solr-service.devuser.yarncluster:8983/solr/admin/collections?action=LIST`).
+
+## Service Properties
+
+The existing Hadoop Service Registry is leveraged as the source of information
for the DNS Service.
+
+The following core functions are supported by the DNS-Server:
+
+### Functional properties
+
+1. Supports creation of DNS records for end-points of the deployed YARN
applications
+2. Record names remain unchanged during restart of containers and/or
applications
+3. Supports reverse lookups (name based on IP). Note, this works only for
+Docker containers because other containers share the IP of the host
+4. Supports security using the standards defined by The Domain Name System
Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable - The service provides the responsiveness (e.g. low-latency)
required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+### Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server)
by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to
act as a
+primary DNS server and does not forward requests to other servers. Rather, a
+primary DNS server can be configured to forward a zone to the registry DNS
+server.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is not in the restricted
+range (5335). However, existing DNS assets may only allow zone forwarding to
+non-custom ports. To support this, the registry DNS server can be started in
+privileged mode.
+
+## DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements
+(labels). Note that these elements must be compatible with DNS conventions
+(see âPreferred Name Syntaxâ in [RFC
1035](https://www.ietf.org/rfc/rfc1035.txt)):
+
+* **domain** - the name of the cluster DNS domain. This name is provided as a
+configuration property. In addition, it is this name that is configured at a
parent DNS
+server as the zone name for the defined registry DNS zone (the zone for which
+the parent DNS server will forward requests to registry DNS). E.g.
yarncluster.com
+* **username** - the name of the application deployer. This name is the simple
short-name (for
+e.g. the primary component of the Kerberos principal) associated with the user
launching
+the application. As the username is one of the elements of DNS names, it is
expected
+that this also conforms to DNS name conventions (RFC 1035 linked above), so it
+is converted to a valid DNS hostname entries using the punycode convention
used
+for internationalized DNS.
+* **application name** - the name of the deployed YARN application. This name
is inferred
+from the YARN registry path to the application's node. Application name,
+rather than application id, was chosen as a way of making it easy for users to
refer to human-readable DNS
+names. This obviously mandates certain uniqueness properties on application
names.
+* **container id** - the YARN assigned ID to a container (e.g.
+container_e3741_1454001598828_01_000004)
+* **component name** - the name assigned to the deployed component (for e.g. a
master
+component). A component is a distributed element of an application or service
that is
+launched in a YARN container (e.g. an HBase master). One can imagine multiple
+components within an application. A component name is not yet a first class
concept in
+YARN, but is a very useful one that we are introducing here for the sake of
registry DNS
+entries. Many frameworks like MapReduce, Slider already have component names
+(though, as mentioned, they are not yet supported in YARN in a first class
fashion).
+* **api** - the api designation for the exposed endpoint
+
+### Notes about DNS Names
+
+* In most instances, the DNS names can be easily distinguished by the number of
+elements/labels that compose the name. The clusterâs domain name is always
the last
+element. After that element is parsed out, reading from right to left, the
first element
+maps to the application user and so on. Wherever it is not easily
distinguishable, naming conventions are used to disambiguate the name using a
prefix such as
+âcontainerâ or suffix such as âapiâ. For example, an endpoint
published as a
+management endpoint will be referenced with the name
*management-api.griduser.yarncluster.com*.
+* Unique application name (per user) is not currently supported/guaranteed by
YARN, but
+it is supported by the YARN service framework. The registry DNS service
currently
+leverages the last element of the ZK path entry for the application as an
+application name. These application names have to be unique for a given user.
+
+## DNS Server Functionality
+
+The primary functions of the DNS service are illustrated in the following
diagram:
+
+
+
+### DNS record creation
+The following figure illustrates at slightly greater detail the DNS record
creation and registration sequence (NOTE: service record updates would follow a
similar sequence of steps,
+distinguished only by the different event type):
+
+
+
+### DNS record removal
+Similarly, record removal follows a similar sequence
+
+
+
+(NOTE: The DNS Zone requires a record as an argument for the deletion method,
thus
+requiring similar parsing logic to identify the specific records that should
be removed).
+
+### DNS Service initialization
+* The DNS service initializes both UDP and TCP listeners on a configured port.
+If a port in the restricted range is desired (such as the standard DNS port
+53), the DNS service can be launched using jsvc as described in the section
+on starting the DNS server.
+* Subsequently, the DNS service listens for inbound DNS requests. Those
requests are
+standard DNS requests from users or other DNS servers (for example, DNS
servers that have the
+RegistryDNS service configured as a forwarder).
+
+## Start the DNS Server
+By default, the DNS server runs on non-privileged port `5335`. Start the server
+with:
+```
+hadoop --daemon start registrydns
+```
+
+If the DNS server is configured to use the standard privileged port `53`, the
+environment variables HADOOP\_REGISTRYDNS\_SECURE\_USER and
+HADOOP\_REGISTRYDNS\_SECURE\_EXTRA\_OPTS must be uncommented in the
hadoop-env.sh
+file. The DNS server should then be launched as root and jsvc will be used to
+reduce the privileges of the daemon after the port has been bound.
+
+## Configuration
+The Registry DNS server reads its configuration properties from the
core-site.xml file. The following are the DNS associated configuration
properties:
+
+| Name | Description |
+| ------------ | ------------- |
+|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs
defining the zookeeper quorum for the [Hadoop
registry](../registry/registry-configuration.html). |
+| hadoop.registry.dns.enabled | The DNS functionality is enabled for the
cluster. Default is false. |
+| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster
associated records. |
+| hadoop.registry.dns.bind-address | Address associated with the network
interface to which the DNS listener should bind. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The
default port is 5335. |
+| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is
enabled. Default is false. |
+| hadoop.registry.dns.public-key | The base64 representation of the
serverâs public key. Leveraged for creating the DNSKEY Record provided for
DNSSEC client requests. |
+| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC
private key file. Must only be readable by the DNS launching identity. See
[dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html)
documentation. |
+| hadoop.registry.dns-ttl | The default TTL value to associate with DNS
records. The default value is set to 1 (a value of 0 has undefined behavior). A
typical value should be approximate to the time it takes YARN to restart a
failed container. |
+| hadoop.registry.dns.zone-subnet | An indicator of the IP range associated
with the cluster containers. The setting is utilized for the generation of the
reverse zone name. |
+| hadoop.registry.dns.zone-mask | The network mask associated with the zone IP
range. If specified, it is utilized to ascertain the IP range possible and
come up with an appropriate reverse zone name. |
+| hadoop.registry.dns.zones-dir | A directory containing zone configuration
files to read during zone initialization. This directory can contain zone
master files named *zone-name.zone*. See
[here](http://www.zytrax.com/books/dns/ch6/mydomain.html) for zone master file
documentation.|
+### Sample configurations
+```
+ <property>
+ <description>The domain name for Hadoop cluster associated
records.</description>
+ <name>hadoop.registry.dns.domain-name</name>
+ <value>ycluster</value>
+ </property>
+
+ <property>
+ <description>The port number for the DNS listener. The default port is
5335.
+ If the standard privileged port 53 is used, make sure start the DNS with
jsvc support.</description>
+ <name>hadoop.registry.dns.bind-port</name>
+ <value>5335</value>
+ </property>
+
+ <property>
+ <description>The DNS functionality is enabled for the cluster. Default is
false.</description>
+ <name>hadoop.registry.dns.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>Address associated with the network interface to which the
DNS listener should bind.</description>
+ <name>hadoop.registry.dns.bind-address</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <description>A comma separated list of hostname:port pairs defining the
zookeeper quorum for the Hadoop registry</description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+ ```
+
+To configure Registry DNS to serve reverse lookup for `172.17.0.0/24`
+```
+ <property>
+ <description>The network mask associated with the zone IP range. If
specified, it is utilized to ascertain the
+ IP range possible and come up with an appropriate reverse zone
name.</description>
+ <name>hadoop.registry.dns.zone-mask</name>
+ <value>255.255.255.0</value>
+ </property>
+
+ <property>
+ <description>An indicator of the IP range associated with the cluster
containers. The setting is utilized for the
+ generation of the reverse zone name.</description>
+ <name>hadoop.registry.dns.zone-subnet</name>
+ <value>172.17.0.0</value>
+ </property>
+```
+
+## Make your cluster use Registry DNS
+You can edit the `/etc/resolv.conf` to make your system use the registry DNS
such as below, where `192.168.154.3` is the ip address of your DNS host. It
should appear before any nameservers that would return NXDOMAIN for lookups in
the domain used by the cluster.
+```
+nameserver 192.168.154.3
+```
+Alternatively, if you have a corporate DNS in your organization, you can
configure zone forwarding so that the Registry DNS resolves hostnames for the
domain used by the cluster.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
