HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1758092 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1758092 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1758092 Branch: refs/heads/HDDS-4 Commit: b17580923b08b60df27ed029d81cd13fe9e5b55f Parents: a1ac52d Author: Xiaoyu Yao <x...@apache.org> Authored: Tue May 22 13:32:28 2018 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Mon Oct 1 11:12:13 2018 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +- .../scm/protocol/ScmBlockLocationProtocol.java | 2 +- .../StorageContainerLocationProtocol.java | 3 +- .../protocolPB/ScmBlockLocationProtocolPB.java | 4 +- .../StorageContainerLocationProtocolPB.java | 2 +- .../apache/hadoop/ozone/OzoneConfigKeys.java | 6 +- .../common/src/main/resources/ozone-default.xml | 36 ++++++--- .../StorageContainerDatanodeProtocol.java | 2 +- .../StorageContainerDatanodeProtocolPB.java | 2 +- .../scm/server/StorageContainerManager.java | 12 +-- .../compose/compose-secure/docker-compose.yaml | 6 +- .../test/compose/compose-secure/docker-config | 12 +-- .../acceptance/ozone-secure.robot | 12 +-- .../ozone/client/protocol/ClientProtocol.java | 2 +- .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 84 ++++++++++++++++++++ .../ozone/om/protocol/OzoneManagerProtocol.java | 4 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 21 +++-- .../apache/hadoop/ozone/om/OzoneManager.java | 4 +- 18 files changed, 157 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index e8b0930..4b0e7c8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -158,9 +158,9 @@ public final class ScmConfigKeys { "ozone.scm.http-address"; public static final String OZONE_SCM_HTTPS_ADDRESS_KEY = "ozone.scm.https-address"; - public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY = - "ozone.scm.kerberos.keytab.file"; - public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal"; + public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY = + "hdds.scm.kerberos.keytab.file"; + public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal"; public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876; public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java index e17f1c2..2d46ae0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java @@ -33,7 +33,7 @@ import java.util.List; * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes * to read/write a block. */ -@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) +@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ScmBlockLocationProtocol { /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 61ffac5..4c2d815 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@ -17,7 +17,6 @@ package org.apache.hadoop.hdds.scm.protocol; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; @@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo; * ContainerLocationProtocol is used by an HDFS node to find the set of nodes * that currently host a container. */ -@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) +@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface StorageContainerLocationProtocol { /** * Asks SCM where a container should be allocated. SCM responds with the http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java index 89bb066..06bbd05 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java @@ -18,11 +18,9 @@ package org.apache.hadoop.hdds.scm.protocolPB; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos .ScmBlockLocationProtocolService; import org.apache.hadoop.hdds.scm.ScmConfigKeys; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.ipc.ProtocolInfo; import org.apache.hadoop.security.KerberosInfo; @@ -35,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo; protocolVersion = 1) @InterfaceAudience.Private @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ScmBlockLocationProtocolPB extends ScmBlockLocationProtocolService.BlockingInterface { } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java index 3bd83f9..f80ba20 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java @@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo; "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol", protocolVersion = 1) @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) @InterfaceAudience.Private public interface StorageContainerLocationProtocolPB extends StorageContainerLocationProtocolService.BlockingInterface { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index fb83052..944605b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -297,12 +297,16 @@ public final class OzoneConfigKeys { public static final long HDDS_LOCK_SUPPRESS_WARNING_INTERVAL_MS_DEAFULT = 10000L; public static final String OZONE_SECURITY_ENABLED_KEY = "ozone.security.enabled"; - public static final String OZONE_SYSTEM_TAGS_KEY = "ozone.system.tags"; public static final boolean OZONE_SECURITY_ENABLED_DEFAULT = false; public static final String OZONE_CONTAINER_COPY_WORKDIR = "hdds.datanode.replication.work.dir"; + public static final String OZONE_OM_KERBEROS_KEYTAB_FILE_KEY = "ozone.om." + + "kerberos.keytab.file"; + public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om" + + ".kerberos.principal"; + /** * There is no need to instantiate this class. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index efb5aa6..d450b37 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -433,6 +433,7 @@ </description> </property> <property> +<<<<<<< HEAD <name>ozone.om.keytab.file</name> <value/> <tag>OM, SECURITY</tag> @@ -442,6 +443,9 @@ </property> <property> <name>ozone.om.db.cache.size.mb</name> +======= + <name>ozone.ksm.db.cache.size.mb</name> +>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar. <value>128</value> <tag>OM, PERFORMANCE</tag> <description> @@ -929,7 +933,7 @@ </property> <property> - <name>ozone.scm.container.creation.lease.timeout</name> + <name>hdds.scm.container.creation.lease.timeout</name> <value>60s</value> <tag>OZONE, SCM</tag> <description> @@ -983,7 +987,11 @@ </description> </property> <property> +<<<<<<< HEAD <name>hdds.container.close.threshold</name> +======= + <name>hdds.scm.container.close.threshold</name> +>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar. <value>0.9f</value> <tag>OZONE, DATANODE</tag> <description> @@ -1199,6 +1207,7 @@ </description> </property> <property> +<<<<<<< HEAD <name>hdds.db.profile</name> <value>SSD</value> <tag>OZONE, OM, PERFORMANCE, REQUIRED</tag> @@ -1227,15 +1236,18 @@ <property> <name>ozone.scm.kerberos.keytab.file</name> +======= + <name>hdds.scm.kerberos.keytab.file</name> +>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar. <value></value> <tag> OZONE, SECURITY</tag> <description> The keytab file used by each SCM daemon to login as its service principal. The principal name is configured with - ozone.scm.kerberos.principal. + hdds.scm.kerberos.principal. </description> </property> <property> - <name>ozone.scm.kerberos.principal</name> + <name>hdds.scm.kerberos.principal</name> <value></value> <tag> OZONE, SECURITY</tag> <description>The SCM service principal. Ex scm/_h...@realm.com</description> @@ -1245,38 +1257,38 @@ <name>ozone.om.kerberos.keytab.file</name> <value></value> <tag> OZONE, SECURITY</tag> - <description> The keytab file used by KSM daemon to login as its + <description> The keytab file used by OzoneManager daemon to login as its service principal. The principal name is configured with - hdds.ksm.kerberos.principal. + ozone.om.kerberos.principal. </description> </property> <property> <name>ozone.om.kerberos.principal</name> <value></value> <tag> OZONE, SECURITY</tag> - <description>The KSM service principal. Ex ksm/_h...@realm.com</description> + <description>The OzoneManager service principal. Ex om/_h...@realm.com</description> </property> <property> - <name>ozone.scm.web.authentication.kerberos.principal</name> + <name>hdds.scm.web.authentication.kerberos.principal</name> <value>HTTP/_h...@example.com</value> </property> <property> - <name>ozone.scm.web.authentication.kerberos.keytab</name> + <name>hdds.scm.web.authentication.kerberos.keytab</name> <value>/etc/security/keytabs/HTTP.keytab</value> </property> <property> - <name>hdds.ksm.web.authentication.kerberos.principal</name> + <name>ozone.om.web.authentication.kerberos.principal</name> <value>HTTP/_h...@example.com</value> <description> - KSM http server kerberos principal. + OzoneManager http server kerberos principal. </description> </property> <property> - <name>hdds.ksm.web.authentication.kerberos.keytab</name> + <name>ozone.om.web.authentication.kerberos.keytab</name> <value>/etc/security/keytabs/HTTP.keytab</value> <description> - KSM http server kerberos keytab. + OzoneManager http server kerberos keytab. </description> </property> </configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java index 8049e9d..3600581 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java @@ -44,7 +44,7 @@ import org.apache.hadoop.security.KerberosInfo; * Protoc file that defines this protocol. */ @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) @InterfaceAudience.Private public interface StorageContainerDatanodeProtocol { /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java index 9c32ef8..9006e91 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java @@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo; "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol", protocolVersion = 1) @KerberosInfo( - serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, + serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY) public interface StorageContainerDatanodeProtocolPB extends StorageContainerDatanodeProtocolService.BlockingInterface { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 424d313..0d54f59 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -109,8 +109,8 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_M import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.util.ExitUtil.terminate; /** @@ -346,16 +346,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl throws IOException, AuthenticationException { LOG.debug("Ozone security is enabled. Attempting login for SCM user. " + "Principal: {}, keytab: {}", this.scmConf.get - (OZONE_SCM_KERBEROS_PRINCIPAL_KEY), - this.scmConf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY)); + (HDDS_SCM_KERBEROS_PRINCIPAL_KEY), + this.scmConf.get(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY)); if (SecurityUtil.getAuthenticationMethod(conf).equals (AuthenticationMethod.KERBEROS)) { UserGroupInformation.setConfiguration(this.scmConf); InetSocketAddress socAddr = HddsServerUtil .getScmBlockClientBindAddress(conf); - SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, - OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); + SecurityUtil.login(conf, HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, + HDDS_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); } else { throw new AuthenticationException(SecurityUtil.getAuthenticationMethod (conf) + " authentication method not support. " http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml index 2661163..db211bc 100644 --- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml @@ -40,15 +40,15 @@ services: env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","datanode"] - ksm: + om: image: ahadoop/ozone:v1 - hostname: ksm + hostname: om volumes: - ${OZONEDIR}:/opt/hadoop ports: - 9874:9874 environment: - ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION env_file: - ./docker-config command: ["/opt/hadoop/bin/ozone","ksm"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config index 678c75a..360b69a 100644 --- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config +++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.ksm.address=om OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.enabled=True OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id @@ -25,13 +25,13 @@ OZONE-SITE.XML_hdds.scm.client.address=scm OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/s...@example.com OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab -OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/k...@example.com -OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab +OZONE-SITE.XML_ozone.om.kerberos.principal=om/o...@example.com +OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab OZONE-SITE.XML_ozone.security.enabled=true OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/s...@example.com OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab -OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/k...@example.com -OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab +OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/o...@example.com +OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.scm.client.address=scm HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode @@ -57,7 +57,7 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH OZONE_DATANODE_SECURE_USER=root CONF_DIR=/etc/security/keytabs -KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser +KERBEROS_KEYTABS=dn nn om scm HTTP testuser KERBEROS_KEYSTORES=hadoop KERBEROS_SERVER=ozone.kdc JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot index 4a78980..7fc1088 100644 --- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot +++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot @@ -26,7 +26,7 @@ ${version} *** Test Cases *** Daemons are running - Is daemon running ksm + Is daemon running om Is daemon running scm Is daemon running datanode Is daemon running ozone.kdc @@ -45,15 +45,15 @@ Test rest interface Should contain ${result} 200 OK Test ozone cli - ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root + ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root Should contain ${result} Client cannot authenticate via # Authenticate testuser Execute on 0 datanode kinit -k testuser/datan...@example.com -t /etc/security/keytabs/testuser.keytab - Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root - ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' + Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root + ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")' Should contain ${result} createdOn - Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB - ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' + Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB + ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name' Should Be Equal ${result} bill *** Keywords *** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index f3c710b..6183b8b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -40,7 +40,7 @@ import org.apache.hadoop.security.KerberosInfo; * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and * {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST. */ -@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY) +@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY) public interface ClientProtocol { /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java new file mode 100644 index 0000000..cc25dbe --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.ksm; + +import org.apache.hadoop.ozone.OzoneAcl; +/** + * KSM Constants. + */ +public final class KSMConfigKeys { + /** + * Never constructed. + */ + private KSMConfigKeys() { + } + + + public static final String OZONE_KSM_HANDLER_COUNT_KEY = + "ozone.ksm.handler.count.key"; + public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20; + + public static final String OZONE_KSM_ADDRESS_KEY = + "ozone.ksm.address"; + public static final String OZONE_KSM_BIND_HOST_DEFAULT = + "0.0.0.0"; + public static final int OZONE_KSM_PORT_DEFAULT = 9862; + + public static final String OZONE_KSM_HTTP_ENABLED_KEY = + "ozone.ksm.http.enabled"; + public static final String OZONE_KSM_HTTP_BIND_HOST_KEY = + "ozone.ksm.http-bind-host"; + public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY = + "ozone.ksm.https-bind-host"; + public static final String OZONE_KSM_HTTP_ADDRESS_KEY = + "ozone.ksm.http-address"; + public static final String OZONE_KSM_HTTPS_ADDRESS_KEY = + "ozone.ksm.https-address"; + public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; + public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874; + public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875; + + // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB. + public static final String OZONE_KSM_DB_CACHE_SIZE_MB = + "ozone.ksm.db.cache.size.mb"; + public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128; + + public static final String OZONE_KSM_USER_MAX_VOLUME = + "ozone.ksm.user.max.volume"; + public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024; + + // KSM Default user/group permissions + public static final String OZONE_KSM_USER_RIGHTS = + "ozone.ksm.user.rights"; + public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT = + OzoneAcl.OzoneACLRights.READ_WRITE; + + public static final String OZONE_KSM_GROUP_RIGHTS = + "ozone.ksm.group.rights"; + public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT = + OzoneAcl.OzoneACLRights.READ_WRITE; + + public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK = + "ozone.key.deleting.limit.per.task"; + public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000; + + public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL = + "ozone.om.web.authentication.kerberos.principal"; + public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE = + "ozone.om.web.authentication.kerberos.keytab"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 6f11e08..ac19b05 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -26,10 +26,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo; - -import org.apache.hadoop.ozone.protocol.proto - .OzoneManagerProtocolProtos.OzoneAclInfo; import java.io.IOException; import java.util.List; import org.apache.hadoop.security.KerberosInfo; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 5c96067..3fb5499 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -118,10 +118,10 @@ public final class TestSecureOzoneCluster { private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc) throws Exception { createPrincipal(scmKeytab, - conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)); - createPrincipal(spnegoKeytab, - conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY), - conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); + conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)); + createPrincipal(spnegoKeytab, + conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY), + conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); createPrincipal(omKeyTab, conf.get(OMConfigKeys .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY)); @@ -154,7 +154,7 @@ public final class TestSecureOzoneCluster { "kerberos"); conf.set(OZONE_ADMINISTRATORS, curUser); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + host + "@" + realm); conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + host + "@" + realm); @@ -162,19 +162,18 @@ public final class TestSecureOzoneCluster { conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + host + "@" + realm); conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, - "HTTP_KSM/" + host + "@" + realm); + "HTTP_OM/" + host + "@" + realm); scmKeytab = new File(workDir, "scm.keytab"); spnegoKeytab = new File(workDir, "http.keytab"); omKeyTab = new File(workDir, "om.keytab"); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, scmKeytab.getAbsolutePath()); conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY, spnegoKeytab.getAbsolutePath()); conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, omKeyTab.getAbsolutePath()); - } @Test @@ -205,7 +204,7 @@ public final class TestSecureOzoneCluster { @Test public void testSecureScmStartupFailure() throws Exception { initSCM(); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, ""); + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, ""); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); @@ -215,9 +214,9 @@ public final class TestSecureOzoneCluster { StorageContainerManager.createSCM(null, conf); }); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/_h...@example.com"); - conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, + conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "/etc/security/keytabs/scm.keytab"); testCommonKerberosFailures( http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1758092/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 4c2b725..184af71 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -188,8 +188,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl if (SecurityUtil.getAuthenticationMethod(conf).equals (AuthenticationMethod.KERBEROS)) { LOG.debug("Ozone security is enabled. Attempting login for KSM user. " - + "Principal: {},keytab: {}", conf.get - (OZONE_OM_KERBEROS_PRINCIPAL_KEY), + + "Principal: {},keytab: {}", conf.get( + OZONE_OM_KERBEROS_PRINCIPAL_KEY), conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY)); UserGroupInformation.setConfiguration(conf); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
