This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-6030
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-6030 by this push:
new cf6c985 HDDS-6033. Create a new docker compose env for External Root
CA (#2966)
cf6c985 is described below
commit cf6c985e283e102226bd61693cc6af5f988b40e7
Author: Vivek Ratnavel Subramanian <[email protected]>
AuthorDate: Tue Jan 11 09:26:29 2022 -0800
HDDS-6033. Create a new docker compose env for External Root CA (#2966)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +
.../hadoop/hdds/security/x509/SecurityConfig.java | 7 +
.../common/src/main/resources/ozone-default.xml | 11 +-
.../client/DefaultCertificateClient.java | 1 -
.../hdds/security/x509/keys/SecurityUtil.java | 10 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 5 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 +-
.../hdds/scm/server/SCMSecurityProtocolServer.java | 34 ++-
.../hdds/scm/server/StorageContainerManager.java | 191 ++++++------
.../scm/server/TestSCMSecurityProtocolServer.java | 2 +-
hadoop-ozone/dist/pom.xml | 5 +
.../main/compose/ozonesecure-ha-externalCA/.env | 21 ++
.../ozonesecure-ha-externalCA/docker-compose.yaml | 335 +++++++++++++++++++++
.../ozonesecure-ha-externalCA/docker-config | 148 +++++++++
.../ozonesecure-ha-externalCA/keystore/dn1.jks | Bin 0 -> 1777 bytes
.../ozonesecure-ha-externalCA/keystore/dn2.jks | Bin 0 -> 1777 bytes
.../ozonesecure-ha-externalCA/keystore/dn3.jks | Bin 0 -> 1777 bytes
.../ozonesecure-ha-externalCA/keystore/om1.jks | Bin 0 -> 1777 bytes
.../ozonesecure-ha-externalCA/keystore/om2.jks | Bin 0 -> 1777 bytes
.../ozonesecure-ha-externalCA/keystore/om3.jks | Bin 0 -> 1777 bytes
.../ozonesecure-ha-externalCA/keystore/scm1.jks | Bin 0 -> 1779 bytes
.../ozonesecure-ha-externalCA/keystore/scm2.jks | Bin 0 -> 1779 bytes
.../ozonesecure-ha-externalCA/keystore/scm3.jks | Bin 0 -> 1779 bytes
.../compose/ozonesecure-ha-externalCA/krb5.conf | 41 +++
.../main/compose/ozonesecure-ha-externalCA/test.sh | 38 +++
.../truststore/rootCA.jks | Bin 0 -> 1194 bytes
26 files changed, 754 insertions(+), 99 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 01c0aeb..f8a1f43 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -134,6 +134,8 @@ public final class HddsConfigKeys {
public static final String HDDS_CUSTOM_ROOT_CA_ENABLED =
"hdds.custom.rootca.enabled";
public static final boolean HDDS_CUSTOM_ROOT_CA_ENABLED_DEFAULT = false;
+ public static final String HDDS_CUSTOM_KEYSTORE_TYPE =
+ "hdds.custom.keystore.type";
public static final String HDDS_CUSTOM_KEYSTORE_FILE_PATH =
"hdds.custom.keystore.file.path";
public static final String HDDS_CUSTOM_KEYSTORE_FILE_PASSWORD =
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index dd6da05..34feb53 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -40,6 +40,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CUSTOM_KEYSTORE_TYPE;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
@@ -114,6 +115,7 @@ public class SecurityConfig {
private boolean grpcTlsUseTestCert;
private final boolean isCustomCAEnabled;
// The following configs are used only when custom Root CA is enabled
+ private final String keystoreType;
private final String keystoreFilePath;
private char[] keystoreFilePassword;
private char[] keystoreKeyPassword;
@@ -198,6 +200,7 @@ public class SecurityConfig {
this.configuration.get(HDDS_CUSTOM_KEYSTORE_FILE_PATH);
this.truststoreFilePath =
this.configuration.get(HDDS_CUSTOM_TRUSTSTORE_FILE_PATH);
+ this.keystoreType = this.configuration.get(HDDS_CUSTOM_KEYSTORE_TYPE);
try {
this.keystoreFilePassword =
this.configuration.getPassword(HDDS_CUSTOM_KEYSTORE_FILE_PASSWORD);
@@ -454,6 +457,10 @@ public class SecurityConfig {
return keystoreFilePath;
}
+ public String getKeystoreType() {
+ return keystoreType;
+ }
+
public char[] getKeystoreFilePassword() {
return keystoreFilePassword.clone();
}
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4511781..4232c1f 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3002,11 +3002,20 @@
</property>
<property>
+ <name>hdds.custom.keystore.type</name>
+ <value>JKS</value>
+ <tag>OZONE, SECURITY</tag>
+ <description>
+ The type of Keystore used to store the key and certificates. Example,
"JKS", "JCEKS", "PKCS12", etc.
+ </description>
+ </property>
+
+ <property>
<name>hdds.custom.keystore.file.path</name>
<value></value>
<tag>OZONE, SECURITY</tag>
<description>
- The keystore file in JCEKS format that contains the key and the
certificate signed by the custom Root CA.
+ The keystore file that contains the key and the certificate signed by
the custom Root CA.
</description>
</property>
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
index c751f1f..8f88a38 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
@@ -174,7 +174,6 @@ public abstract class DefaultCertificateClient implements
CertificateClient {
if (securityConfig.isCustomCAEnabled()) {
Certificate cert = SecurityUtil.getCustomCertificate(securityConfig);
- // String caCertsPath = securityConfig.getCaCertsPath();
if (cert != null) {
CertificateFactory cf = null;
try {
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
index d4e1eb4..ee3f7cf 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -43,6 +43,7 @@ import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.UnrecoverableKeyException;
import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
@@ -149,11 +150,12 @@ public final class SecurityUtil {
IOException, KeyStoreException, java.security.cert.CertificateException,
NoSuchAlgorithmException {
String keystorePath = securityConfig.getKeystoreFilePath();
+ String keystoreType = securityConfig.getKeystoreType();
char[] keystoreFilePassword = securityConfig.getKeystoreFilePassword();
KeyStore keystore;
try (FileInputStream is = new FileInputStream(keystorePath)) {
- keystore = KeyStore.getInstance(KeyStore.getDefaultType());
+ keystore = KeyStore.getInstance(keystoreType);
keystore.load(is, keystoreFilePassword);
}
return keystore;
@@ -191,8 +193,8 @@ public final class SecurityUtil {
return null;
}
- public static Certificate getCustomCertificate(SecurityConfig securityConfig)
- throws IOException {
+ public static X509Certificate getCustomCertificate(
+ SecurityConfig securityConfig) throws IOException {
try {
KeyStore keystore = getCustomKeystore(securityConfig);
char[] keystoreKeyPassword = securityConfig.getKeystoreKeyPassword();
@@ -201,7 +203,7 @@ public final class SecurityUtil {
Key key = keystore.getKey(keyAlias, keystoreKeyPassword);
if (key instanceof PrivateKey) {
// Get certificate of public key
- return keystore.getCertificate(keyAlias);
+ return (X509Certificate) keystore.getCertificate(keyAlias);
}
} catch (Exception ex) {
throw new SCMSecurityException("Error while getting Certificate from " +
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
index 3821575..d5d6503 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAManagerImpl.java
@@ -73,6 +73,7 @@ public class SCMHAManagerImpl implements SCMHAManager {
final StorageContainerManager scm) throws IOException {
this.conf = conf;
this.scm = scm;
+ this.exitManager = new ExitManager();
if (SCMHAUtils.isSCMHAEnabled(conf)) {
this.transactionBuffer = new SCMHADBTransactionBufferImpl(scm);
this.ratisServer = new SCMRatisServerImpl(conf, scm,
@@ -367,7 +368,9 @@ public class SCMHAManagerImpl implements SCMHAManager {
if (scm.getRootCertificateServer() != null) {
scm.getRootCertificateServer().reinitialize(metadataStore);
}
- scm.getScmCertificateServer().reinitialize(metadataStore);
+ if (scm.getScmCertificateServer() != null) {
+ scm.getScmCertificateServer().reinitialize(metadataStore);
+ }
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index 8fa1866..3a05c56 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -369,7 +369,7 @@ public class SCMStateMachine extends BaseStateMachine {
termIndex =
scm.getScmHAManager().installCheckpoint(checkpoint);
} catch (Exception e) {
- LOG.error("Failed to reinitialize SCMStateMachine.");
+ LOG.error("Failed to reinitialize SCMStateMachine.", e);
return;
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index 2b27861..ede522d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.scm.update.server.SCMCRLStore;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import
org.apache.hadoop.hdds.scm.protocol.SCMSecurityProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfig;
@@ -91,16 +92,19 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
private final InetSocketAddress rpcAddress;
private final ProtocolMessageMetrics metrics;
private final StorageContainerManager storageContainerManager;
+ private final SecurityConfig securityConfig;
SCMSecurityProtocolServer(OzoneConfiguration conf,
CertificateServer rootCertificateServer,
CertificateServer scmCertificateServer,
- X509Certificate rootCACert, StorageContainerManager scm)
+ X509Certificate rootCACert, StorageContainerManager scm,
+ SecurityConfig securityConfig)
throws IOException {
this.storageContainerManager = scm;
this.rootCertificateServer = rootCertificateServer;
this.scmCertificateServer = scmCertificateServer;
this.rootCACertificate = rootCACert;
+ this.securityConfig = securityConfig;
final int handlerCount =
conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT);
@@ -145,6 +149,10 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
public String getDataNodeCertificate(
DatanodeDetailsProto dnDetails,
String certSignReq) throws IOException {
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("Get DataNode Certificate is not " +
+ "supported when custom CA is enabled.");
+ }
LOGGER.info("Processing CSR for dn {}, UUID: {}", dnDetails.getHostName(),
dnDetails.getUuid());
Objects.requireNonNull(dnDetails);
@@ -161,6 +169,10 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
@Override
public String getOMCertificate(OzoneManagerDetailsProto omDetails,
String certSignReq) throws IOException {
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("Get OM Certificate is not supported " +
+ "when custom CA is enabled.");
+ }
LOGGER.info("Processing CSR for om {}, UUID: {}", omDetails.getHostName(),
omDetails.getUuid());
Objects.requireNonNull(omDetails);
@@ -178,6 +190,10 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
@Override
public String getSCMCertificate(ScmNodeDetailsProto scmNodeDetails,
String certSignReq) throws IOException {
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("Get SCM Certificate is not supported " +
+ "when custom CA is enabled.");
+ }
Objects.requireNonNull(scmNodeDetails);
String primaryScmId =
storageContainerManager.getScmStorageConfig().getPrimaryScmNodeId();
@@ -285,7 +301,13 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
*/
@Override
public String getCACertificate() throws IOException {
- LOGGER.debug("Getting CA certificate.");
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("Get CA Certificate is not supported " +
+ "when custom CA is enabled.");
+ }
+ if(LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Getting CA certificate.");
+ }
try {
return CertificateCodec.getPEMEncodedString(
scmCertificateServer.getCACertificate());
@@ -325,6 +347,10 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
@Override
public List<String> listCACertificate() throws IOException {
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("List CA Certificate is not supported " +
+ "when custom CA is enabled.");
+ }
List<String> caCerts =
listCertificate(NodeType.SCM, 0, 10, false);
return caCerts;
@@ -332,6 +358,10 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol {
@Override
public String getRootCACertificate() throws IOException {
+ if (securityConfig.isCustomCAEnabled()) {
+ throw new SCMSecurityException("Get RootCA Certificate is not supported"
+
+ " when custom CA is enabled.");
+ }
LOGGER.debug("Getting Root CA certificate.");
if (storageContainerManager.getScmStorageConfig()
.checkPrimarySCMIdInitialized()) {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index ce7e9de..8d8abdd 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -341,7 +341,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
// Creates the SCM DBs or opens them if it exists.
// A valid pointer to the store is required by all the other services
below.
- initalizeMetadataStore(conf, configurator);
+ initializeMetadataStore(conf, configurator);
eventQueue = new EventQueue();
serviceManager = new SCMServiceManager();
@@ -356,9 +356,7 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
// Authenticate SCM if security is enabled, this initialization can only
// be done after the metadata store is initialized.
if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
- if (!securityConfig.isCustomCAEnabled()) {
- initializeCAnSecurityProtocol(conf, configurator);
- }
+ initializeCAnSecurityProtocol(conf, configurator);
} else {
// if no Security, we do not create a Certificate Server at all.
// This allows user to boot SCM without security temporarily
@@ -468,9 +466,11 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
private void initializeCertificateClient() throws IOException {
securityConfig = new SecurityConfig(configuration);
- if (OzoneSecurityUtil.isSecurityEnabled(configuration) &&
- scmStorageConfig.checkPrimarySCMIdInitialized() &&
- !securityConfig.isCustomCAEnabled()) {
+ if (securityConfig.isCustomCAEnabled()) {
+ scmCertificateClient = new SCMCertificateClient(
+ securityConfig, scmStorageConfig.getScmCertSerialId());
+ } else if (OzoneSecurityUtil.isSecurityEnabled(configuration) &&
+ scmStorageConfig.checkPrimarySCMIdInitialized()) {
scmCertificateClient = new SCMCertificateClient(
securityConfig, scmStorageConfig.getScmCertSerialId());
}
@@ -658,47 +658,50 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
.setCRLSequenceId(getLastSequenceIdForCRL()).build();
- final CertificateServer scmCertificateServer;
- final CertificateServer rootCertificateServer;
- // If primary SCM node Id is set it means this is a cluster which has
- // performed init with SCM HA version code.
- if (scmStorageConfig.checkPrimarySCMIdInitialized()) {
- // Start specific instance SCM CA server.
- String subject = SCM_SUB_CA_PREFIX +
- InetAddress.getLocalHost().getHostName();
- if (configurator.getCertificateServer() != null) {
- scmCertificateServer = configurator.getCertificateServer();
- } else {
- scmCertificateServer = new DefaultCAServer(subject,
- scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(),
- certificateStore, new DefaultProfile(),
- scmCertificateClient.getComponentName());
- // INTERMEDIARY_CA which issues certs to DN and OM.
- scmCertificateServer.init(new SecurityConfig(configuration),
- CertificateServer.CAType.INTERMEDIARY_CA);
- }
-
- if (primaryScmNodeId.equals(scmStorageConfig.getScmId())) {
+ CertificateServer scmCertificateServer = null;
+ CertificateServer rootCertificateServer = null;
+ // Initialize Certificate Server and root CA server if custom CA is not
+ // enabled
+ if (!securityConfig.isCustomCAEnabled()) {
+ // If primary SCM node Id is set it means this is a cluster which has
+ // performed init with SCM HA version code.
+ if (scmStorageConfig.checkPrimarySCMIdInitialized()) {
+ // Start specific instance SCM CA server.
+ String subject = SCM_SUB_CA_PREFIX +
+ InetAddress.getLocalHost().getHostName();
if (configurator.getCertificateServer() != null) {
- rootCertificateServer = configurator.getCertificateServer();
+ scmCertificateServer = configurator.getCertificateServer();
} else {
- rootCertificateServer =
- HASecurityUtils.initializeRootCertificateServer(
- conf, certificateStore, scmStorageConfig, new
DefaultCAProfile());
+ scmCertificateServer = new DefaultCAServer(subject,
+ scmStorageConfig.getClusterID(), scmStorageConfig.getScmId(),
+ certificateStore, new DefaultProfile(),
+ scmCertificateClient.getComponentName());
+ // INTERMEDIARY_CA which issues certs to DN and OM.
+ scmCertificateServer.init(new SecurityConfig(configuration),
+ CertificateServer.CAType.INTERMEDIARY_CA);
+ }
+
+ if (primaryScmNodeId.equals(scmStorageConfig.getScmId())) {
+ if (configurator.getCertificateServer() != null) {
+ rootCertificateServer = configurator.getCertificateServer();
+ } else {
+ rootCertificateServer =
+ HASecurityUtils.initializeRootCertificateServer(
+ conf, certificateStore, scmStorageConfig,
+ new DefaultCAProfile());
+ }
+ persistPrimarySCMCerts();
}
- persistPrimarySCMCerts();
} else {
- rootCertificateServer = null;
+ // On a upgraded cluster primary scm nodeId will not be set as init
will
+ // not be run again after upgrade. So for a upgraded cluster where init
+ // has not happened again we will have setup like before where it has
+ // one CA server which is issuing certificates to DN and OM.
+ rootCertificateServer =
+ HASecurityUtils.initializeRootCertificateServer(conf,
+ certificateStore, scmStorageConfig, new DefaultProfile());
+ scmCertificateServer = rootCertificateServer;
}
- } else {
- // On a upgraded cluster primary scm nodeId will not be set as init will
- // not be run again after upgrade. So for a upgraded cluster where init
- // has not happened again we will have setup like before where it has
- // one CA server which is issuing certificates to DN and OM.
- rootCertificateServer =
- HASecurityUtils.initializeRootCertificateServer(conf,
- certificateStore, scmStorageConfig, new DefaultProfile());
- scmCertificateServer = rootCertificateServer;
}
// We need to pass getCACertificate as rootCA certificate,
@@ -706,7 +709,8 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
securityProtocolServer = new SCMSecurityProtocolServer(conf,
rootCertificateServer, scmCertificateServer,
scmCertificateClient != null ?
- scmCertificateClient.getCACertificate() : null, this);
+ scmCertificateClient.getCACertificate() : null,
+ this, securityConfig);
if (securityConfig.isContainerTokenEnabled()) {
containerTokenMgr = createContainerTokenSecretManager(configuration);
@@ -757,35 +761,44 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
private ContainerTokenSecretManager createContainerTokenSecretManager(
OzoneConfiguration conf) throws IOException {
+ String certId;
long expiryTime = conf.getTimeDuration(
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME,
HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT,
TimeUnit.MILLISECONDS);
- // Means this is an upgraded cluster and it has no sub-ca,
- // so SCM Certificate client is not initialized. To make Tokens
- // work let's use root CA cert and create SCM Certificate client with
- // root CA cert.
- if (scmCertificateClient == null) {
- Preconditions.checkState(
- !scmStorageConfig.checkPrimarySCMIdInitialized());
-
- String certSerialNumber;
- try {
- certSerialNumber = getScmCertificateServer().getCACertificate()
- .getSerialNumber().toString();
- } catch (CertificateException ex) {
- LOG.error("Get CA Certificate failed", ex);
- throw new IOException(ex);
- } catch (IOException ex) {
- LOG.error("Get CA Certificate failed", ex);
- throw ex;
+ if (securityConfig.isCustomCAEnabled()) {
+ X509Certificate cert =
+ org.apache.hadoop.hdds.security.x509.keys.SecurityUtil
+ .getCustomCertificate(securityConfig);
+ Preconditions.checkState(cert != null);
+ certId = cert.getSerialNumber().toString();
+ } else {
+ // Means this is an upgraded cluster, and it has no sub-ca,
+ // so SCM Certificate client is not initialized. To make Tokens
+ // work let's use root CA cert and create SCM Certificate client with
+ // root CA cert.
+ if (scmCertificateClient == null) {
+ Preconditions.checkState(
+ !scmStorageConfig.checkPrimarySCMIdInitialized());
+
+ String certSerialNumber;
+ try {
+ certSerialNumber = getScmCertificateServer().getCACertificate()
+ .getSerialNumber().toString();
+ } catch (CertificateException ex) {
+ LOG.error("Get CA Certificate failed", ex);
+ throw new IOException(ex);
+ } catch (IOException ex) {
+ LOG.error("Get CA Certificate failed", ex);
+ throw ex;
+ }
+ scmCertificateClient = new SCMCertificateClient(securityConfig,
+ certSerialNumber, SCM_ROOT_CA_COMPONENT_NAME);
}
- scmCertificateClient = new SCMCertificateClient(securityConfig,
- certSerialNumber, SCM_ROOT_CA_COMPONENT_NAME);
+ certId = scmCertificateClient.getCertificate().getSerialNumber()
+ .toString();
}
- String certId = scmCertificateClient.getCertificate().getSerialNumber()
- .toString();
return new ContainerTokenSecretManager(securityConfig,
expiryTime, certId);
}
@@ -796,8 +809,8 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
* @param configurator - configurator
* @throws IOException - on Failure
*/
- private void initalizeMetadataStore(OzoneConfiguration conf,
- SCMConfigurator configurator)
+ private void initializeMetadataStore(OzoneConfiguration conf,
+ SCMConfigurator configurator)
throws IOException {
if(configurator.getMetadataStore() != null) {
scmMetadataStore = configurator.getMetadataStore();
@@ -1358,27 +1371,29 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
// Fetch all CA's and persist during startup on bootstrap nodes. This
// is primarily being done to persist primary SCM Cert and Root CA.
// TODO: see if we can avoid doing this during every restart.
- if (primaryScmNodeId != null && !primaryScmNodeId.equals(
- scmStorageConfig.getScmId())) {
- List<String> pemEncodedCerts =
- scmCertificateClient.listCA();
-
- // Write the primary SCM CA and Root CA during startup.
- for (String cert : pemEncodedCerts) {
- try {
- X509Certificate x509Certificate =
- CertificateCodec.getX509Certificate(cert);
- if (certificateStore.getCertificateByID(
- x509Certificate.getSerialNumber(), VALID_CERTS) == null) {
- LOG.info("Persist certificate serialId {} on Scm Bootstrap Node " +
- "{}", x509Certificate.getSerialNumber(),
- scmStorageConfig.getScmId());
- certificateStore.storeValidScmCertificate(
- x509Certificate.getSerialNumber(), x509Certificate);
+ if (!securityConfig.isCustomCAEnabled()) {
+ if (primaryScmNodeId != null && !primaryScmNodeId.equals(
+ scmStorageConfig.getScmId())) {
+ List<String> pemEncodedCerts =
+ scmCertificateClient.listCA();
+
+ // Write the primary SCM CA and Root CA during startup.
+ for (String cert : pemEncodedCerts) {
+ try {
+ X509Certificate x509Certificate =
+ CertificateCodec.getX509Certificate(cert);
+ if (certificateStore.getCertificateByID(
+ x509Certificate.getSerialNumber(), VALID_CERTS) == null) {
+ LOG.info("Persist certificate serialId {} on Scm Bootstrap " +
+ "Node {}", x509Certificate.getSerialNumber(),
+ scmStorageConfig.getScmId());
+ certificateStore.storeValidScmCertificate(
+ x509Certificate.getSerialNumber(), x509Certificate);
+ }
+ } catch (CertificateException ex) {
+ LOG.error("Error while decoding CA Certificate", ex);
+ throw new IOException(ex);
}
- } catch (CertificateException ex) {
- LOG.error("Error while decoding CA Certificate", ex);
- throw new IOException(ex);
}
}
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
index e1a4e62..352e314 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
@@ -44,7 +44,7 @@ public class TestSCMSecurityProtocolServer {
config.set(OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY,
OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT + ":0");
securityProtocolServer = new SCMSecurityProtocolServer(config, null,
- null, null, null);
+ null, null, null, null);
}
@After
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index 72ce7b7..3bff9ea 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -112,6 +112,11 @@
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
+ <configuration>
+ <nonFilteredFileExtensions>
+ <nonFilteredFileExtension>jks</nonFilteredFileExtension>
+ </nonFilteredFileExtensions>
+ </configuration>
<executions>
<execution>
<id>copy-compose-files</id>
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/.env
new file mode 100644
index 0000000..85cf1d2
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/.env
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+HADOOP_VERSION=3
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
+OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
+OZONE_OPTS=
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/docker-compose.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/docker-compose.yaml
new file mode 100644
index 0000000..b6e8009
--- /dev/null
+++
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/docker-compose.yaml
@@ -0,0 +1,335 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ kdc:
+ image: ${OZONE_TESTKRB5_IMAGE}
+ hostname: kdc
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ command: ["krb5kdc","-n"]
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.100
+ kms:
+ image: apache/hadoop:${HADOOP_VERSION}
+ ports:
+ - 9600:9600
+ env_file:
+ - ./docker-config
+ volumes:
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ../../libexec/transformation.py:/opt/transformation.py
+ environment:
+ HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
+ command: ["hadoop", "kms"]
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.101
+ datanode1:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/dn1.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9864:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ - "recon: 172.25.0.115"
+ env_file:
+ - docker-config
+ environment:
+ WAITFOR: scm3.org:9894
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.102
+ datanode2:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/dn2.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9866:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ - "recon: 172.25.0.115"
+ env_file:
+ - docker-config
+ environment:
+ WAITFOR: scm3.org:9894
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.103
+ datanode3:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/dn3.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9868:9999
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ - "recon: 172.25.0.115"
+ env_file:
+ - docker-config
+ environment:
+ WAITFOR: scm3.org:9894
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.104
+ om1:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om1
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/om1.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9880:9874
+ - 9890:9872
+ #- 18001:18001
+ environment:
+ WAITFOR: scm3.org:9894
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ OZONE_OPTS:
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.111
+ om2:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om2
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/om2.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9882:9874
+ - 9892:9872
+ #- 18002:18002
+ environment:
+ WAITFOR: scm3.org:9894
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ OZONE_OPTS:
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.112
+ om3:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: om3
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/om3.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9884:9874
+ - 9894:9872
+ #- 18003:18003
+ environment:
+ WAITFOR: scm3.org:9894
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ OZONE_OPTS:
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ extra_hosts:
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.113
+ s3g:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: s3g
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ ports:
+ - 9878:9878
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","s3g"]
+ environment:
+ OZONE_OPTS:
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.114
+ scm1.org:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: scm1.org
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/scm1.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9990:9876
+ - 9992:9860
+ env_file:
+ - docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-3}"
+ OZONE_OPTS:
+ command: ["sh", "-c",
+ "/opt/hadoop/bin/ozone scm --init;
+ exec /opt/hadoop/bin/ozone scm"]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.116
+ scm2.org:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: scm2.org
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/scm2.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9994:9876
+ - 9996:9860
+ env_file:
+ - docker-config
+ environment:
+ WAITFOR: scm1.org:9894
+ ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-3}"
+ OZONE_OPTS:
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.117
+ scm3.org:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: scm3.org
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ - ${PWD}/keystore/scm3.jks:/etc/security/keystore.jks
+ - ${PWD}/truststore/rootCA.jks:/etc/security/truststore.jks
+ ports:
+ - 9998:9876
+ - 10002:9860
+ env_file:
+ - docker-config
+ environment:
+ WAITFOR: scm2.org:9894
+ ENSURE_SCM_BOOTSTRAPPED: /data/metadata/scm/current/VERSION
+ OZONE-SITE.XML_hdds.scm.safemode.min.datanode:
"${OZONE_SAFEMODE_MIN_DATANODES:-3}"
+ OZONE_OPTS:
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.118
+ recon:
+ image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+ hostname: recon
+ volumes:
+ - ../..:/opt/hadoop
+ - ../_keytabs:/etc/security/keytabs
+ - ./krb5.conf:/etc/krb5.conf
+ ports:
+ - 9888:9888
+ env_file:
+ - ./docker-config
+ environment:
+ OZONE_OPTS:
+ command: ["/opt/hadoop/bin/ozone","recon"]
+ extra_hosts:
+ - "om1: 172.25.0.111"
+ - "om2: 172.25.0.112"
+ - "om3: 172.25.0.113"
+ - "scm1.org: 172.25.0.116"
+ - "scm2.org: 172.25.0.117"
+ - "scm3.org: 172.25.0.118"
+ networks:
+ ozone_net:
+ ipv4_address: 172.25.0.115
+networks:
+ ozone_net:
+ ipam:
+ driver: default
+ config:
+ - subnet: "172.25.0.0/24"
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/docker-config
new file mode 100644
index 0000000..be6b40a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/docker-config
@@ -0,0 +1,148 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=ofs://id1
+
+OZONE-SITE.XML_ozone.om.service.ids=id1
+OZONE-SITE.XML_ozone.om.internal.service.id=id1
+OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3
+OZONE-SITE.XML_ozone.om.address.id1.om1=om1
+OZONE-SITE.XML_ozone.om.address.id1.om2=om2
+OZONE-SITE.XML_ozone.om.address.id1.om3=om3
+OZONE-SITE.XML_ozone.om.http-address.id1.om1=om1
+OZONE-SITE.XML_ozone.om.http-address.id1.om2=om2
+OZONE-SITE.XML_ozone.om.http-address.id1.om3=om3
+OZONE-SITE.XML_ozone.om.ratis.enable=true
+
+OZONE-SITE.XML_ozone.scm.service.ids=scmservice
+OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
+OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org
+OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org
+OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org
+OZONE-SITE.XML_ozone.scm.ratis.enable=true
+
+OZONE-SITE.XML_ozone.om.volume.listall.allowed=false
+
+OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
+OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
+OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
+OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.block.token.enabled=false
+OZONE-SITE.XML_hdds.container.token.enabled=false
+OZONE-SITE.XML_hdds.grpc.tls.enabled=false
+OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
+OZONE-SITE.XML_hdds.container.report.interval=60s
+
+OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.recon.om.snapshot.task.initial.delay=20s
+OZONE-SITE.XML_ozone.recon.address=recon:9891
+
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_ozone.acl.enabled=true
+OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
+OZONE-SITE.XML_ozone.administrators="testuser/[email protected],testuser/[email protected],recon/[email protected],om/[email protected],om/[email protected],om/[email protected]"
+
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1](testuser2.*)
RULE:[2:$1@$0](.*)s/.*/root/
+CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
+
+
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/[email protected]
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/[email protected]
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
+OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/[email protected]
+
+OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
+OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/[email protected]
+
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+
+OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
+OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
+
+OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
+OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
+OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
+
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/[email protected]
+OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/[email protected]
+OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/[email protected]
+OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/[email protected]
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/[email protected]
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
+OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
+
+# Enable external CA
+OZONE-SITE.XML_hdds.custom.rootca.enabled=true
+OZONE-SITE.XML_hdds.custom.keystore.type=PKCS12
+OZONE-SITE.XML_hdds.custom.keystore.file.path=/etc/security/keystore.jks
+OZONE-SITE.XML_hdds.custom.keystore.file.password=admin1
+OZONE-SITE.XML_hdds.custom.keystore.key.password=admin1
+OZONE-SITE.XML_hdds.custom.truststore.file.path=/etc/security/truststore.jks
+OZONE-SITE.XML_hdds.custom.truststore.password=admin1
+
+CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
+CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
+CORE-SITE.XML_hadoop.http.authentication.type=kerberos
+CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/[email protected]
+CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+
+
+CORE-SITE.XML_hadoop.security.authorization=true
+HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
+HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
+
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See
http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+OZONE_DATANODE_SECURE_USER=root
+JAVA_HOME=/usr/lib/jvm/jre
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+
+OZONE_CONF_DIR=/etc/hadoop
+OZONE_LOG_DIR=/var/log/hadoop
+
+no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn1.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn1.jks
new file mode 100644
index 0000000..9118677
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn1.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn2.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn2.jks
new file mode 100644
index 0000000..3ae7266
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn2.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn3.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn3.jks
new file mode 100644
index 0000000..69c6296
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/dn3.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om1.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om1.jks
new file mode 100644
index 0000000..340a897
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om1.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om2.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om2.jks
new file mode 100644
index 0000000..6b4e278
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om2.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om3.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om3.jks
new file mode 100644
index 0000000..3e9cedf
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/om3.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm1.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm1.jks
new file mode 100644
index 0000000..1009980
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm1.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm2.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm2.jks
new file mode 100644
index 0000000..80a224b
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm2.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm3.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm3.jks
new file mode 100644
index 0000000..2360783
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/keystore/scm3.jks
differ
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/krb5.conf
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/krb5.conf
new file mode 100644
index 0000000..eefc5b9
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/krb5.conf
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+default = FILE:/var/log/krb5libs.log
+kdc = FILE:/var/log/krb5kdc.log
+admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = kdc
+ admin_server = kdc
+ max_renewable_life = 7d
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
+
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/test.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/test.sh
new file mode 100755
index 0000000..ccebaee
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/test.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:HA
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="id1"
+export SCM=scm1.org
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+execute_robot_test ${SCM} kinit.robot
+
+execute_robot_test ${SCM} freon
+
+stop_docker_env
+
+generate_report
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/truststore/rootCA.jks
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/truststore/rootCA.jks
new file mode 100644
index 0000000..188201b
Binary files /dev/null and
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha-externalCA/truststore/rootCA.jks
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
