This is an automated email from the ASF dual-hosted git repository.
pifta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new bd8bb39468 HDDS-9015. Block CSR request in SCM for
"hdds.x509.rootca.certificate.polling.interval" time period (#5064)
bd8bb39468 is described below
commit bd8bb39468ed1693b94b75ef85db4710cd693618
Author: Sammi Chen <[email protected]>
AuthorDate: Fri Aug 4 23:47:53 2023 +0800
HDDS-9015. Block CSR request in SCM for
"hdds.x509.rootca.certificate.polling.interval" time period (#5064)
---
.../hadoop/hdds/scm/exceptions/SCMException.java | 3 +-
.../client/DefaultCertificateClient.java | 10 +-
.../src/main/proto/ScmServerProtocol.proto | 1 +
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 23 +++++
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 8 ++
.../hdds/scm/ha/StatefulServiceStateManager.java | 9 ++
.../scm/ha/StatefulServiceStateManagerImpl.java | 11 +++
.../hdds/scm/security/RootCARotationManager.java | 107 ++++++++++++++++++++-
.../hdds/scm/server/SCMClientProtocolServer.java | 17 ++--
.../hdds/scm/server/SCMSecurityProtocolServer.java | 43 +++------
.../hdds/scm/server/StorageContainerManager.java | 9 +-
.../scm/security/TestRootCARotationManager.java | 83 ++++++++++++++++
12 files changed, 272 insertions(+), 52 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
index 1cfc288274..9144d85e07 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
@@ -136,6 +136,7 @@ public class SCMException extends IOException {
INVALID_PIPELINE_STATE,
DUPLICATED_PIPELINE_ID,
TIMEOUT,
- CA_ROTATION_IN_PROGRESS
+ CA_ROTATION_IN_PROGRESS,
+ CA_ROTATION_IN_POST_PROGRESS,
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
index 6a950df524..a6583fc014 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
@@ -43,6 +43,7 @@ import java.time.Duration;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
@@ -408,7 +409,9 @@ public abstract class DefaultCertificateClient implements
CertificateClient {
chain.add(lastInsertedCert);
List<X509Certificate> caCertList =
OzoneSecurityUtil.convertToX509(listCA());
- while (!getAllRootCaCerts().contains(lastInsertedCert)) {
+ Set<X509Certificate> rootCaCertList = getAllRootCaCerts();
+ while (!rootCaCertList.isEmpty() &&
+ !rootCaCertList.contains(lastInsertedCert)) {
Optional<X509Certificate> issuerOpt =
getIssuerForCert(lastInsertedCert, caCertList);
if (issuerOpt.isPresent()) {
@@ -1403,8 +1406,9 @@ public abstract class DefaultCertificateClient implements
CertificateClient {
public synchronized void setCACertificate(X509Certificate cert)
throws Exception {
caCertId = cert.getSerialNumber().toString();
+ String pemCert = CertificateCodec.getPEMEncodedString(cert);
certificateMap.put(caCertId,
- CertificateCodec.getCertPathFromPemEncodedString(
- CertificateCodec.getPEMEncodedString(cert)));
+ CertificateCodec.getCertPathFromPemEncodedString(pemCert));
+ pemEncodedCACerts = Arrays.asList(pemCert);
}
}
diff --git
a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
index 0d468ed0ab..37f1d4adbf 100644
--- a/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
+++ b/hadoop-hdds/interface-server/src/main/proto/ScmServerProtocol.proto
@@ -134,6 +134,7 @@ enum Status {
DUPLICATED_PIPELINE_ID = 41;
TIMEOUT = 42;
CA_ROTATION_IN_PROGRESS = 43;
+ CA_ROTATION_IN_POST_PROGRESS = 44;
}
/**
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
index 152e8d38b5..27a97a0349 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -28,8 +28,10 @@ import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.safemode.Precheck;
+import org.apache.hadoop.hdds.scm.security.RootCARotationManager;
import org.apache.hadoop.hdds.scm.server.ContainerReportQueue;
import
org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReport;
+import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.util.StringUtils;
@@ -225,4 +227,25 @@ public final class ScmUtils {
+ SCMEvents.INCREMENTAL_CONTAINER_REPORT.getName());
}
+ public static void checkIfCertSignRequestAllowed(
+ RootCARotationManager rotationManager, boolean isScmCertRenew,
+ OzoneConfiguration config, String operation) throws SCMException {
+ if (rotationManager != null) {
+ if (rotationManager.isRotationInProgress() && !isScmCertRenew) {
+ throw new SCMException("Root CA and Sub CA rotation is in-progress." +
+ " Please try the operation later again.",
+ SCMException.ResultCodes.CA_ROTATION_IN_PROGRESS);
+ }
+ if (rotationManager.isPostRotationInProgress()) {
+ SecurityConfig securityConfig = new SecurityConfig(config);
+ throw new SCMException("The operation " + operation +
+ " is prohibited due to root CA " +
+ "and sub CA rotation have just finished. " +
+ "The prohibition state will last at most " +
+ securityConfig.getRootCaCertificatePollingInterval() + ". " +
+ "Please try the operation later again.",
+ SCMException.ResultCodes.CA_ROTATION_IN_POST_PROGRESS);
+ }
+ }
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
index 12f285e6bc..eb1b7d56d1 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
@@ -82,4 +82,12 @@ public abstract class StatefulService implements SCMService {
}
}
+
+ /**
+ * Deletes the persisted configuration mapped to the specified serviceName.
+ * @throws IOException on failure
+ */
+ protected final void deleteConfiguration() throws IOException {
+ stateManager.deleteConfiguration(getServiceName());
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
index a6a8a50ae5..c13058c490 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManager.java
@@ -52,6 +52,15 @@ public interface StatefulServiceStateManager {
*/
ByteString readConfiguration(String serviceName) throws IOException;
+ /**
+ * Deletes the persisted configuration mapped to the specified serviceName.
+ * @param serviceName name of the {@link StatefulService}, obtained through
+ * {@link SCMService#getServiceName()}
+ * @throws IOException on failure
+ */
+ @Replicate
+ void deleteConfiguration(String serviceName) throws IOException;
+
/**
* Sets the updated reference to the table when reloading SCM state.
* @param statefulServiceConfig table from
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
index 1e7a756f41..abf3b72a13 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceStateManagerImpl.java
@@ -81,6 +81,17 @@ public final class StatefulServiceStateManagerImpl
return statefulServiceConfig.get(serviceName);
}
+ /**
+ * Deletes the persisted configuration mapped to the specified serviceName.
+ * @param serviceName name of the {@link StatefulService}, obtained through
+ * {@link SCMService#getServiceName()}
+ * @throws IOException on failure
+ */
+ @Override
+ public void deleteConfiguration(String serviceName) throws IOException {
+ statefulServiceConfig.delete(serviceName);
+ }
+
/**
* {@inheritDoc}
*/
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
index 0158eb1bc0..faa7889489 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
@@ -24,14 +24,16 @@ import
com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CertInfoProto;
import org.apache.hadoop.hdds.scm.ha.HASecurityUtils;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
-import org.apache.hadoop.hdds.scm.ha.SCMService;
import org.apache.hadoop.hdds.scm.ha.SCMServiceException;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
+import org.apache.hadoop.hdds.scm.ha.StatefulService;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import
org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
import
org.apache.hadoop.hdds.security.x509.certificate.authority.profile.DefaultCAProfile;
import
org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
@@ -56,6 +58,7 @@ import java.security.cert.X509Certificate;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Executors;
@@ -75,7 +78,7 @@ import static
org.apache.hadoop.ozone.OzoneConsts.SCM_ROOT_CA_COMPONENT_NAME;
/**
* Root CA Rotation Service is a service in SCM to control the CA rotation.
*/
-public class RootCARotationManager implements SCMService {
+public class RootCARotationManager extends StatefulService {
public static final Logger LOG =
LoggerFactory.getLogger(RootCARotationManager.class);
@@ -89,11 +92,13 @@ public class RootCARotationManager implements SCMService {
private final Duration renewalGracePeriod;
private final Date timeOfDay;
private final Duration ackTimeout;
+ private final Duration rootCertPollInterval;
private final SCMCertificateClient scmCertClient;
private final AtomicBoolean isRunning = new AtomicBoolean(false);
private final AtomicBoolean isProcessing = new AtomicBoolean(false);
private final AtomicReference<Long> processStartTime =
new AtomicReference<>();
+ private final AtomicBoolean isPostProcessing = new AtomicBoolean(false);
private final String threadName = this.getClass().getSimpleName();
private final String newCAComponent = SCM_ROOT_CA_COMPONENT_NAME +
HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX +
@@ -105,6 +110,7 @@ public class RootCARotationManager implements SCMService {
private ScheduledFuture waitAckTask;
private ScheduledFuture waitAckTimeoutTask;
private final RootCARotationMetrics metrics;
+ private ScheduledFuture clearPostProcessingTask;
/**
* Constructs RootCARotationManager with the specified arguments.
@@ -131,6 +137,7 @@ public class RootCARotationManager implements SCMService {
* (4) Rotation Committed
*/
public RootCARotationManager(StorageContainerManager scm) {
+ super(scm.getStatefulServiceStateManager());
this.scm = scm;
this.ozoneConf = scm.getConfiguration();
this.secConf = new SecurityConfig(ozoneConf);
@@ -141,6 +148,7 @@ public class RootCARotationManager implements SCMService {
renewalGracePeriod = secConf.getRenewalGracePeriod();
timeOfDay = Date.from(LocalDateTime.parse(secConf.getCaRotationTimeOfDay())
.atZone(ZoneId.systemDefault()).toInstant());
+ rootCertPollInterval = secConf.getRootCaCertificatePollingInterval();
executorService = Executors.newScheduledThreadPool(1,
new ThreadFactoryBuilder().setNameFormat(threadName)
@@ -178,14 +186,25 @@ public class RootCARotationManager implements SCMService {
if (waitAckTimeoutTask != null) {
waitAckTask.cancel(true);
}
+ if (clearPostProcessingTask != null) {
+ clearPostProcessingTask.cancel(true);
+ }
isProcessing.set(false);
processStartTime.set(null);
+ isPostProcessing.set(false);
}
return;
}
if (isRunning.compareAndSet(false, true)) {
LOG.info("notifyStatusChanged: enable monitor task");
+ // enable post rotation task if needed.
+ try {
+ checkAndHandlePostProcessing();
+ } catch (IOException | CertificateException e) {
+ throw new RuntimeException(
+ "Error while checking post-processing state.", e);
+ }
}
return;
}
@@ -252,6 +271,10 @@ public class RootCARotationManager implements SCMService {
return isProcessing.get();
}
+ public boolean isPostRotationInProgress() {
+ return isPostProcessing.get();
+ }
+
/**
* Task to monitor certificate lifetime and start rotation if needed.
*/
@@ -662,14 +685,16 @@ public class RootCARotationManager implements SCMService {
processStartTime.set(null);
// save root certificate to certStore
+ X509Certificate rootCACert = null;
try {
if (scm.getCertificateStore().getCertificateByID(
rootCACertHolder.getSerialNumber(), VALID_CERTS) == null) {
LOG.info("Persist root certificate {} to cert store",
rootCACertId);
+ rootCACert =
+ CertificateCodec.getX509Certificate(rootCACertHolder);
scm.getCertificateStore().storeValidCertificate(
- rootCACertHolder.getSerialNumber(),
- CertificateCodec.getX509Certificate(rootCACertHolder),
+ rootCACertHolder.getSerialNumber(), rootCACert,
HddsProtos.NodeType.SCM);
}
} catch (CertificateException | IOException e) {
@@ -683,6 +708,17 @@ public class RootCARotationManager implements SCMService {
String msg = "Root certificate " + rootCACertId +
" rotation is finished successfully after " + timeTaken + "
ns";
cleanupAndStop(msg);
+
+ // set the isPostProcessing to true, which will block the CSR
+ // signing in this period.
+ enterPostProcessing(rootCertPollInterval.toMillis());
+ // save the new root certificate to rocksdb through ratis
+ if (rootCACert != null) {
+ saveConfiguration(new CertInfo.Builder()
+ .setX509Certificate(rootCACert)
+ .setTimestamp(rootCACert.getNotBefore().getTime())
+ .build().getProtobuf());
+ }
} catch (Throwable e) {
LOG.error("Execution error", e);
handler.resetRotationPrepareAcks();
@@ -695,6 +731,21 @@ public class RootCARotationManager implements SCMService {
}
}
+ private void enterPostProcessing(long delay) {
+ isPostProcessing.set(true);
+ LOG.info("isPostProcessing is true for {} ms", delay);
+ clearPostProcessingTask = executorService.schedule(() -> {
+ isPostProcessing.set(false);
+ LOG.info("isPostProcessing is false");
+ try {
+ deleteConfiguration();
+ LOG.info("Stateful configuration is deleted");
+ } catch (IOException e) {
+ LOG.error("Failed to delete stateful configuration", e);
+ }
+ }, delay, TimeUnit.MILLISECONDS);
+ }
+
/**
* Stops scheduled monitor task.
*/
@@ -728,4 +779,52 @@ public class RootCARotationManager implements SCMService {
}
return false;
}
+
+ private void checkAndHandlePostProcessing() throws IOException,
+ CertificateException {
+ CertInfoProto proto = readConfiguration(CertInfoProto.class);
+ if (proto == null) {
+ LOG.info("No {} configuration found in stateful storage",
+ getServiceName());
+ return;
+ }
+
+ X509Certificate cert =
+ CertificateCodec.getX509Certificate(proto.getX509Certificate());
+
+ List<X509Certificate> scmCertChain = scmCertClient.getTrustChain();
+ Preconditions.checkArgument(scmCertChain.size() > 1);
+ X509Certificate rootCert = scmCertChain.get(scmCertChain.size() - 1);
+
+ int result = rootCert.getSerialNumber().compareTo(cert.getSerialNumber());
+ if (result > 0) {
+ // this could happen if the previous stateful configuration is not
deleted
+ LOG.warn("Root CA certificate ID {} in stateful storage is smaller than"
+
+ " current scm's root certificate ID {}", cert.getSerialNumber(),
+ rootCert.getSerialNumber());
+
+ deleteConfiguration();
+ LOG.warn("Stateful configuration is deleted");
+ return;
+ } else if (result < 0) {
+ // this should not happen
+ throw new RuntimeException("Root CA certificate ID " +
+ cert.getSerialNumber() + " in stateful storage is bigger than " +
+ "current scm's root CA certificate ID " +
rootCert.getSerialNumber());
+ }
+
+ Date issueTime = rootCert.getNotBefore();
+ Date now = Calendar.getInstance().getTime();
+ Duration gap = Duration.between(issueTime.toInstant(), now.toInstant());
+ gap = gap.minus(rootCertPollInterval);
+ if (gap.isNegative()) {
+ long delay = -gap.toMillis();
+ enterPostProcessing(delay);
+ } else {
+ // this could happen if the service stopped for a long and restarts
+ LOG.info("Root CA certificate ID {} in stateful storage has already " +
+ "come out of post-processing state", cert.getSerialNumber());
+ deleteConfiguration();
+ }
+ }
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index a344ad56fd..1da84af6ef 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -113,6 +113,7 @@ import java.util.stream.Stream;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService.newReflectiveBlockingService;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmUtils.checkIfCertSignRequestAllowed;
import static
org.apache.hadoop.hdds.scm.ha.HASecurityUtils.createSCMRatisTLSConfig;
import static
org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
@@ -131,14 +132,14 @@ public class SCMClientProtocolServer implements
private final RPC.Server clientRpcServer;
private final InetSocketAddress clientRpcAddress;
private final StorageContainerManager scm;
+ private final OzoneConfiguration config;
private final ProtocolMessageMetrics<ProtocolMessageEnum> protocolMetrics;
- public SCMClientProtocolServer(
- OzoneConfiguration conf,
+ public SCMClientProtocolServer(OzoneConfiguration conf,
StorageContainerManager scm,
- ReconfigurationHandler reconfigurationHandler
- ) throws IOException {
+ ReconfigurationHandler reconfigurationHandler) throws IOException {
this.scm = scm;
+ this.config = conf;
final int handlerCount =
conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY,
OZONE_SCM_HANDLER_COUNT_DEFAULT);
@@ -803,12 +804,8 @@ public class SCMClientProtocolServer implements
throw new SCMException("SCM HA not enabled.",
ResultCodes.INTERNAL_ERROR);
}
- if (scm.getRootCARotationManager() != null &&
- scm.getRootCARotationManager().isRotationInProgress()) {
- throw new SCMException(("Root CA and Sub CA rotation is in-progress." +
- " Please try the operation later again."),
- ResultCodes.CA_ROTATION_IN_PROGRESS);
- }
+ checkIfCertSignRequestAllowed(scm.getRootCARotationManager(),
+ false, config, "transferLeadership");
boolean auditSuccess = true;
final Map<String, String> auditMap = Maps.newHashMap();
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index a273d49495..180c4574a3 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -83,6 +83,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import static
org.apache.hadoop.hdds.scm.ScmUtils.checkIfCertSignRequestAllowed;
import static
org.apache.hadoop.hdds.security.exception.SCMSecretKeyException.ErrorCode.SECRET_KEY_NOT_ENABLED;
import static
org.apache.hadoop.hdds.security.exception.SCMSecretKeyException.ErrorCode.SECRET_KEY_NOT_INITIALIZED;
import static
org.apache.hadoop.hdds.security.exception.SCMSecurityException.ErrorCode.CERTIFICATE_NOT_FOUND;
@@ -111,6 +112,7 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
private final ProtocolMessageMetrics secretKeyMetrics;
private final StorageContainerManager storageContainerManager;
private final CertificateClient scmCertificateClient;
+ private final OzoneConfiguration config;
// SecretKey may not be enabled when neither block token nor container
// token is enabled.
@@ -127,6 +129,7 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
this.rootCertificateServer = rootCertificateServer;
this.scmCertificateServer = scmCertificateServer;
this.scmCertificateClient = scmCertClient;
+ this.config = conf;
this.secretKeyManager = secretKeyManager;
final int handlerCount =
conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
@@ -190,13 +193,9 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
LOGGER.info("Processing CSR for dn {}, UUID: {}", dnDetails.getHostName(),
dnDetails.getUuid());
Objects.requireNonNull(dnDetails);
- if (storageContainerManager.getRootCARotationManager() != null &&
- storageContainerManager.getRootCARotationManager()
- .isRotationInProgress()) {
- throw new SCMException(("Root CA and Sub CA rotation is in-progress." +
- " Please try the operation later again."),
- SCMException.ResultCodes.CA_ROTATION_IN_PROGRESS);
- }
+ checkIfCertSignRequestAllowed(
+ storageContainerManager.getRootCARotationManager(), false, config,
+ "getDataNodeCertificate");
return getEncodedCertToString(certSignReq, NodeType.DATANODE);
}
@@ -208,13 +207,9 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
nodeDetails.getNodeType(), nodeDetails.getHostName(),
nodeDetails.getUuid());
Objects.requireNonNull(nodeDetails);
- if (storageContainerManager.getRootCARotationManager() != null &&
- storageContainerManager.getRootCARotationManager()
- .isRotationInProgress()) {
- throw new SCMException(("Root CA and Sub CA rotation is in-progress." +
- " Please try the operation later again."),
- SCMException.ResultCodes.CA_ROTATION_IN_PROGRESS);
- }
+ checkIfCertSignRequestAllowed(
+ storageContainerManager.getRootCARotationManager(), false, config,
+ "getCertificate");
return getEncodedCertToString(certSignReq, nodeDetails.getNodeType());
}
@@ -289,13 +284,9 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
LOGGER.info("Processing CSR for om {}, UUID: {}", omDetails.getHostName(),
omDetails.getUuid());
Objects.requireNonNull(omDetails);
- if (storageContainerManager.getRootCARotationManager() != null &&
- storageContainerManager.getRootCARotationManager()
- .isRotationInProgress()) {
- throw new SCMException(("Root CA and Sub CA rotation is in-progress." +
- " Please try the operation later again."),
- SCMException.ResultCodes.CA_ROTATION_IN_PROGRESS);
- }
+ checkIfCertSignRequestAllowed(
+ storageContainerManager.getRootCARotationManager(), false, config,
+ "getOMCertificate");
return getEncodedCertToString(certSignReq, NodeType.OM);
}
@@ -332,13 +323,9 @@ public class SCMSecurityProtocolServer implements
SCMSecurityProtocol,
+ storageContainerManager.getClusterId());
}
- if (storageContainerManager.getRootCARotationManager() != null &&
- storageContainerManager.getRootCARotationManager()
- .isRotationInProgress() && !isRenew) {
- throw new SCMException(("Root CA and Sub CA rotation is in-progress." +
- " Please try the operation later again."),
- SCMException.ResultCodes.CA_ROTATION_IN_PROGRESS);
- }
+ checkIfCertSignRequestAllowed(
+ storageContainerManager.getRootCARotationManager(), isRenew, config,
+ "getSCMCertificate");
LOGGER.info("Processing CSR for scm {}, nodeId: {}",
scmNodeDetails.getHostName(), scmNodeDetails.getScmNodeId());
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 08636f1f0f..3cb4fa8754 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -192,6 +192,7 @@ import static
org.apache.hadoop.hdds.HddsUtils.preserveThreadName;
import static org.apache.hadoop.hdds.ratis.RatisHelper.newJvmPauseMonitor;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EVENT_REPORT_EXEC_WAIT_THRESHOLD_DEFAULT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EVENT_REPORT_QUEUE_WAIT_THRESHOLD_DEFAULT;
+import static
org.apache.hadoop.hdds.scm.ScmUtils.checkIfCertSignRequestAllowed;
import static
org.apache.hadoop.hdds.scm.security.SecretKeyManagerService.isSecretKeyEnable;
import static
org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore.CertType.VALID_CERTS;
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser;
@@ -2144,12 +2145,8 @@ public final class StorageContainerManager extends
ServiceRuntimeInfoImpl
throw new IOException("Cannot remove current leader.");
}
- if (rootCARotationManager != null &&
- rootCARotationManager.isRotationInProgress()) {
- throw new SCMException(("Root CA and Sub CA rotation is in-progress." +
- " Please try the operation later again."),
- ResultCodes.CA_ROTATION_IN_PROGRESS);
- }
+ checkIfCertSignRequestAllowed(rootCARotationManager, false, configuration,
+ "removePeerFromHARing");
Preconditions.checkNotNull(getScmHAManager().getRatisServer()
.getDivision().getGroup());
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
index 1294150824..db6bfdf003 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
@@ -25,11 +25,14 @@ import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
import org.apache.hadoop.hdds.scm.ha.SCMServiceManager;
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
+import org.apache.hadoop.hdds.scm.ha.StatefulServiceStateManager;
import org.apache.hadoop.hdds.scm.server.SCMSecurityProtocolServer;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import
org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
+import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import
org.apache.hadoop.hdds.security.x509.certificate.utils.SelfSignedCertificate;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.ozone.test.GenericTestUtils;
@@ -61,8 +64,12 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_O
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL;
import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.slf4j.event.Level.INFO;
@@ -83,6 +90,7 @@ public class TestRootCARotationManager {
private SCMStorageConfig scmStorageConfig;
private SCMSecurityProtocolServer scmSecurityProtocolServer;
private RootCARotationHandlerImpl handler;
+ private StatefulServiceStateManager statefulServiceStateManager;
private File testDir;
private String cID = UUID.randomUUID().toString();
private String scmID = UUID.randomUUID().toString();
@@ -112,6 +120,8 @@ public class TestRootCARotationManager {
scmStorageConfig.setClusterId(cID);
scmSecurityProtocolServer = Mockito.mock(SCMSecurityProtocolServer.class);
handler = Mockito.mock(RootCARotationHandlerImpl.class);
+ statefulServiceStateManager =
+ Mockito.mock(StatefulServiceStateManager.class);
when(scmContext.isLeader()).thenReturn(true);
when(scm.getConfiguration()).thenReturn(ozoneConfig);
when(scm.getScmCertificateClient()).thenReturn(scmCertClient);
@@ -127,6 +137,10 @@ public class TestRootCARotationManager {
Mockito.doNothing().when(scmSecurityProtocolServer)
.setRootCertificateServer(Mockito.anyObject());
Mockito.doNothing().when(handler).rotationPrepare(Mockito.anyString());
+ when(scm.getStatefulServiceStateManager())
+ .thenReturn(statefulServiceStateManager);
+ when(statefulServiceStateManager.readConfiguration(Mockito.anyString()))
+ .thenReturn(null);
}
@AfterEach
@@ -256,6 +270,75 @@ public class TestRootCARotationManager {
100, 10000);
}
+ @Test
+ public void testPostProcessingCheck() throws Exception {
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT2S");
+ ozoneConfig.set(HDDS_X509_RENEW_GRACE_DURATION, "PT15S");
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_ACK_TIMEOUT, "PT2S");
+ ozoneConfig.set(HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL, "PT10S");
+ Date date = Calendar.getInstance().getTime();
+ date.setMinutes(date.getMinutes() + 5);
+ ozoneConfig.set(HDDS_X509_CA_ROTATION_TIME_OF_DAY,
+ String.format("%02d", date.getHours()) + ":" +
+ String.format("%02d", date.getMinutes()) + ":" +
+ String.format("%02d", date.getSeconds()));
+
+ X509Certificate cert = generateX509Cert(ozoneConfig,
+ LocalDateTime.now(), Duration.ofSeconds(60));
+ scmCertClient.setCACertificate(cert);
+ CertificateCodec certCodec = new CertificateCodec(securityConfig,
+ "scm/sub-ca");
+ certCodec.writeCertificate(CertificateCodec.getCertificateHolder(cert));
+ rootCARotationManager = new RootCARotationManager(scm);
+ rootCARotationManager.setRootCARotationHandler(handler);
+ GenericTestUtils.LogCapturer logs =
+ GenericTestUtils.LogCapturer.captureLogs(RootCARotationManager.LOG);
+ GenericTestUtils.setLogLevel(RootCARotationManager.LOG, INFO);
+ rootCARotationManager.start();
+ rootCARotationManager.notifyStatusChanged();
+
+ GenericTestUtils.waitFor(
+ () -> logs.getOutput().contains("No RootCARotationManager " +
+ "configuration found in stateful storage"),
+ 100, 10000);
+
+ when(statefulServiceStateManager.readConfiguration(Mockito.anyString()))
+ .thenReturn(new CertInfo.Builder().setX509Certificate(cert)
+ .setTimestamp(cert.getNotBefore().getTime())
+ .build().getProtobuf().toByteString());
+
+ logs.clearOutput();
+ when(scmContext.isLeader()).thenReturn(false);
+ rootCARotationManager.notifyStatusChanged();
+ when(scmContext.isLeader()).thenReturn(true);
+ rootCARotationManager.notifyStatusChanged();
+ GenericTestUtils.waitFor(
+ () -> logs.getOutput().contains("isPostProcessing is true for"),
+ 100, 10000);
+
+ logs.clearOutput();
+ when(scmContext.isLeader()).thenReturn(false);
+ rootCARotationManager.notifyStatusChanged();
+ GenericTestUtils.waitFor(
+ () -> logs.getOutput().contains("disable monitor task"),
+ 100, 10000);
+
+ logs.clearOutput();
+ when(scmContext.isLeader()).thenReturn(true);
+ rootCARotationManager.notifyStatusChanged();
+ GenericTestUtils.waitFor(
+ () -> logs.getOutput().contains("isPostProcessing is true for"),
+ 100, 10000);
+
+ GenericTestUtils.waitFor(
+ () -> logs.getOutput().contains("isPostProcessing is false"),
+ 100, 10000);
+ doNothing().when(statefulServiceStateManager)
+ .deleteConfiguration(Mockito.anyString());
+ verify(statefulServiceStateManager, times(1))
+ .deleteConfiguration(Mockito.anyString());
+ }
+
private X509Certificate generateX509Cert(
OzoneConfiguration conf, LocalDateTime startDate,
Duration certLifetime) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
