This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 47fcf451da HDDS-7380. Remove expired certificates from SCM DB (#5089)
47fcf451da is described below
commit 47fcf451daaac411099eaa534adc2f65f4325d9b
Author: Galsza <[email protected]>
AuthorDate: Fri Aug 4 05:21:36 2023 +0200
HDDS-7380. Remove expired certificates from SCM DB (#5089)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +++
.../hadoop/hdds/security/SecurityConfig.java | 14 +++++++++
.../common/src/main/resources/ozone-default.xml | 8 +++++
.../hdds/security/x509/CertificateTestUtils.java | 22 ++++++++++++-
.../certificate/authority/CertificateStore.java | 8 +++++
.../x509/certificate/authority/MockCAStore.java | 4 +++
.../hdds/scm/security/RootCARotationManager.java | 18 +++++++++++
.../hadoop/hdds/scm/server/SCMCertStore.java | 32 +++++++++++++++++++
.../scm/security/TestRootCARotationManager.java | 2 ++
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 36 ++++++++++++++++++----
.../main/compose/ozonesecure/root-ca-rotation.yaml | 1 +
.../compose/ozonesecure/test-root-ca-rotation.sh | 1 +
12 files changed, 144 insertions(+), 7 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 6194688101..1200bd8bfa 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -230,6 +230,11 @@ public final class HddsConfigKeys {
"hdds.x509.ca.rotation.enabled";
public static final boolean HDDS_X509_CA_ROTATION_ENABLED_DEFAULT = false;
+ public static final String HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL =
+ "hdds.x509.expired.certificate.check.interval";
+ public static final String
+ HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL_DEFAULT = "P1D";
+
public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION =
"hdds.container.replication.compression";
public static final String HDDS_X509_ROOTCA_CERTIFICATE_FILE =
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
index 3dd0e2e9bc..49af626dbe 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/SecurityConfig.java
@@ -52,6 +52,8 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_ENABLE
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_ENABLED_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY_DEFAULT;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_POLLING_INTERVAL;
@@ -137,6 +139,7 @@ public class SecurityConfig {
private final SslProvider grpcSSLProvider;
private final Duration rootCaCertificatePollingInterval;
private final boolean autoCARotationEnabled;
+ private final Duration expiredCertificateCheckInterval;
/**
* Constructs a SecurityConfig.
@@ -243,6 +246,13 @@ public class SecurityConfig {
this.rootCaCertificatePollingInterval =
Duration.parse(rootCaCertificatePollingIntervalString);
+ String expiredCertificateCheckIntervalString = configuration.get(
+ HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL,
+ HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL_DEFAULT);
+
+ this.expiredCertificateCheckInterval =
+ Duration.parse(expiredCertificateCheckIntervalString);
+
this.externalRootCaCert = configuration.get(
HDDS_X509_ROOTCA_CERTIFICATE_FILE,
HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT);
@@ -577,6 +587,10 @@ public class SecurityConfig {
return autoCARotationEnabled;
}
+ public Duration getExpiredCertificateCheckInterval() {
+ return expiredCertificateCheckInterval;
+ }
+
/**
* Return true if using test certificates with authority as localhost. This
* should be used only for unit test where certificates are generated by
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index aa1b092413..5b3153632b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2294,6 +2294,14 @@
trust stores and requests a new certificate to be signed.
</description>
</property>
+ <property>
+ <name>hdds.x509.expired.certificate.check.interval</name>
+ <value>P1D</value>
+ <description>Interval to use for removing expired certificates. A
background
+ task to remove expired certificates from the scm metadata store is
+ scheduled to run at the rate this configuration option specifies.
+ </description>
+ </property>
<property>
<name>ozone.scm.security.handler.count.key</name>
<value>2</value>
diff --git
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateTestUtils.java
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateTestUtils.java
index 09d34fd0bb..c6e0750a87 100644
---
a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateTestUtils.java
+++
b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateTestUtils.java
@@ -107,6 +107,26 @@ public final class CertificateTestUtils {
*/
public static X509Certificate createSelfSignedCert(KeyPair keys,
String commonName, Duration expiresIn) throws Exception {
+ return createSelfSignedCert(keys, commonName, expiresIn,
+ BigInteger.valueOf(keys.getPublic().hashCode()));
+ }
+
+ /**
+ * Creates a self-signed certificate and returns it as an X509Certificate.
+ * The given keys and common name are being used in the certificate.
+ * The certificate will expire after the specified duration and its id
+ * will be the specified id.
+ *
+ * @param keys the keypair to use for the certificate
+ * @param commonName the common name used in the certificate
+ * @param expiresIn the lifespan of the certificate
+ * @param certId the id of the generated certificate
+ * @return the X509Certificate representing a self-signed certificate
+ * @throws Exception in case any error occurs during the certificate creation
+ */
+ public static X509Certificate createSelfSignedCert(KeyPair keys,
+ String commonName, Duration expiresIn, BigInteger certId)
+ throws Exception {
final Instant now = Instant.now();
final Date notBefore = Date.from(now);
final Date notAfter = Date.from(now.plus(expiresIn));
@@ -121,7 +141,7 @@ public final class CertificateTestUtils {
final X509v3CertificateBuilder certificateBuilder =
new JcaX509v3CertificateBuilder(
x500Name,
- BigInteger.valueOf(keys.getPublic().hashCode()),
+ certId,
notBefore,
notAfter,
x500Name,
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
index 9cc5dee141..e78206ca58 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
@@ -106,6 +106,14 @@ public interface CertificateStore {
*/
void removeExpiredCertificate(BigInteger serialID) throws IOException;
+ /**
+ * Deletes all non-revoked expired certificates from the store.
+ *
+ * @throws IOException - on failure
+ */
+ @Replicate
+ void removeAllExpiredCertificates() throws IOException;
+
/**
* Retrieves a Certificate based on the Serial number of that certificate.
* @param serialID - ID of the certificate.
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
index a2b2e775c7..650af970ca 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/MockCAStore.java
@@ -71,6 +71,10 @@ public class MockCAStore implements CertificateStore {
}
+ @Override
+ public void removeAllExpiredCertificates() {
+ }
+
@Override
public X509Certificate getCertificateByID(BigInteger serialID,
CertType certType)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
index 0646916091..0158eb1bc0 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
@@ -219,6 +219,24 @@ public class RootCARotationManager implements SCMService {
LOG.info("Monitor task for root certificate {} is started with " +
"interval {}.", scmCertClient.getCACertificate().getSerialNumber(),
checkInterval);
+ executorService.scheduleAtFixedRate(this::removeExpiredCertTask, 0,
+ secConf.getExpiredCertificateCheckInterval().toMillis(),
+ TimeUnit.MILLISECONDS);
+ LOG.info("Scheduling expired certificate removal with interval {}s",
+ secConf.getExpiredCertificateCheckInterval().getSeconds());
+ }
+
+ private void removeExpiredCertTask() {
+ if (!isRunning.get()) {
+ return;
+ }
+ if (scm.getCertificateStore() != null) {
+ try {
+ scm.getCertificateStore().removeAllExpiredCertificates();
+ } catch (IOException e) {
+ LOG.error("Failed to remove some expired certificates", e);
+ }
+ }
}
public boolean isRunning() {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
index b3dc7522b8..539caf6d17 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMCertStore.java
@@ -25,6 +25,7 @@ import java.math.BigInteger;
import java.security.cert.CRLException;
import java.security.cert.X509CRL;
import java.security.cert.X509Certificate;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@@ -51,6 +52,7 @@ import
org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateSto
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.bouncycastle.asn1.x509.CRLReason;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.cert.X509v2CRLBuilder;
@@ -220,6 +222,36 @@ public final class SCMCertStore implements
CertificateStore {
// TODO: Later this allows removal of expired certificates from the system.
}
+ @Override
+ public void removeAllExpiredCertificates() throws IOException {
+ lock.lock();
+ try (BatchOperation batchOperation =
+ scmMetadataStore.getBatchHandler().initBatchOperation()) {
+ addExpiredCertsToBeRemoved(batchOperation,
+ scmMetadataStore.getValidCertsTable());
+ addExpiredCertsToBeRemoved(batchOperation,
+ scmMetadataStore.getValidSCMCertsTable());
+ scmMetadataStore.getStore().commitBatchOperation(batchOperation);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void addExpiredCertsToBeRemoved(BatchOperation batchOperation,
+ Table<BigInteger, X509Certificate> certTable) throws IOException {
+ try (TableIterator<BigInteger, ? extends Table.KeyValue<BigInteger,
+ X509Certificate>> certsIterator = certTable.iterator()) {
+ Instant now = Instant.now();
+ while (certsIterator.hasNext()) {
+ Table.KeyValue<BigInteger, X509Certificate> certEntry =
+ certsIterator.next();
+ if (certEntry.getValue().getNotAfter().toInstant().isBefore(now)) {
+ certTable.deleteWithBatch(batchOperation, certEntry.getKey());
+ }
+ }
+ }
+ }
+
@Override
public X509Certificate getCertificateByID(BigInteger serialID,
CertType certType)
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
index d2d946b89d..1294150824 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/security/TestRootCARotationManager.java
@@ -58,6 +58,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_ACK_TI
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_CHECK_INTERNAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_CA_ROTATION_TIME_OF_DAY;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_GRACE_DURATION_TOKEN_CHECKS_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION;
import static org.junit.Assert.assertEquals;
@@ -195,6 +196,7 @@ public class TestRootCARotationManager {
ozoneConfig.set(HDDS_X509_CA_ROTATION_CHECK_INTERNAL, "PT2S");
ozoneConfig.set(HDDS_X509_RENEW_GRACE_DURATION, "PT15S");
ozoneConfig.set(HDDS_X509_CA_ROTATION_ACK_TIMEOUT, "PT2S");
+ ozoneConfig.set(HDDS_X509_EXPIRED_CERTIFICATE_CHECK_INTERVAL, "PT15S");
Date date = Calendar.getInstance().getTime();
date.setSeconds(date.getSeconds() + 10);
ozoneConfig.set(HDDS_X509_CA_ROTATION_TIME_OF_DAY,
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
index 93bb02f28b..6c399502ad 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
@@ -23,11 +23,11 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStore;
import org.apache.hadoop.hdds.scm.metadata.SCMMetadataStoreImpl;
import org.apache.hadoop.hdds.security.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.CertificateTestUtils;
import org.apache.hadoop.hdds.security.x509.certificate.CertInfo;
import org.apache.hadoop.hdds.security.x509.certificate.authority.CRLApprover;
import
org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
import
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCRLApprover;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
@@ -48,6 +48,7 @@ import java.nio.file.Path;
import java.security.KeyPair;
import java.security.cert.X509CRLEntry;
import java.security.cert.X509Certificate;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -265,10 +266,8 @@ public class TestSCMCertStore {
}
private X509Certificate generateX509Cert() throws Exception {
- return CertificateCodec.getX509Certificate(
- CertificateCodec.getPEMEncodedString(
- KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30,
- "SHA256withRSA")));
+ return KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30,
+ "SHA256withRSA");
}
private long getTableSize(Table<?, ?> table) throws IOException {
@@ -311,11 +310,36 @@ public class TestSCMCertStore {
}
+ @Test
+ public void testRemoveAllCertificates() throws Exception {
+ X509Certificate scmCert = CertificateTestUtils.createSelfSignedCert(
+ keyPair, "1", Duration.ofDays(1), BigInteger.valueOf(1));
+ X509Certificate expiredScmCert = CertificateTestUtils.createSelfSignedCert(
+ keyPair, "2", Duration.ofNanos(1), BigInteger.valueOf(2));
+ X509Certificate nonScmCert = CertificateTestUtils.createSelfSignedCert(
+ keyPair, "3", Duration.ofDays(1), BigInteger.valueOf(3));
+ X509Certificate expiredNonScmCert =
+ CertificateTestUtils.createSelfSignedCert(
+ keyPair, "4", Duration.ofNanos(1), BigInteger.valueOf(4));
+ scmCertStore.storeValidCertificate(
+ scmCert.getSerialNumber(), scmCert, SCM);
+ scmCertStore.storeValidCertificate(
+ expiredScmCert.getSerialNumber(), expiredScmCert, SCM);
+ scmCertStore.storeValidCertificate(
+ nonScmCert.getSerialNumber(), nonScmCert, OM);
+ scmCertStore.storeValidCertificate(
+ expiredNonScmCert.getSerialNumber(), expiredNonScmCert, OM);
+ //Listing OM certs still lists SCM certificates as well
+ checkListCerts(OM, 4);
+ checkListCerts(SCM, 2);
+ scmCertStore.removeAllExpiredCertificates();
+ checkListCerts(OM, 2);
+ checkListCerts(SCM, 1);
+ }
private void checkListCerts(NodeType role, int expected) throws Exception {
List<X509Certificate> certificateList = scmCertStore.listCertificate(role,
BigInteger.valueOf(0), 10, VALID_CERTS);
Assertions.assertEquals(expected, certificateList.size());
}
-
}
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml
b/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml
index 9b998b7f1d..b27d5f7a35 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml
@@ -36,6 +36,7 @@ x-root-cert-rotation-config:
- OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout=2s
-
OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.HttpCrossOriginFilterInitializer
- OZONE-SITE.XML_hdds.x509.ca.rotation.enabled=true
+ - OZONE-SITE.XML_hdds.x509.expired.certificate.check.interval=PT15s
services:
datanode:
<<: *root-cert-rotation-config
diff --git
a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh
b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh
index c712cba68e..5e97638cb8 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/test-root-ca-rotation.sh
@@ -54,6 +54,7 @@ execute_commands_in_container scm "ozone sh volume create
/r-v1 && ozone sh buck
# wait for second root CA rotation
wait_for_execute_command scm 180 "ozone admin cert info 3"
wait_for_execute_command om 30 "find /data/metadata/om/certs/ROOTCA-3.crt"
+wait_for_execute_command scm 60 "! ozone admin cert info 1"
execute_robot_test scm -v PREFIX:"rootca2"
certrotation/root-ca-rotation-client-checks.robot
# check the metrics
execute_robot_test scm scmha/root-ca-rotation.robot
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
