This is an automated email from the ASF dual-hosted git repository.
pifta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2ac31e1bcb HDDS-7398. Tool to remove old certs from the scm db (#3972)
2ac31e1bcb is described below
commit 2ac31e1bcb6ad585774436e6967f277e09170964
Author: Galsza <[email protected]>
AuthorDate: Mon Dec 19 11:47:31 2022 +0100
HDDS-7398. Tool to remove old certs from the scm db (#3972)
---
hadoop-hdds/tools/pom.xml | 4 +
.../hadoop/hdds/scm/cli/cert/CertCommands.java | 1 +
.../hadoop/hdds/scm/cli/cert/CleanExpired.java | 117 +++++++++++++++++++++
.../hadoop/hdds/scm/cli/cert/TestCleanExpired.java | 100 ++++++++++++++++++
4 files changed, 222 insertions(+)
diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml
index 29cbeaf0e1..43e8ef297e 100644
--- a/hadoop-hdds/tools/pom.xml
+++ b/hadoop-hdds/tools/pom.xml
@@ -99,5 +99,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>slf4j-reload4j</artifactId>
<version>${slf4j.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-server-scm</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CertCommands.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CertCommands.java
index 21ba03599e..6b50cb451b 100644
---
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CertCommands.java
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CertCommands.java
@@ -40,6 +40,7 @@ import picocli.CommandLine.Spec;
subcommands = {
InfoSubcommand.class,
ListSubcommand.class,
+ CleanExpired.class,
})
@MetaInfServices(SubcommandWithParent.class)
diff --git
a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CleanExpired.java
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CleanExpired.java
new file mode 100644
index 0000000000..b5a2ec523f
--- /dev/null
+++
b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/cert/CleanExpired.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.cert;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.cli.GenericParentCommand;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.utils.HAUtils;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.util.concurrent.Callable;
+
+/**
+ * This is the handler to clean SCM database from expired certificates.
+ */
[email protected](
+ name = "clean",
+ description = "Clean expired certificates from the SCM metadata. " +
+ "This command is only supported when the SCM is shutdown.",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class CleanExpired implements Callable<Void>, SubcommandWithParent {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(CleanExpired.class);
+
+ @CommandLine.Option(names = {"--db"},
+ required = true,
+ description = "Database file path")
+ private String dbFilePath;
+
+ @CommandLine.Spec
+ private CommandLine.Model.CommandSpec spec;
+
+ @Override
+ public Void call() {
+ GenericParentCommand parent =
+ (GenericParentCommand) spec.root().userObject();
+
+ OzoneConfiguration configuration = parent.createOzoneConfiguration();
+
+ File db = new File(dbFilePath);
+ if (!db.exists()) {
+ LOG.error("DB path does not exist: " + dbFilePath);
+ return null;
+ }
+ if (!db.isDirectory()) {
+ LOG.error("DB path does not point to a directory: " + dbFilePath);
+ return null;
+ }
+
+ try {
+ DBStore dbStore = HAUtils.loadDB(
+ configuration, db.getParentFile(),
+ db.getName(), new SCMDBDefinition());
+ removeExpiredCertificates(dbStore);
+ } catch (Exception e) {
+ LOG.error("Error trying to open file: " + dbFilePath +
+ " failed with exception: " + e);
+ }
+ return null;
+ }
+
+ @VisibleForTesting
+ void removeExpiredCertificates(DBStore dbStore) {
+ try {
+ Table<BigInteger, X509Certificate> certsTable =
+ SCMDBDefinition.VALID_CERTS.getTable(dbStore);
+ TableIterator<BigInteger, ? extends Table.KeyValue<BigInteger,
+ X509Certificate>> tableIterator = certsTable.iterator();
+ while (tableIterator.hasNext()) {
+ Table.KeyValue<?, ?> certPair = tableIterator.next();
+ X509Certificate certificate = (X509Certificate) certPair.getValue();
+ if (Instant.now().isAfter(certificate.getNotAfter().toInstant())) {
+ LOG.info("Certificate with id " + certPair.getKey() +
+ " and value: " + certificate + "will be deleted");
+ tableIterator.removeFromDB();
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("Error when trying to open " +
+ "certificate table from db: " + e);
+ }
+ }
+
+ @Override
+ public Class<?> getParentType() {
+ return CertCommands.class;
+ }
+}
diff --git
a/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/cert/TestCleanExpired.java
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/cert/TestCleanExpired.java
new file mode 100644
index 0000000000..b169e6359d
--- /dev/null
+++
b/hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/cert/TestCleanExpired.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.cli.cert;
+
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.X509Certificate;
+import java.sql.Date;
+import java.time.Duration;
+import java.time.Instant;
+
+/**
+ * Test the cleaning tool for expired certificates.
+ */
+public class TestCleanExpired {
+
+ private CleanExpired cmd;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ @Mock
+ private DBStore dbStore;
+ @Mock
+ private Table<BigInteger, X509Certificate> mockTable;
+ @Mock
+ private TableIterator iterator;
+ @Mock
+ private Table.KeyValue<BigInteger, X509Certificate> kv;
+ @Mock
+ private Table.KeyValue<BigInteger, X509Certificate> kv2;
+ @Mock
+ private X509Certificate nonExpiredCert;
+ @Mock
+ private X509Certificate expiredCert;
+
+ @BeforeEach
+ public void setup() throws IOException {
+ MockitoAnnotations.initMocks(this);
+ cmd = new CleanExpired();
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @AfterEach
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ @Test
+ public void testOnlyExpiredCertsRemoved()
+ throws Exception {
+ Mockito.when(SCMDBDefinition.VALID_CERTS.getTable(dbStore))
+ .thenReturn(mockTable);
+ Mockito.when(mockTable.iterator()).thenReturn(iterator);
+ Mockito.when(nonExpiredCert.getNotAfter())
+ .thenReturn(Date.from(Instant.now().plus(Duration.ofDays(365))));
+ Mockito.when(expiredCert.getNotAfter())
+ .thenReturn(Date.from(Instant.now().minus(Duration.ofDays(365))));
+ Mockito.when(iterator.hasNext()).thenReturn(true, true, false);
+ Mockito.when(iterator.next()).thenReturn(kv, kv2);
+ Mockito.when(kv.getValue()).thenReturn(expiredCert);
+ Mockito.when(kv2.getValue()).thenReturn(nonExpiredCert);
+
+ cmd.removeExpiredCertificates(dbStore);
+ Mockito.verify(iterator, Mockito.times(1)).removeFromDB();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
