This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 47525b297a HDDS-12050. Implement TransactionInfoRepair command for SCM
(#7689)
47525b297a is described below
commit 47525b297aadfd4810fc818790fe6dab0b5206fe
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Thu Jan 23 23:23:52 2025 +0530
HDDS-12050. Implement TransactionInfoRepair command for SCM (#7689)
---
.../repair/{om => }/TransactionInfoRepair.java | 48 +++++++++++++++------
.../apache/hadoop/ozone/repair/om/OMRepair.java | 1 +
.../apache/hadoop/ozone/repair/scm/SCMRepair.java | 2 +
.../repair/{om => }/TestTransactionInfoRepair.java | 49 +++++++++++++---------
4 files changed, 69 insertions(+), 31 deletions(-)
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/TransactionInfoRepair.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java
similarity index 72%
rename from
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/TransactionInfoRepair.java
rename to
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java
index ca66440720..4fca8e40a0 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/TransactionInfoRepair.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/TransactionInfoRepair.java
@@ -19,15 +19,17 @@
* permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.repair.om;
+package org.apache.hadoop.ozone.repair;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.hdds.utils.TransactionInfo;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.debug.RocksDBUtils;
-import org.apache.hadoop.ozone.repair.RepairTool;
+import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
@@ -38,14 +40,13 @@
import java.util.List;
import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
-import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.TRANSACTION_INFO_TABLE;
/**
- * Tool to update the highest term-index in transactionInfoTable.
+ * Tool to update the highest term-index in transaction info table.
*/
@CommandLine.Command(
name = "update-transaction",
- description = "CLI to update the highest index in transactionInfoTable.
Currently it is only supported for OM.",
+ description = "CLI to update the highest index in transaction info table.",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class
)
@@ -58,17 +59,18 @@ public class TransactionInfoRepair extends RepairTool {
@CommandLine.Option(names = {"--term"},
required = true,
- description = "Highest term of transactionInfoTable. The input should be
non-zero long integer.")
+ description = "Highest term to set. The input should be non-zero long
integer.")
private long highestTransactionTerm;
@CommandLine.Option(names = {"--index"},
required = true,
- description = "Highest index of transactionInfoTable. The input should
be non-zero long integer.")
+ description = "Highest index to set. The input should be non-zero long
integer.")
private long highestTransactionIndex;
@Override
public void execute() throws Exception {
- if (checkIfServiceIsRunning("OM")) {
+ final Component component = getComponent();
+ if (checkIfServiceIsRunning(component.name())) {
return;
}
List<ColumnFamilyHandle> cfHandleList = new ArrayList<>();
@@ -76,9 +78,10 @@ public void execute() throws Exception {
dbPath);
try (ManagedRocksDB db = ManagedRocksDB.open(dbPath, cfDescList,
cfHandleList)) {
- ColumnFamilyHandle transactionInfoCfh =
RocksDBUtils.getColumnFamilyHandle(TRANSACTION_INFO_TABLE, cfHandleList);
+ String columnFamilyName = component.columnFamilyDefinition.getName();
+ ColumnFamilyHandle transactionInfoCfh =
RocksDBUtils.getColumnFamilyHandle(columnFamilyName, cfHandleList);
if (transactionInfoCfh == null) {
- throw new IllegalArgumentException(TRANSACTION_INFO_TABLE +
+ throw new IllegalArgumentException(columnFamilyName +
" is not in a column family in DB for the given path.");
}
TransactionInfo originalTransactionInfo =
@@ -102,11 +105,32 @@ public void execute() throws Exception {
}
} catch (RocksDBException exception) {
error("Failed to update the RocksDB for the given path: %s", dbPath);
- error(
- "Make sure that Ozone entity (OM) is not running for the give
database path and current host.");
throw new IOException("Failed to update RocksDB.", exception);
} finally {
IOUtils.closeQuietly(cfHandleList);
}
}
+
+ private Component getComponent() {
+ final String parent = spec().parent().name();
+ switch (parent) {
+ case "om":
+ return Component.OM;
+ case "scm":
+ return Component.SCM;
+ default:
+ throw new IllegalStateException("Unknown component: " + parent);
+ }
+ }
+
+ private enum Component {
+ OM(OMDBDefinition.TRANSACTION_INFO_TABLE),
+ SCM(SCMDBDefinition.TRANSACTIONINFO);
+
+ private final DBColumnFamilyDefinition<String, TransactionInfo>
columnFamilyDefinition;
+
+ Component(DBColumnFamilyDefinition<String, TransactionInfo>
columnFamilyDefinition) {
+ this.columnFamilyDefinition = columnFamilyDefinition;
+ }
+ }
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
index c8e9f6e9e4..d738ec129b 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/om/OMRepair.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.repair.om;
import org.apache.hadoop.hdds.cli.RepairSubcommand;
+import org.apache.hadoop.ozone.repair.TransactionInfoRepair;
import org.apache.hadoop.ozone.repair.om.quota.QuotaRepair;
import org.kohsuke.MetaInfServices;
import picocli.CommandLine;
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/scm/SCMRepair.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/scm/SCMRepair.java
index d7e61a8ed2..8ef59540a8 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/scm/SCMRepair.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/repair/scm/SCMRepair.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.repair.scm;
import org.apache.hadoop.hdds.cli.RepairSubcommand;
+import org.apache.hadoop.ozone.repair.TransactionInfoRepair;
import org.apache.hadoop.ozone.repair.scm.cert.CertRepair;
import org.kohsuke.MetaInfServices;
import picocli.CommandLine;
@@ -30,6 +31,7 @@
description = "Operational tool to repair SCM.",
subcommands = {
CertRepair.class,
+ TransactionInfoRepair.class
}
)
@MetaInfServices(RepairSubcommand.class)
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/om/TestTransactionInfoRepair.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java
similarity index 75%
rename from
hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/om/TestTransactionInfoRepair.java
rename to
hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java
index 3ad1c4f840..f1ad9b57ed 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/om/TestTransactionInfoRepair.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/repair/TestTransactionInfoRepair.java
@@ -15,18 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.ozone.repair.om;
+package org.apache.hadoop.ozone.repair;
+import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.debug.RocksDBUtils;
-import org.apache.hadoop.ozone.repair.OzoneRepair;
+import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.server.protocol.TermIndex;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.MockedStatic;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDB;
@@ -35,7 +37,6 @@
import static org.apache.ozone.test.IntLambda.withTextFromSystemIn;
import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
-import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.TRANSACTION_INFO_TABLE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
@@ -51,7 +52,6 @@
*/
public class TestTransactionInfoRepair {
-
private static final String DB_PATH = "testDBPath";
private static final long TEST_TERM = 1;
private static final long TEST_INDEX = 1;
@@ -69,10 +69,11 @@ void cleanup() {
IOUtils.closeQuietly(out, err);
}
- @Test
- public void testUpdateTransactionInfoTableSuccessful() {
+ @ParameterizedTest
+ @ValueSource(strings = {"om", "scm"})
+ public void testUpdateTransactionInfoTableSuccessful(String component) {
ManagedRocksDB mdb = mockRockDB();
- testCommand(mdb, mock(ColumnFamilyHandle.class));
+ testCommand(component, mdb, mock(ColumnFamilyHandle.class));
assertThat(out.getOutput())
.contains(
@@ -81,16 +82,18 @@ public void testUpdateTransactionInfoTableSuccessful() {
);
}
- @Test
- public void testCommandWhenTableNotInDBForGivenPath() {
+ @ParameterizedTest
+ @ValueSource(strings = {"om", "scm"})
+ public void testCommandWhenTableNotInDBForGivenPath(String component) {
ManagedRocksDB mdb = mockRockDB();
- testCommand(mdb, null);
+ testCommand(component, mdb, null);
assertThat(err.getOutput())
- .contains(TRANSACTION_INFO_TABLE + " is not in a column family in DB
for the given path");
+ .contains(getColumnFamilyName(component) + " is not in a column family
in DB for the given path");
}
- @Test
- public void testCommandWhenFailToUpdateRocksDBForGivenPath() throws
Exception {
+ @ParameterizedTest
+ @ValueSource(strings = {"om", "scm"})
+ public void testCommandWhenFailToUpdateRocksDBForGivenPath(String component)
throws Exception {
ManagedRocksDB mdb = mockRockDB();
RocksDB rdb = mdb.get();
@@ -98,18 +101,18 @@ public void
testCommandWhenFailToUpdateRocksDBForGivenPath() throws Exception {
doThrow(RocksDBException.class).when(rdb)
.put(eq(mock), any(byte[].class), any(byte[].class));
- testCommand(mdb, mock);
+ testCommand(component, mdb, mock);
assertThat(err.getOutput())
.contains("Failed to update RocksDB.");
}
-
- private void testCommand(ManagedRocksDB mdb, ColumnFamilyHandle
columnFamilyHandle) {
+ private void testCommand(String component, ManagedRocksDB mdb,
ColumnFamilyHandle columnFamilyHandle) {
+ final String expectedColumnFamilyName = getColumnFamilyName(component);
try (MockedStatic<ManagedRocksDB> mocked =
mockStatic(ManagedRocksDB.class);
MockedStatic<RocksDBUtils> mockUtil = mockStatic(RocksDBUtils.class))
{
mocked.when(() -> ManagedRocksDB.open(anyString(), anyList(),
anyList())).thenReturn(mdb);
- mockUtil.when(() -> RocksDBUtils.getColumnFamilyHandle(anyString(),
anyList()))
+ mockUtil.when(() ->
RocksDBUtils.getColumnFamilyHandle(eq(expectedColumnFamilyName), anyList()))
.thenReturn(columnFamilyHandle);
mockUtil.when(() -> RocksDBUtils.getValue(eq(mdb),
eq(columnFamilyHandle), eq(TRANSACTION_INFO_KEY),
@@ -128,7 +131,7 @@ private void testCommand(ManagedRocksDB mdb,
ColumnFamilyHandle columnFamilyHand
CommandLine cli = new OzoneRepair().getCmd();
withTextFromSystemIn("y")
.execute(() -> cli.execute(
- "om",
+ component,
"update-transaction",
"--db", DB_PATH,
"--term", String.valueOf(TEST_TERM),
@@ -137,6 +140,14 @@ private void testCommand(ManagedRocksDB mdb,
ColumnFamilyHandle columnFamilyHand
}
}
+ private String getColumnFamilyName(String component) {
+ switch (component) {
+ case "om": return OMDBDefinition.TRANSACTION_INFO_TABLE.getName();
+ case "scm": return SCMDBDefinition.TRANSACTIONINFO.getName();
+ default: return "";
+ }
+ }
+
private ManagedRocksDB mockRockDB() {
ManagedRocksDB db = mock(ManagedRocksDB.class);
RocksDB rocksDB = mock(RocksDB.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
