This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9b29eae46a HDDS-11117. Introduce debug CLI command to show the value
schema of any rocksDB (#6914)
9b29eae46a is described below
commit 9b29eae46ad19ba648765f22c30a1c294f403243
Author: Tejaskriya <[email protected]>
AuthorDate: Mon Jul 15 14:30:32 2024 +0530
HDDS-11117. Introduce debug CLI command to show the value schema of any
rocksDB (#6914)
---
.../org/apache/hadoop/ozone/debug/TestLDBCli.java | 27 +++
.../org/apache/hadoop/ozone/debug/ValueSchema.java | 185 +++++++++++++++++++++
2 files changed, 212 insertions(+)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java
index 1f6e33624c..7af0b5f9aa 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLDBCli.java
@@ -59,12 +59,15 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Stream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.STAND_ALONE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* This class tests `ozone debug ldb` CLI that reads from a RocksDB directory.
@@ -96,6 +99,7 @@ public class TestLDBCli {
cmd = new CommandLine(new RDBParser())
.addSubcommand(new DBScanner())
+ .addSubcommand(new ValueSchema())
.setOut(pstdout)
.setErr(pstderr);
@@ -287,6 +291,29 @@ public class TestLDBCli {
assertEquals("", stderr.toString());
}
+ @Test
+ void testSchemaCommand() throws IOException {
+ // Prepare dummy table
+ prepareTable(KEY_TABLE, false);
+
+ // Prepare scan args
+ List<String> completeScanArgs = new ArrayList<>(Arrays.asList(
+ "--db", dbStore.getDbLocation().getAbsolutePath(),
+ "value-schema",
+ "--column-family", KEY_TABLE));
+
+ int exitCode = cmd.execute(completeScanArgs.toArray(new String[0]));
+ // Check exit code. Print stderr if not expected
+ assertEquals(0, exitCode, stderr.toString());
+
+ // Check stdout
+ Pattern p = Pattern.compile(".*keyName.*", Pattern.MULTILINE);
+ Matcher m = p.matcher(stdout.toString());
+ assertTrue(m.find());
+ // Check stderr
+ assertEquals("", stderr.toString());
+ }
+
/**
* Converts String input to a Map and compares to the given Map input.
* @param expected expected result Map
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java
new file mode 100644
index 0000000000..a5029b3e6b
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ValueSchema.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.JsonUtils;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.kohsuke.MetaInfServices;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.ParameterizedType;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+
+/**
+ * Get schema of value for scm.db, om.db or container db file.
+ */
[email protected](
+ name = "value-schema",
+ description = "Schema of value in metadataTable"
+)
+@MetaInfServices(SubcommandWithParent.class)
+public class ValueSchema implements Callable<Void>, SubcommandWithParent {
+
+ @CommandLine.ParentCommand
+ private RDBParser parent;
+
+ public static final Logger LOG = LoggerFactory.getLogger(ValueSchema.class);
+
+ @CommandLine.Spec
+ private static CommandLine.Model.CommandSpec spec;
+
+ @CommandLine.Option(names = {"--column_family", "--column-family", "--cf"},
+ required = true,
+ description = "Table name")
+ private String tableName;
+
+ @CommandLine.Option(names = {"--dnSchema", "--dn-schema", "-d"},
+ description = "Datanode DB Schema Version: V1/V2/V3",
+ defaultValue = "V3")
+ private String dnDBSchemaVersion;
+
+ @CommandLine.Option(names = {"--depth"},
+ description = "The level till which the value-schema should be shown.
Values in the range [0-10] are allowed)",
+ defaultValue = "10")
+ private int depth;
+
+ @Override
+ public Void call() throws Exception {
+ if (depth < 0 || depth > 10) {
+ throw new IOException("depth should be specified in the range [0, 10]");
+ }
+
+ boolean success = true;
+
+ String dbPath = parent.getDbPath();
+ Map<String, Object> fields = new HashMap<>();
+ success = getValueFields(dbPath, fields);
+
+ out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(fields));
+
+ if (!success) {
+ // Trick to set exit code to 1 on error.
+ // TODO: Properly set exit code hopefully by refactoring GenericCli
+ throw new Exception(
+ "Exit code is non-zero. Check the error message above");
+ }
+ return null;
+ }
+
+ private boolean getValueFields(String dbPath, Map<String, Object>
valueSchema) {
+
+ dbPath = removeTrailingSlashIfNeeded(dbPath);
+ DBDefinitionFactory.setDnDBSchemaVersion(dnDBSchemaVersion);
+ DBDefinition dbDefinition =
DBDefinitionFactory.getDefinition(Paths.get(dbPath), new OzoneConfiguration());
+ if (dbDefinition == null) {
+ err().println("Error: Incorrect DB Path");
+ return false;
+ }
+ final DBColumnFamilyDefinition<?, ?> columnFamilyDefinition =
+ dbDefinition.getColumnFamily(tableName);
+ if (columnFamilyDefinition == null) {
+ err().print("Error: Table with name '" + tableName + "' not found");
+ return false;
+ }
+
+ Class<?> c = columnFamilyDefinition.getValueType();
+ valueSchema.put(c.getSimpleName(), getFieldsStructure(c, depth));
+
+ return true;
+ }
+
+ private Object getFieldsStructure(Class<?> clazz, int currentDepth) {
+ if (clazz.isPrimitive() || String.class.equals(clazz)) {
+ return clazz.getSimpleName();
+ } else if (currentDepth == 0) {
+ return "struct";
+ } else {
+ Map<String, Object> finalFields = new HashMap<>();
+ List<Field> clazzFields = getAllFields(clazz);
+ for (Field field : clazzFields) {
+ Class<?> fieldClass;
+ try {
+ if (Collection.class.isAssignableFrom(field.getType())) {
+ fieldClass = (Class<?>) ((ParameterizedType)
field.getGenericType()).getActualTypeArguments()[0];
+ } else {
+ fieldClass = field.getType();
+ }
+ } catch (ClassCastException ex) {
+ fieldClass = field.getType();
+ }
+ finalFields.put(field.getName(), getFieldsStructure(fieldClass,
currentDepth - 1));
+ }
+ return finalFields;
+ }
+ }
+
+ private List<Field> getAllFields(Class clazz) {
+ // NOTE: Schema of interface type, like ReplicationConfig, cannot be
fetched.
+ // An empty list "[]" will be shown for such types of fields.
+ if (clazz == null) {
+ return Collections.emptyList();
+ }
+ // get all fields, including inherited ones, of clazz
+ List<Field> result = new ArrayList<>(getAllFields(clazz.getSuperclass()));
+ List<Field> filteredFields = Arrays.stream(clazz.getDeclaredFields())
+ .filter(f -> !Modifier.isStatic(f.getModifiers()))
+ .collect(Collectors.toList());
+ result.addAll(filteredFields);
+ return result;
+ }
+
+ private static PrintWriter err() {
+ return spec.commandLine().getErr();
+ }
+
+ private static PrintWriter out() {
+ return spec.commandLine().getOut();
+ }
+
+ @Override
+ public Class<?> getParentType() {
+ return RDBParser.class;
+ }
+
+ private String removeTrailingSlashIfNeeded(String dbPath) {
+ if (dbPath.endsWith(OzoneConsts.OZONE_URI_DELIMITER)) {
+ dbPath = dbPath.substring(0, dbPath.length() - 1);
+ }
+ return dbPath;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
