This is an automated email from the ASF dual-hosted git repository.
prashantpogde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c8e63d2b1d HDDS-8961. Write a standalone tool to create a visual graph
from compaction log files. (#5061)
c8e63d2b1d is described below
commit c8e63d2b1d01b3abd3a8fbc290a1e4ebada4556f
Author: Hemant Kumar <[email protected]>
AuthorDate: Sun Jul 16 00:12:40 2023 -0700
HDDS-8961. Write a standalone tool to create a visual graph from compaction
log files. (#5061)
---
hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 8 ++
.../RelationshipEdge.java => graph/Edge.java} | 19 ++--
.../org/apache/ozone/graph/PrintableGraph.java | 124 +++++++++++++++++++++
.../ozone/{rocksdiff => graph}/package-info.java | 2 +-
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 24 +++-
.../org/apache/ozone/rocksdiff/package-info.java | 2 +-
.../apache/hadoop/ozone/client/ObjectStore.java | 12 ++
.../ozone/client/protocol/ClientProtocol.java | 10 ++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 13 +++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 13 +++
...OzoneManagerProtocolClientSideTranslatorPB.java | 28 ++++-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 2 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 5 +
.../src/main/proto/OmClientProtocol.proto | 22 +++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 28 +++++
.../protocolPB/OzoneManagerRequestHandler.java | 19 ++++
.../hadoop/ozone/client/ClientProtocolStub.java | 5 +
.../ozone/debug/CompactionLogDagPrinter.java | 71 ++++++++++++
pom.xml | 11 ++
20 files changed, 399 insertions(+), 20 deletions(-)
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
index 28b582b7f0..18b27808b8 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/pom.xml
@@ -82,6 +82,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-rocks-native</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.jgrapht</groupId>
+ <artifactId>jgrapht-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jgrapht</groupId>
+ <artifactId>jgrapht-ext</artifactId>
+ </dependency>
</dependencies>
<build>
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RelationshipEdge.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/Edge.java
similarity index 75%
rename from
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RelationshipEdge.java
rename to
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/Edge.java
index b43ef9ff95..b7829a255b 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RelationshipEdge.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/Edge.java
@@ -6,24 +6,25 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocksdiff;
+package org.apache.ozone.graph;
-// import org.jgrapht.graph.DefaultEdge;
-// Enable this import and extend DefaultEdge if We need to
-// pcitorially represent the DAG constructed.
+import org.jgrapht.graph.DefaultEdge;
-//class RelationshipEdge extends DefaultEdge {
-class RelationshipEdge {
- //@Override
+/**
+ * Overrides the {@link DefaultEdge} so that it doesn't print source and target
+ * vertex names in the image.
+ */
+public class Edge extends DefaultEdge {
+ @Override
public String toString() {
return "";
}
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/PrintableGraph.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/PrintableGraph.java
new file mode 100644
index 0000000000..16adaebf22
--- /dev/null
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/PrintableGraph.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.graph;
+
+import com.google.common.graph.EndpointPair;
+import com.google.common.graph.MutableGraph;
+import com.mxgraph.layout.hierarchical.mxHierarchicalLayout;
+import com.mxgraph.layout.mxIGraphLayout;
+import com.mxgraph.util.mxCellRenderer;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.ozone.rocksdiff.CompactionNode;
+import org.jgrapht.Graph;
+import org.jgrapht.ext.JGraphXAdapter;
+import org.jgrapht.graph.DefaultDirectedGraph;
+
+import javax.imageio.ImageIO;
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Wrapped over {@link Graph} to get an image of {@link MutableGraph}.
+ */
+public class PrintableGraph {
+
+ /**
+ * Enum to print different type of node's name in the graph image.
+ */
+ public enum GraphType {
+ /**
+ * To use SST file name as node name.
+ */
+ FILE_NAME,
+
+ /**
+ * To use SST file name and total key in the file as node name.
+ */
+ KEY_SIZE,
+
+ /**
+ * To use SST file name and cumulative key as node name.
+ */
+ CUMULATIVE_SIZE
+ }
+
+ private final Graph<String, Edge> graph;
+
+ public PrintableGraph(MutableGraph<CompactionNode> guavaGraph,
+ GraphType graphType) {
+ this.graph = getGraph(guavaGraph, graphType);
+ }
+
+ public void generateImage(String fileName) throws IOException {
+ if (CollectionUtils.isEmpty(graph.vertexSet())) {
+ throw new IOException("Graph is empty.");
+ }
+
+ JGraphXAdapter<String, Edge> jGraphXAdapter = new JGraphXAdapter<>(graph);
+ mxIGraphLayout mxIGraphLayout =
+ new mxHierarchicalLayout(jGraphXAdapter);
+ mxIGraphLayout.execute(jGraphXAdapter.getDefaultParent());
+
+ BufferedImage bufferedImage =
+ mxCellRenderer.createBufferedImage(jGraphXAdapter, null, 2,
+ Color.WHITE, true, null);
+
+ File newFIle = new File(fileName);
+ ImageIO.write(bufferedImage, "PNG", newFIle);
+ }
+
+ /**
+ * Convert guava's {@link MutableGraph} to jgrapht's {@link Graph}.
+ */
+ public Graph<String, Edge> getGraph(
+ MutableGraph<CompactionNode> guavaGraph,
+ GraphType graphType
+ ) {
+
+ Graph<String, Edge> jgrapht =
+ new DefaultDirectedGraph<>(Edge.class);
+
+ for (CompactionNode node : guavaGraph.nodes()) {
+ jgrapht.addVertex(getVertex(node, graphType));
+ }
+
+ for (EndpointPair<CompactionNode> edge : guavaGraph.edges()) {
+ jgrapht.addEdge(getVertex(edge.source(), graphType),
+ getVertex(edge.target(), graphType));
+ }
+
+ return jgrapht;
+ }
+
+ private String getVertex(CompactionNode node, GraphType graphType) {
+ switch (graphType) {
+ case KEY_SIZE:
+ return
+ node.getFileName() + "::" + node.getTotalNumberOfKeys();
+ case CUMULATIVE_SIZE:
+ return
+ node.getFileName() + "::" + node.getCumulativeKeysReverseTraversal();
+ case FILE_NAME:
+ default:
+ return node.getFileName();
+ }
+ }
+}
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/package-info.java
similarity index 96%
copy from
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
copy to
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/package-info.java
index 459528aefe..2d986cd076 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/package-info.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone;
+package org.apache.ozone.graph;
/**
* Generic ozone specific classes.
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
index 3388d86cba..03183ac300 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
@@ -21,9 +21,11 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.graph.GraphBuilder;
import com.google.common.graph.MutableGraph;
+
import java.io.FileNotFoundException;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Objects;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -37,6 +39,8 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.graph.PrintableGraph;
+import org.apache.ozone.graph.PrintableGraph.GraphType;
import org.rocksdb.AbstractEventListener;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.CompactionJobInfo;
@@ -1081,11 +1085,13 @@ public class RocksDBCheckpointDiffer implements
AutoCloseable,
}
}
- public MutableGraph<CompactionNode> getForwardCompactionDAG() {
+ @VisibleForTesting
+ MutableGraph<CompactionNode> getForwardCompactionDAG() {
return forwardCompactionDAG;
}
- public MutableGraph<CompactionNode> getBackwardCompactionDAG() {
+ @VisibleForTesting
+ MutableGraph<CompactionNode> getBackwardCompactionDAG() {
return backwardCompactionDAG;
}
@@ -1540,4 +1546,18 @@ public class RocksDBCheckpointDiffer implements
AutoCloseable,
public BootstrapStateHandler.Lock getBootstrapStateLock() {
return lock;
}
+
+ public String pngPrintMutableGraph(String fileName, GraphType graphType)
+ throws IOException {
+ Objects.requireNonNull(fileName, "Image file name is required.");
+ Objects.requireNonNull(graphType, "Graph type is required.");
+
+ PrintableGraph graph;
+ synchronized (this) {
+ graph = new PrintableGraph(forwardCompactionDAG, graphType);
+ }
+
+ graph.generateImage(fileName);
+ return fileName;
+ }
}
diff --git
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
index 459528aefe..270acccc8e 100644
---
a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
+++
b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone;
+package org.apache.ozone.rocksdiff;
/**
* Generic ozone specific classes.
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 69e4f68eb6..6453b2b88a 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -593,6 +593,18 @@ public class ObjectStore {
volumeName, bucketName, snapshotPrefix, prevSnapshot);
}
+ /**
+ * Create an image of the current compaction log DAG in the OM.
+ * @param fileName name of the image file.
+ * @param graphType type of node name to use in the graph image.
+ * @return path of the image file.
+ * @throws IOException
+ */
+ public String printCompactionLogDag(String fileName,
+ String graphType) throws IOException {
+ return proxy.printCompactionLogDag(fileName, graphType);
+ }
+
/**
* An Iterator to iterate over {@link OzoneSnapshot} list.
*/
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 2c780d93ba..f721eba5e4 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -1045,6 +1045,16 @@ public interface ClientProtocol {
void deleteSnapshot(String volumeName,
String bucketName, String snapshotName) throws IOException;
+ /**
+ * Create an image of the current compaction log DAG in the OM.
+ * @param fileName name of the image file.
+ * @param graphType type of node name to use in the graph image.
+ * @return path of the image file.
+ * @throws IOException
+ */
+ String printCompactionLogDag(String fileName, String graphType)
+ throws IOException;
+
/**
* List snapshots in a volume/bucket.
* @param volumeName volume name
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 8bf0527b64..5b18f47275 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -987,6 +987,19 @@ public class RpcClient implements ClientProtocol {
ozoneManagerClient.deleteSnapshot(volumeName, bucketName, snapshotName);
}
+ /**
+ * Create an image of the current compaction log DAG in the OM.
+ * @param fileName name of the image file.
+ * @param graphType type of node name to use in the graph image.
+ * @return path of the image file.
+ * @throws IOException
+ */
+ @Override
+ public String printCompactionLogDag(String fileName,
+ String graphType) throws IOException {
+ return ozoneManagerClient.printCompactionLogDag(fileName, graphType);
+ }
+
@Override
public SnapshotDiffResponse snapshotDiff(String volumeName,
String bucketName,
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 1ae3842e1f..711db38bcb 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -270,6 +270,7 @@ public final class OmUtils {
case ListSnapshotDiffJobs:
case TransferLeadership:
case SetSafeMode:
+ case PrintCompactionLogDag:
return true;
case CreateVolume:
case SetVolumeProperty:
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 962eb9eb57..b9a9f1e7b2 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -691,6 +691,19 @@ public interface OzoneManagerProtocol
"this to be implemented");
}
+ /**
+ * Create an image of the current compaction log DAG in the OM.
+ * @param fileName name of the image file.
+ * @param graphType type of node name to use in the graph image.
+ * @return path of the image file.
+ * @throws IOException
+ */
+ default String printCompactionLogDag(String fileName, String graphType)
+ throws IOException {
+ throw new UnsupportedOperationException("OzoneManager does not require " +
+ "this to be implemented");
+ }
+
/**
* List snapshots in a volume/bucket.
* @param volumeName volume name
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index e8b1bd2254..a6bb29859e 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -31,8 +31,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceAudience;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.TransferLeadershipRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos
- .UpgradeFinalizationStatus;
+import
org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.io.Text;
@@ -153,6 +152,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMReque
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest;
@@ -1190,6 +1190,30 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
handleError(omResponse);
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String printCompactionLogDag(String fileName, String graphType)
+ throws IOException {
+ final PrintCompactionLogDagRequest.Builder request =
+ PrintCompactionLogDagRequest.newBuilder();
+
+ if (fileName != null) {
+ request.setFileName(fileName);
+ }
+ if (graphType != null) {
+ request.setGraphType(graphType);
+ }
+
+ final OMRequest omRequest = createOMRequest(Type.PrintCompactionLogDag)
+ .setPrintCompactionLogDagRequest(request.build())
+ .build();
+ final OMResponse omResponse = submitRequest(omRequest);
+ handleError(omResponse);
+ return omResponse.getPrintCompactionLogDagResponse().getImagePath();
+ }
+
/**
* {@inheritDoc}
*/
diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index ab39f190b6..a361067ae7 100644
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@ -468,6 +468,8 @@ EPL 2.0
jakarta.annotation:jakarta.annotation-api
jakarta.ws.rs:jakarta.ws.rs-api
+ org.jgrapht:jgrapht-core
+ org.jgrapht:jgrapht-ext
CDDL + GPLv2 with classpath exception
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt
b/hadoop-ozone/dist/src/main/license/jar-report.txt
index 51d11cf11c..6fdc30d2b0 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -3,6 +3,7 @@ share/ozone/lib/animal-sniffer-annotations.jar
share/ozone/lib/annotations.jar
share/ozone/lib/annotations.jar
share/ozone/lib/aopalliance.jar
+share/ozone/lib/antlr4-runtime.jar
share/ozone/lib/aopalliance-repackaged.jar
share/ozone/lib/aspectjrt.jar
share/ozone/lib/aspectjweaver.jar
@@ -145,6 +146,10 @@ share/ozone/lib/jetty-util-ajax.jar
share/ozone/lib/jetty-util.jar
share/ozone/lib/jetty-webapp.jar
share/ozone/lib/jetty-xml.jar
+share/ozone/lib/jgraph.jar
+share/ozone/lib/jgrapht-core.jar
+share/ozone/lib/jgrapht-ext.jar
+share/ozone/lib/jgraphx.jar
share/ozone/lib/jmespath-java.jar
share/ozone/lib/jna.jar
share/ozone/lib/jna-platform.jar
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 760d68557a..ece9b2637c 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -140,6 +140,7 @@ enum Type {
ListSnapshotDiffJobs = 122;
CancelSnapshotDiff = 123;
SetSafeMode = 124;
+ PrintCompactionLogDag = 125;
}
enum SafeMode {
@@ -265,11 +266,12 @@ message OMRequest {
optional SnapshotPurgeRequest SnapshotPurgeRequest =
118;
optional RecoverLeaseRequest RecoverLeaseRequest =
119;
- optional SetTimesRequest SetTimesRequest =
120;
+ optional SetTimesRequest SetTimesRequest =
120;
optional RefetchSecretKeyRequest RefetchSecretKeyRequest =
121;
optional ListSnapshotDiffJobRequest ListSnapshotDiffJobRequest =
122;
- optional CancelSnapshotDiffRequest CancelSnapshotDiffRequest = 123;
+ optional CancelSnapshotDiffRequest CancelSnapshotDiffRequest =
123;
optional SetSafeModeRequest SetSafeModeRequest =
124;
+ optional PrintCompactionLogDagRequest PrintCompactionLogDagRequest =
125;
}
message OMResponse {
@@ -378,13 +380,14 @@ message OMResponse {
optional SnapshotMoveDeletedKeysResponse SnapshotMoveDeletedKeysResponse =
116;
optional hdds.TransferLeadershipResponseProto TransferOmLeadershipResponse
= 117;
- optional SnapshotPurgeResponse SnapshotPurgeResponse =
118;
+ optional SnapshotPurgeResponse SnapshotPurgeResponse =
118;
optional RecoverLeaseResponse RecoverLeaseResponse =
119;
- optional SetTimesResponse SetTimesResponse =
120;
+ optional SetTimesResponse SetTimesResponse =
120;
optional RefetchSecretKeyResponse RefetchSecretKeyResponse =
121;
optional ListSnapshotDiffJobResponse ListSnapshotDiffJobResponse =
122;
- optional CancelSnapshotDiffResponse cancelSnapshotDiffResponse = 123;
+ optional CancelSnapshotDiffResponse cancelSnapshotDiffResponse =
123;
optional SetSafeModeResponse SetSafeModeResponse =
124;
+ optional PrintCompactionLogDagResponse PrintCompactionLogDagResponse =
125;
}
enum Status {
@@ -1779,6 +1782,11 @@ message DeleteSnapshotRequest {
optional uint64 deletionTime = 4;
}
+message PrintCompactionLogDagRequest {
+ optional string fileName = 1;
+ optional string graphType = 2;
+}
+
message SnapshotMoveDeletedKeysRequest {
optional SnapshotInfo fromSnapshot = 1;
repeated SnapshotMoveKeyInfos nextDBKeys = 2;
@@ -1870,6 +1878,10 @@ message DeleteSnapshotResponse {
}
+message PrintCompactionLogDagResponse {
+ optional string imagePath = 1;
+}
+
message SnapshotMoveDeletedKeysResponse {
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index cc3fa2b416..f50d0c8c5f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -256,6 +256,7 @@ import static
org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.PREPARE_MARKER_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_RATIS_SNAPSHOT_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT;
@@ -304,7 +305,9 @@ import static
org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.getRaftGr
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerInterServiceProtocolProtos.OzoneManagerInterService;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus;
+import static org.apache.ozone.graph.PrintableGraph.GraphType.FILE_NAME;
+import org.apache.ozone.graph.PrintableGraph;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
import org.apache.ratis.protocol.RaftGroupId;
@@ -4643,6 +4646,31 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
bucket, jobStatus, listAll);
}
+ public String printCompactionLogDag(String fileName,
+ String graphType)
+ throws IOException {
+
+ if (StringUtils.isBlank(fileName)) {
+ fileName = "dag-" + System.currentTimeMillis();
+ }
+
+ // Append the tmp file prefix and image file suffix.
+ fileName = "/tmp" + OZONE_URI_DELIMITER + fileName + ".png";
+
+ PrintableGraph.GraphType type;
+
+ try {
+ type = PrintableGraph.GraphType.valueOf(graphType);
+ } catch (IllegalArgumentException e) {
+ type = FILE_NAME;
+ }
+
+ return getMetadataManager()
+ .getStore()
+ .getRocksDBCheckpointDiffer()
+ .pngPrintMutableGraph(fileName, type);
+ }
+
private String reconfOzoneAdmins(String newVal) {
getConfiguration().set(OZONE_ADMINISTRATORS, newVal);
Collection<String> admins =
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index fd099d395e..b1dbcf7a5e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -85,6 +85,8 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFile
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RefetchSecretKeyResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.InfoBucketResponse;
@@ -339,6 +341,13 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
SetSafeModeResponse setSafeModeResponse =
setSafeMode(request.getSetSafeModeRequest());
responseBuilder.setSetSafeModeResponse(setSafeModeResponse);
+ break;
+ case PrintCompactionLogDag:
+ PrintCompactionLogDagResponse printCompactionLogDagResponse =
+ printCompactionLogDag(request.getPrintCompactionLogDagRequest());
+ responseBuilder
+ .setPrintCompactionLogDagResponse(printCompactionLogDagResponse);
+ break;
default:
responseBuilder.setSuccess(false);
responseBuilder.setMessage("Unrecognized Command Type: " + cmdType);
@@ -1323,6 +1332,16 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
return builder.build();
}
+ private PrintCompactionLogDagResponse printCompactionLogDag(
+ PrintCompactionLogDagRequest printCompactionLogDagRequest)
+ throws IOException {
+ String imagePath = impl.printCompactionLogDag(
+ printCompactionLogDagRequest.getFileName(),
+ printCompactionLogDagRequest.getGraphType());
+ return PrintCompactionLogDagResponse.newBuilder()
+ .setImagePath(imagePath)
+ .build();
+ }
public OzoneManager getOzoneManager() {
return impl;
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index ea9d128877..66f823459b 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -631,6 +631,11 @@ public class ClientProtocolStub implements ClientProtocol {
}
+ public String printCompactionLogDag(String fileName,
+ String graphType) throws IOException {
+ return null;
+ }
+
@Override
public SnapshotDiffResponse snapshotDiff(String volumeName,
String bucketName,
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/CompactionLogDagPrinter.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/CompactionLogDagPrinter.java
new file mode 100644
index 0000000000..f8a4d88584
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/CompactionLogDagPrinter.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.debug;
+
+import org.apache.hadoop.hdds.cli.SubcommandWithParent;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientException;
+import org.apache.hadoop.ozone.shell.Handler;
+import org.apache.hadoop.ozone.shell.OzoneAddress;
+import org.kohsuke.MetaInfServices;
+import picocli.CommandLine;
+
+import java.io.IOException;
+
+/**
+ * Handler to generate image for current compaction DAG in the OM.
+ * ozone sh snapshot print-log-dag.
+ */
[email protected](
+ name = "print-log-dag",
+ aliases = "pld",
+ description = "Create an image of the current compaction log DAG in OM.")
+@MetaInfServices(SubcommandWithParent.class)
+public class CompactionLogDagPrinter extends Handler
+ implements SubcommandWithParent {
+
+ @CommandLine.Option(names = {"-f", "--file-name"},
+ description = "Name of the image file. (optional)")
+ private String fileName;
+
+ // TODO: Change graphType to enum.
+ @CommandLine.Option(names = {"-t", "--graph-type"},
+ description = "Type of node name to use in the graph image.\n" +
+ "Accepted values are: \n" +
+ " file_name: to use file name as node name in DAG,\n" +
+ " key_size: to show the no. of keys in the file along with file " +
+ "name in the DAG node name,\n" +
+ " cumulative_size: to show the cumulative size along with file " +
+ "name in the DAG node name.",
+ defaultValue = "file_name")
+ private String graphType;
+
+ @Override
+ public Class<?> getParentType() {
+ return OzoneDebug.class;
+ }
+
+ @Override
+ protected void execute(OzoneClient client, OzoneAddress address)
+ throws IOException, OzoneClientException {
+ String imagePath = client.getObjectStore()
+ .printCompactionLogDag(fileName, graphType);
+ System.out.println("DAG image is created on path: " + imagePath);
+ }
+}
diff --git a/pom.xml b/pom.xml
index 480f7de6e8..336f11366e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -306,6 +306,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<lz4.version>1.9.3</lz4.version>
<snappy.version>1.1.8</snappy.version>
<zstd.version>1.4.9</zstd.version>
+ <jgrapht.version>1.0.1</jgrapht.version>
<vault.driver.version>5.1.0</vault.driver.version>
<native.lib.tmp.dir></native.lib.tmp.dir>
@@ -1560,6 +1561,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<artifactId>vault-java-driver</artifactId>
<version>${vault.driver.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.jgrapht</groupId>
+ <artifactId>jgrapht-core</artifactId>
+ <version>${jgrapht.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jgrapht</groupId>
+ <artifactId>jgrapht-ext</artifactId>
+ <version>${jgrapht.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
