This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a2d650a HDDS-3473. Ozone chunkinfo CLI should display block file path
info (#886)
a2d650a is described below
commit a2d650a2ebe16e1c51195b6aaca7afb983c382e7
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Fri May 15 14:31:53 2020 +0530
HDDS-3473. Ozone chunkinfo CLI should display block file path info (#886)
---
.../container/common/impl/ChunkLayOutVersion.java | 16 ++++++----
.../src/main/smoketest/basic/ozone-shell.robot | 4 +--
.../src/main/smoketest/debug/ozone-debug.robot | 36 ++++++++++++++++++++++
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 17 ++++++----
.../hadoop/ozone/debug/ContainerChunkInfo.java | 10 +++---
5 files changed, 63 insertions(+), 20 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
index 055f448..ff4ba41 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java
@@ -40,17 +40,15 @@ public enum ChunkLayOutVersion {
FILE_PER_CHUNK(1, "One file per chunk") {
@Override
- public File getChunkFile(ContainerData containerData, BlockID blockID,
+ public File getChunkFile(File chunkDir, BlockID blockID,
ChunkInfo info) throws StorageContainerException {
- File chunksLoc = verifyChunkDirExists(containerData);
- return chunksLoc.toPath().resolve(info.getChunkName()).toFile();
+ return chunkDir.toPath().resolve(info.getChunkName()).toFile();
}
},
FILE_PER_BLOCK(2, "One file per block") {
@Override
- public File getChunkFile(ContainerData containerData, BlockID blockID,
+ public File getChunkFile(File chunkDir, BlockID blockID,
ChunkInfo info) throws StorageContainerException {
- File chunkDir = verifyChunkDirExists(containerData);
return new File(chunkDir, blockID.getLocalID() + ".block");
}
};
@@ -118,9 +116,15 @@ public enum ChunkLayOutVersion {
return description;
}
- public abstract File getChunkFile(ContainerData containerData,
+ public abstract File getChunkFile(File chunkDir,
BlockID blockID, ChunkInfo info) throws StorageContainerException;
+ public File getChunkFile(ContainerData containerData, BlockID blockID,
+ ChunkInfo info) throws StorageContainerException {
+ File chunksLoc = verifyChunkDirExists(containerData);
+ return getChunkFile(chunksLoc, blockID, info);
+ }
+
@Override
public String toString() {
return "ChunkLayout:v" + version;
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
index 6675fde..e358ea2 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
@@ -18,7 +18,7 @@ Documentation Test ozone shell CLI usage
Library OperatingSystem
Resource ../commonlib.robot
Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit
test user testuser testuser.keytab
-Test Timeout 3 minute
+Test Timeout 2 minute
Suite Setup Generate prefix
*** Variables ***
@@ -120,8 +120,6 @@ Test key handling
Should Not Contain ${result} NOTICE.txt.1 exists
${result} = Execute ozone sh key info
${protocol}${server}/${volume}/bb1/key1 | jq -r '. | select(.name=="key1")'
Should contain ${result} creationTime
- ${result} = Execute ozone debug chunkinfo
${protocol}${server}/${volume}/bb1/key1 | jq -r '.[]'
- Should contain ${result} chunks
${result} = Execute ozone sh key list
${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="key1") | .name'
Should Be Equal ${result} key1
Execute ozone sh key rename
${protocol}${server}/${volume}/bb1 key1 key2
diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
new file mode 100644
index 0000000..39e561a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test ozone Debug CLI
+Library OperatingSystem
+Resource ../commonlib.robot
+Test Timeout 2 minute
+Suite Setup Write key
+*** Variables ***
+
+*** Keywords ***
+Write key
+ Execute ozone sh volume create o3://om/vol1 --quota 100TB
+ Execute ozone sh bucket create o3://om/vol1/bucket1
+ Execute ozone sh key put o3://om/vol1/bucket1/debugKey
/opt/hadoop/NOTICE.txt
+
+*** Test Cases ***
+Test ozone debug
+ ${result} = Execute ozone debug chunkinfo
o3://om/vol1/bucket1/debugKey | jq -r '.[]'
+ Should contain ${result} files
+ ${result} = Execute ozone debug chunkinfo
o3://om/vol1/bucket1/debugKey | jq -r '.[].files[0]'
+ File Should Exist ${result}
+
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java
index fa57636..3d28d0a 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ChunkKeyHandler.java
@@ -36,6 +36,8 @@ import
org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@@ -102,6 +104,8 @@ public class ChunkKeyHandler extends KeyHandler {
.getLatestVersionLocations().getBlocksLatestVersionOnly();
// querying the keyLocations.The OM is queried to get containerID and
// localID pertaining to a given key
+ ChunkLayOutVersion chunkLayOutVersion = ChunkLayOutVersion
+ .getConfiguredVersion(getConf());
for (OmKeyLocationInfo keyLocation:locationInfos) {
ContainerChunkInfo containerChunkInfoVerbose = new ContainerChunkInfo();
ContainerChunkInfo containerChunkInfo = new ContainerChunkInfo();
@@ -128,9 +132,10 @@ public class ChunkKeyHandler extends KeyHandler {
chunkDetails.setChunkName(chunkInfo.getChunkName());
chunkDetails.setChunkOffset(chunkInfo.getOffset());
chunkDetailsList.add(chunkDetails);
- chunkPaths.add(getChunkLocationPath(containerData.getContainerPath())
- + File.separator
- + chunkInfo.getChunkName());
+ chunkPaths.add(chunkLayOutVersion.getChunkFile(new File(
+ getChunkLocationPath(containerData.getContainerPath())),
+ keyLocation.getBlockID(),
+ ChunkInfo.getFromProtoBuf(chunkInfo)).toString());
}
containerChunkInfoVerbose
.setContainerPath(containerData.getContainerPath());
@@ -138,7 +143,7 @@ public class ChunkKeyHandler extends KeyHandler {
.setDataNodeList(keyLocation.getPipeline().getNodes());
containerChunkInfoVerbose.setPipeline(keyLocation.getPipeline());
containerChunkInfoVerbose.setChunkInfos(chunkDetailsList);
- containerChunkInfo.setChunks(chunkPaths);
+ containerChunkInfo.setFiles(chunkPaths);
List<ChunkDataNodeDetails> chunkDataNodeDetails = new
ArrayList<ChunkDataNodeDetails>();
for (DatanodeDetails datanodeDetails:keyLocation
@@ -153,11 +158,11 @@ public class ChunkKeyHandler extends KeyHandler {
Gson gson = new GsonBuilder().create();
if (isVerbose()) {
element = gson.toJsonTree(containerChunkInfoVerbose);
- jsonObj.add("container Id :" + containerId + ""
+ jsonObj.add("container Id :" + containerId + " "
+ "blockId :" + keyLocation.getLocalID() + "", element);
} else {
element = gson.toJsonTree(containerChunkInfo);
- jsonObj.add("container Id :" + containerId + ""
+ jsonObj.add("container Id :" + containerId + " "
+ "blockId :" + keyLocation.getLocalID() + "", element);
}
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java
index 85f0bee..0e969c7 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/ContainerChunkInfo.java
@@ -32,7 +32,7 @@ public class ContainerChunkInfo {
private String containerPath;
private List<DatanodeDetails> dataNodeList;
private List<ChunkDetails> chunkInfos;
- private List<String> chunks;
+ private List<String> files;
private List<ChunkDataNodeDetails> chunkDataNodeDetails;
private UUID pipelineID;
private Pipeline pipeline;
@@ -42,8 +42,8 @@ public class ContainerChunkInfo {
this.chunkDataNodeDetails = chunkDataNodeDetails;
}
- public void setChunks(List<String> chunks) {
- this.chunks = chunks;
+ public void setFiles(List<String> files) {
+ this.files = files;
}
public void setPipelineID(UUID pipelineID) {
@@ -83,8 +83,8 @@ public class ContainerChunkInfo {
+ ", pipeline="
+ pipeline
+ '}'
- + "chunks="
- + chunks
+ + "files="
+ + files
+ "chunkdatanodeDetails="
+ chunkDataNodeDetails
+ "PipelineID="
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
