This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c5a92a2 HDDS-5824. `ozone sh volume/bucket/key list` should print
valid JSON array (#2713)
c5a92a2 is described below
commit c5a92a20614f22f2c37d599a2ae15ee59c0ecfa9
Author: Siyao Meng <[email protected]>
AuthorDate: Thu Oct 7 11:02:57 2021 -0700
HDDS-5824. `ozone sh volume/bucket/key list` should print valid JSON array
(#2713)
---
hadoop-hdds/docs/content/interface/CSI.md | 4 +-
hadoop-hdds/docs/content/interface/CSI.zh.md | 4 +-
hadoop-hdds/docs/content/interface/Cli.md | 7 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 16 ++-
.../dist/src/main/smoketest/basic/links.robot | 8 +-
.../src/main/smoketest/basic/ozone-shell-lib.robot | 10 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../dist/src/main/smoketest/ozonefs/ozonefs.robot | 18 ++--
.../hadoop/ozone/shell/TestOzoneShellHA.java | 112 ++++++++++++++++++---
.../org/apache/hadoop/ozone/shell/Handler.java | 17 ++++
.../ozone/shell/bucket/ListBucketHandler.java | 10 +-
.../hadoop/ozone/shell/keys/ListKeyHandler.java | 8 +-
.../ozone/shell/volume/ListVolumeHandler.java | 9 +-
13 files changed, 162 insertions(+), 63 deletions(-)
diff --git a/hadoop-hdds/docs/content/interface/CSI.md
b/hadoop-hdds/docs/content/interface/CSI.md
index f7dbfd3..59b24c9 100644
--- a/hadoop-hdds/docs/content/interface/CSI.md
+++ b/hadoop-hdds/docs/content/interface/CSI.md
@@ -70,7 +70,7 @@ Attach the pod scm-0 and put a key into the /s3v/pvc* bucket.
```bash
kubectl exec -it scm-0 bash
[hadoop@scm-0 ~]$ ozone sh bucket list s3v
-{
+[ {
"metadata" : { },
"volumeName" : "s3v",
"name" : "pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b",
@@ -78,7 +78,7 @@ kubectl exec -it scm-0 bash
"versioning" : false,
"creationTime" : "2020-06-11T08:19:47.469Z",
"encryptionKeyName" : null
-}
+} ]
[hadoop@scm-0 ~]$ ozone sh key put
/s3v/pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b/A LICENSE.txt
```
diff --git a/hadoop-hdds/docs/content/interface/CSI.zh.md
b/hadoop-hdds/docs/content/interface/CSI.zh.md
index 505c911..b243d1e 100644
--- a/hadoop-hdds/docs/content/interface/CSI.zh.md
+++ b/hadoop-hdds/docs/content/interface/CSI.zh.md
@@ -69,7 +69,7 @@ kubectl create -f /ozone/kubernetes/examples/ozone/pv-test
```bash
kubectl exec -it scm-0 bash
[hadoop@scm-0 ~]$ ozone sh bucket list s3v
-{
+[ {
"metadata" : { },
"volumeName" : "s3v",
"name" : "pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b",
@@ -77,7 +77,7 @@ kubectl exec -it scm-0 bash
"versioning" : false,
"creationTime" : "2020-06-11T08:19:47.469Z",
"encryptionKeyName" : null
-}
+} ]
[hadoop@scm-0 ~]$ ozone sh key put
/s3v/pvc-861e2d8b-2232-4cd1-b43c-c0c26697ab6b/A LICENSE.txt
```
diff --git a/hadoop-hdds/docs/content/interface/Cli.md
b/hadoop-hdds/docs/content/interface/Cli.md
index cbdc499..3cbe518 100644
--- a/hadoop-hdds/docs/content/interface/Cli.md
+++ b/hadoop-hdds/docs/content/interface/Cli.md
@@ -116,7 +116,7 @@ $ ozone sh volume info /vol1
```shell
$ ozone sh volume list /
-{
+[ {
"metadata" : { },
"name" : "s3v",
"admin" : "hadoop",
@@ -135,8 +135,9 @@ $ ozone sh volume list /
"aclList" : [ "ALL" ]
} ],
"quota" : 1152921504606846976
-}
-....
+}, {
+ ....
+} ]
```
## Bucket operations
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
index 1cc7e82..b46e48d 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/JsonUtils.java
@@ -25,6 +25,8 @@ import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
@@ -37,13 +39,13 @@ public final class JsonUtils {
// ObjectMapper is thread safe as long as we always configure instance
// before use.
private static final ObjectMapper MAPPER;
- private static final ObjectWriter WRITTER;
+ private static final ObjectWriter WRITER;
static {
MAPPER = new ObjectMapper()
.setSerializationInclusion(JsonInclude.Include.NON_NULL)
.registerModule(new JavaTimeModule())
.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
- WRITTER = MAPPER.writerWithDefaultPrettyPrinter();
+ WRITER = MAPPER.writerWithDefaultPrettyPrinter();
}
private JsonUtils() {
@@ -52,13 +54,21 @@ public final class JsonUtils {
public static String toJsonStringWithDefaultPrettyPrinter(Object obj)
throws IOException {
- return WRITTER.writeValueAsString(obj);
+ return WRITER.writeValueAsString(obj);
}
public static String toJsonString(Object obj) throws IOException {
return MAPPER.writeValueAsString(obj);
}
+ public static ArrayNode createArrayNode() {
+ return MAPPER.createArrayNode();
+ }
+
+ public static ObjectNode createObjectNode(Object next) {
+ return MAPPER.valueToTree(next);
+ }
+
/**
* Deserialize a list of elements from a given string,
* each element in the list is in the given type.
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
b/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
index 9b07af1..7c2af60 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/links.robot
@@ -86,15 +86,15 @@ Key read passthrough
Key Should Match Local File ${source}/bucket1/key2
/opt/hadoop/NOTICE.txt
Key list passthrough
- ${target_list} = Execute ozone sh key list
${target}/link1 | jq -r '.name'
- ${source_list} = Execute ozone sh key list
${source}/bucket1 | jq -r '.name'
+ ${target_list} = Execute ozone sh key list
${target}/link1 | jq -r '.[].name'
+ ${source_list} = Execute ozone sh key list
${source}/bucket1 | jq -r '.[].name'
Should Be Equal ${target_list}
${source_list}
Should Contain ${source_list} key1
Should Contain ${source_list} key2
Key delete passthrough
Execute ozone sh key delete
${target}/link1/key2
- ${source_list} = Execute ozone sh key list
${source}/bucket1 | jq -r '.name'
+ ${source_list} = Execute ozone sh key list
${source}/bucket1 | jq -r '.[].name'
Should Not Contain ${source_list} key2
Bucket list contains links
@@ -155,5 +155,5 @@ Source bucket not affected by deleting link
Execute ozone sh bucket delete
${target}/link1
${bucket_list} = Execute ozone sh bucket list
${target}
Should Not Contain ${bucket_list} link1
- ${source_list} = Execute ozone sh key list
${source}/bucket1 | jq -r '.name'
+ ${source_list} = Execute ozone sh key list
${source}/bucket1 | jq -r '.[].name'
Should Contain ${source_list} key1
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
index 7671e20..56f100e 100644
--- a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell-lib.robot
@@ -34,9 +34,9 @@ Test ozone shell
Should contain ${result} VOLUME_NOT_FOUND
${result} = Execute ozone sh volume create
${protocol}${server}/${volume} --space-quota 100TB --namespace-quota 100
Should not contain ${result} Failed
- ${result} = Execute ozone sh volume list
${protocol}${server}/ | jq -r '. | select(.name=="${volume}")'
+ ${result} = Execute ozone sh volume list
${protocol}${server}/ | jq -r '.[] | select(.name=="${volume}")'
Should contain ${result} creationTime
- ${result} = Execute ozone sh volume list | jq -r '. |
select(.name=="${volume}")'
+ ${result} = Execute ozone sh volume list | jq -r '.[] |
select(.name=="${volume}")'
Should contain ${result} creationTime
# TODO: Disable updating the owner, acls should be used to give access to
other user.
Execute ozone sh volume setquota
${protocol}${server}/${volume} --space-quota 10TB --namespace-quota 100
@@ -56,7 +56,7 @@ Test ozone shell
Should Be Equal ${result} 1099511627776
${result} = Execute ozone sh bucket info
${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="bb1") |
.quotaInNamespace'
Should Be Equal ${result} 1000
- ${result} = Execute ozone sh bucket list
${protocol}${server}/${volume}/ | jq -r '. | select(.name=="bb1") | .volumeName'
+ ${result} = Execute ozone sh bucket list
${protocol}${server}/${volume}/ | jq -r '.[] | select(.name=="bb1") |
.volumeName'
Should Be Equal ${result} ${volume}
Run Keyword Test key handling ${protocol}
${server} ${volume}
Execute ozone sh volume clrquota --space-quota
${protocol}${server}/${volume}
@@ -146,10 +146,10 @@ Test key handling
Should Not Contain ${result} NOTICE.txt.1 exists
${result} = Execute ozone sh key info
${protocol}${server}/${volume}/bb1/key1 | jq -r '. | select(.name=="key1")'
Should contain ${result} creationTime
- ${result} = Execute ozone sh key list
${protocol}${server}/${volume}/bb1 | jq -r '. | select(.name=="key1") | .name'
+ ${result} = Execute ozone sh key list
${protocol}${server}/${volume}/bb1 | jq -r '.[] | select(.name=="key1") | .name'
Should Be Equal ${result} key1
Execute ozone sh key rename
${protocol}${server}/${volume}/bb1 key1 key2
- ${result} = Execute ozone sh key list
${protocol}${server}/${volume}/bb1 | jq -r '.name'
+ ${result} = Execute ozone sh key list
${protocol}${server}/${volume}/bb1 | jq -r '.[].name'
Should Be Equal ${result} key2
Execute ozone sh key delete
${protocol}${server}/${volume}/bb1/key2
diff --git a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
index 0c08bb1..c3d3c7c 100644
--- a/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/omha/testOMHA.robot
@@ -73,7 +73,7 @@ Write Test File
${testFilePath} = Set Variable ${TEMPDIR}/${fileName}
Copy File ${TEST_FILE}
${testFilePath}
Execute ozone fs -copyFromLocal
${testFilePath} o3fs://${BUCKET}.${VOLUME}.${OM_SERVICE_ID}/
- ${result} = Execute ozone sh key list
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} | jq -r '.name'
+ ${result} = Execute ozone sh key list
o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} | jq -r '.[].name'
Should contain ${result}
${fileName}
Remove File ${testFilePath}
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
index 8b8220c..f888d22 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
@@ -38,19 +38,19 @@ List non-existent bucket
Create dir with parents
Execute ozone fs -mkdir -p ${DEEP_URL}
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should contain ${result} ${DEEP_DIR}
Copy from local
Execute ozone fs -copyFromLocal NOTICE.txt
${DEEP_URL}/
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should contain ${result} NOTICE.txt
${result} = Execute ozone sh key info
${VOLUME}/${BUCKET}/${DEEP_DIR}/NOTICE.txt | jq -r '.replicationFactor'
Should Be Equal ${result} 3
Put
Execute ozone fs -put NOTICE.txt
${DEEP_URL}/PUTFILE.txt
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should contain ${result} PUTFILE.txt
List
@@ -60,14 +60,14 @@ List
Move
Execute ozone fs -mv ${DEEP_URL}/NOTICE.txt
${DEEP_URL}/MOVED.TXT
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should contain ${result} MOVED.TXT
Should not contain ${result} NOTICE.txt
Copy within FS
[Setup] Execute ozone fs -mkdir -p ${DEEP_URL}/subdir1
Execute ozone fs -cp ${DEEP_URL}/MOVED.TXT
${DEEP_URL}/subdir1/NOTICE.txt
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should contain ${result} subdir1/NOTICE.txt
${result} = Execute ozone fs -ls ${DEEP_URL}/subdir1/
@@ -79,17 +79,17 @@ Cat file
Delete file
Execute ozone fs -rm -skipTrash
${DEEP_URL}/subdir1/NOTICE.txt
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should not contain ${result} NOTICE.txt
Delete dir
${result} = Execute ozone fs -rmdir ${DEEP_URL}/subdir1/
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should not contain ${result} subdir1
Touch file
Execute ozone fs -touch
${DEEP_URL}/TOUCHFILE-${SCHEME}.txt
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should contain ${result}
TOUCHFILE-${SCHEME}.txt
Delete file with Trash
@@ -103,7 +103,7 @@ Delete file with Trash
Delete recursively
Execute ozone fs -mkdir -p ${DEEP_URL}/subdir2
Execute ozone fs -rm -skipTrash -r
${DEEP_URL}/subdir2
- ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.name'
+ ${result} = Execute ozone sh key list ${VOLUME}/${BUCKET}
| jq -r '.[].name'
Should not contain ${result} ${DEEP_DIR}/subdir2
List recursively
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index b32ff32..a8b1980 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -22,7 +22,9 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.UUID;
@@ -37,7 +39,6 @@ import org.apache.hadoop.fs.ozone.OzoneFsShell;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
@@ -49,6 +50,8 @@ import org.apache.hadoop.fs.TrashPolicy;
import org.apache.hadoop.ozone.om.TrashPolicyOzone;
import com.google.common.base.Strings;
+import com.google.gson.Gson;
+import com.google.gson.internal.LinkedTreeMap;
import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
@@ -58,6 +61,7 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import org.junit.Before;
@@ -83,6 +87,8 @@ public class TestOzoneShellHA {
private static final Logger LOG =
LoggerFactory.getLogger(TestOzoneShellHA.class);
+ private static final String DEFAULT_ENCODING = UTF_8.name();
+
/**
* Set the timeout for every test.
*/
@@ -91,6 +97,7 @@ public class TestOzoneShellHA {
private static File baseDir;
private static File testFile;
+ private static String testFilePathString;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
private static OzoneShell ozoneShell = null;
@@ -121,7 +128,8 @@ public class TestOzoneShellHA {
baseDir = new File(path);
baseDir.mkdirs();
- testFile = new File(path + OzoneConsts.OZONE_URI_DELIMITER + "testFile");
+ testFilePathString = path + OZONE_URI_DELIMITER + "testFile";
+ testFile = new File(testFilePathString);
testFile.getParentFile().mkdirs();
testFile.createNewFile();
@@ -158,8 +166,8 @@ public class TestOzoneShellHA {
@Before
public void setup() throws UnsupportedEncodingException {
- System.setOut(new PrintStream(out, false, UTF_8.name()));
- System.setErr(new PrintStream(err, false, UTF_8.name()));
+ System.setOut(new PrintStream(out, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(err, false, DEFAULT_ENCODING));
}
@After
@@ -311,8 +319,7 @@ public class TestOzoneShellHA {
"bucket", "create", "o3://" + omServiceId + volumeName + bucketName};
execute(ozoneShell, args);
- String keyName = volumeName + bucketName +
- OzoneConsts.OZONE_URI_DELIMITER + "key";
+ String keyName = volumeName + bucketName + OZONE_URI_DELIMITER + "key";
for (int i = 0; i < 100; i++) {
args = new String[] {
"key", "put", "o3://" + omServiceId + keyName + i,
@@ -325,7 +332,7 @@ public class TestOzoneShellHA {
* Helper function to get nums of keys from info of listing command.
*/
private int getNumOfKeys() throws UnsupportedEncodingException {
- return out.toString(UTF_8.name()).split("key").length - 1;
+ return out.toString(DEFAULT_ENCODING).split("key").length - 1;
}
/**
@@ -337,7 +344,7 @@ public class TestOzoneShellHA {
execute(ozoneShell, args);
String bucketName =
- volumeName + OzoneConsts.OZONE_URI_DELIMITER + "testbucket";
+ volumeName + OZONE_URI_DELIMITER + "testbucket";
for (int i = 0; i < numOfBuckets; i++) {
args = new String[] {
"bucket", "create", "o3://" + omServiceId + bucketName + i};
@@ -350,10 +357,17 @@ public class TestOzoneShellHA {
*/
private int getNumOfBuckets(String bucketPrefix)
throws UnsupportedEncodingException {
- return out.toString(UTF_8.name())
- .split(bucketPrefix).length - 1;
+ return out.toString(DEFAULT_ENCODING).split(bucketPrefix).length - 1;
}
+ /**
+ * Parse output into ArrayList with Gson.
+ * @return ArrayList
+ */
+ private ArrayList<LinkedTreeMap<String, String>> parseOutputIntoArrayList()
+ throws UnsupportedEncodingException {
+ return new Gson().fromJson(out.toString(DEFAULT_ENCODING),
ArrayList.class);
+ }
/**
* Tests ozone sh command URI parsing with volume and bucket create commands.
@@ -434,7 +448,8 @@ public class TestOzoneShellHA {
args = new String[] {"key", "list", startKey, destinationBucket};
out.reset();
execute(ozoneShell, args);
- Assert.assertEquals(0, out.size());
+ // Expect empty JSON array
+ Assert.assertEquals(0, parseOutputIntoArrayList().size());
Assert.assertEquals(0, getNumOfKeys());
// Part of listing buckets test.
@@ -456,7 +471,8 @@ public class TestOzoneShellHA {
out.reset();
args = new String[] {"bucket", "list", startBucket, destinationVolume};
execute(ozoneShell, args);
- Assert.assertEquals(0, out.size());
+ // Expect empty JSON array
+ Assert.assertEquals(0, parseOutputIntoArrayList().size());
Assert.assertEquals(0, getNumOfBuckets("testbucket"));
}
@@ -855,4 +871,76 @@ public class TestOzoneShellHA {
objectStore.getVolume("vol4").deleteBucket("buck4");
objectStore.deleteVolume("vol4");
}
+
+ @Test
+ public void testListVolumeBucketKeyShouldPrintValidJsonArray()
+ throws UnsupportedEncodingException {
+
+ final List<String> testVolumes =
+ Arrays.asList("jsontest-vol1", "jsontest-vol2", "jsontest-vol3");
+ final List<String> testBuckets =
+ Arrays.asList("v1-bucket1", "v1-bucket2", "v1-bucket3");
+ final List<String> testKeys = Arrays.asList("key1", "key2", "key3");
+
+ final String firstVolumePrefix = testVolumes.get(0) + OZONE_URI_DELIMITER;
+ final String keyPathPrefix = firstVolumePrefix +
+ testBuckets.get(0) + OZONE_URI_DELIMITER;
+
+ // Create test volumes, buckets and keys
+ testVolumes.forEach(vol -> execute(ozoneShell, new String[] {
+ "volume", "create", vol}));
+ testBuckets.forEach(bucket -> execute(ozoneShell, new String[] {
+ "bucket", "create", firstVolumePrefix + bucket}));
+ testKeys.forEach(key -> execute(ozoneShell, new String[] {
+ "key", "put", keyPathPrefix + key, testFilePathString}));
+
+ // ozone sh volume list
+ out.reset();
+ execute(ozoneShell, new String[] {"volume", "list"});
+
+ // Expect valid JSON array
+ final ArrayList<LinkedTreeMap<String, String>> volumeListOut =
+ parseOutputIntoArrayList();
+ // Can include s3v and volumes from other test cases that aren't cleaned
up,
+ // hence >= instead of equals.
+ Assert.assertTrue(volumeListOut.size() >= testVolumes.size());
+ final HashSet<String> volumeSet = new HashSet<>(testVolumes);
+ volumeListOut.forEach(map -> volumeSet.remove(map.get("name")));
+ // Should have found all the volumes created for this test
+ Assert.assertEquals(0, volumeSet.size());
+
+ // ozone sh bucket list jsontest-vol1
+ out.reset();
+ execute(ozoneShell, new String[] {"bucket", "list", firstVolumePrefix});
+
+ // Expect valid JSON array as well
+ final ArrayList<LinkedTreeMap<String, String>> bucketListOut =
+ parseOutputIntoArrayList();
+ Assert.assertEquals(testBuckets.size(), bucketListOut.size());
+ final HashSet<String> bucketSet = new HashSet<>(testBuckets);
+ bucketListOut.forEach(map -> bucketSet.remove(map.get("name")));
+ // Should have found all the buckets created for this test
+ Assert.assertEquals(0, bucketSet.size());
+
+ // ozone sh key list jsontest-vol1/v1-bucket1
+ out.reset();
+ execute(ozoneShell, new String[] {"key", "list", keyPathPrefix});
+
+ // Expect valid JSON array as well
+ final ArrayList<LinkedTreeMap<String, String>> keyListOut =
+ parseOutputIntoArrayList();
+ Assert.assertEquals(testKeys.size(), keyListOut.size());
+ final HashSet<String> keySet = new HashSet<>(testKeys);
+ keyListOut.forEach(map -> keySet.remove(map.get("name")));
+ // Should have found all the keys put for this test
+ Assert.assertEquals(0, keySet.size());
+
+ // Clean up
+ testKeys.forEach(key -> execute(ozoneShell, new String[] {
+ "key", "delete", keyPathPrefix + key}));
+ testBuckets.forEach(bucket -> execute(ozoneShell, new String[] {
+ "bucket", "delete", firstVolumePrefix + bucket}));
+ testVolumes.forEach(vol -> execute(ozoneShell, new String[] {
+ "volume", "delete", vol}));
+ }
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java
index d8f49b9..ae5edf5 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/Handler.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.ozone.shell;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.Iterator;
import java.util.concurrent.Callable;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import org.apache.hadoop.hdds.cli.GenericParentCommand;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -118,6 +120,21 @@ public abstract class Handler implements Callable<Void> {
out().println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(o));
}
+ /**
+ * Pretty print the result as a valid JSON array to out().
+ * @return Number of entries actually printed.
+ */
+ protected int printAsJsonArray(Iterator<?> iterator, int limit) {
+ int counter = 0;
+ final ArrayNode arrayNode = JsonUtils.createArrayNode();
+ while (limit > counter && iterator.hasNext()) {
+ arrayNode.add(JsonUtils.createObjectNode(iterator.next()));
+ counter++;
+ }
+ out().println(arrayNode.toPrettyString());
+ return counter;
+ }
+
protected void printMsg(String msg) {
out().println(msg);
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java
index 6c01af3..c3178fb 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/ListBucketHandler.java
@@ -51,16 +51,10 @@ public class ListBucketHandler extends VolumeHandler {
Iterator<? extends OzoneBucket> bucketIterator =
vol.listBuckets(listOptions.getPrefix(), listOptions.getStartItem());
- int counter = 0;
- while (listOptions.getLimit() > counter && bucketIterator.hasNext()) {
- printObjectAsJson(bucketIterator.next());
-
- counter++;
- }
+ int counter = printAsJsonArray(bucketIterator, listOptions.getLimit());
if (isVerbose()) {
- out().printf("Found : %d buckets for volume : %s ",
- counter, volumeName);
+ out().printf("Found : %d buckets for volume : %s ", counter, volumeName);
}
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
index 00d04fb..741619b 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/keys/ListKeyHandler.java
@@ -57,13 +57,7 @@ public class ListKeyHandler extends BucketHandler {
listOptions.getPrefix(), listOptions.getStartItem());
int maxKeyLimit = listOptions.getLimit();
-
- int counter = 0;
- while (maxKeyLimit > counter && keyIterator.hasNext()) {
- OzoneKey ozoneKey = keyIterator.next();
- printObjectAsJson(ozoneKey);
- counter++;
- }
+ int counter = printAsJsonArray(keyIterator, maxKeyLimit);
// More keys were returned notify about max length
if (keyIterator.hasNext()) {
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java
index fb0760e..70ccb8d 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/ListVolumeHandler.java
@@ -84,15 +84,10 @@ public class ListVolumeHandler extends Handler {
listOptions.getPrefix(), listOptions.getStartItem());
}
- int counter = 0;
- while (listOptions.getLimit() > counter && volumeIterator.hasNext()) {
- printObjectAsJson(volumeIterator.next());
- counter++;
- }
+ int counter = printAsJsonArray(volumeIterator, listOptions.getLimit());
if (isVerbose()) {
- out().printf("Found : %d volumes for user : %s ", counter,
- userName);
+ out().printf("Found : %d volumes for user : %s ", counter, userName);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
