devmadhuu commented on code in PR #6658:
URL: https://github.com/apache/ozone/pull/6658#discussion_r1600895423
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java:
##########
@@ -674,6 +685,645 @@ public Response getDeletedDirectorySummary() {
return Response.ok(dirSummary).build();
}
+ /**
+ * This API will list out limited 'count' number of keys after applying
below filters in API parameters:
+ * Default Values of API param filters:
+ * -- replicationType - empty string and filter will not be applied, so
list out all keys irrespective of
+ * replication type.
+ * -- creationTime - empty string and filter will not be applied, so list
out keys irrespective of age.
+ * -- keySize - 0 bytes, which means all keys greater than zero bytes
will be listed, effectively all.
+ * -- startPrefix - /
+ * -- prevKey - ""
+ * -- limit - 1000
+ *
+ * @param replicationType Filter for RATIS or EC replication keys
+ * @param creationDate Filter for keys created after creationDate in
"MM-dd-yyyy HH:mm:ss" string format.
+ * @param keySize Filter for Keys greater than keySize in bytes.
+ * @param startPrefix Filter for startPrefix path.
+ * @param prevKey rocksDB last key of page requested.
+ * @param limit Filter for limited count of keys.
+ *
+ * @return the list of keys in JSON structured format as per respective
bucket layout.
+ *
+ * Now lets consider, we have following OBS, LEGACY and FSO bucket key/files
namespace tree structure
+ *
+ * For OBS Bucket
+ *
+ * /volume1/obs-bucket/key1
+ * /volume1/obs-bucket/key1/key2
+ * /volume1/obs-bucket/key1/key2/key3
+ * /volume1/obs-bucket/key4
+ * /volume1/obs-bucket/key5
+ * /volume1/obs-bucket/key6
+ * For LEGACY Bucket
+ *
+ * /volume1/legacy-bucket/key
+ * /volume1/legacy-bucket/key1/key2
+ * /volume1/legacy-bucket/key1/key2/key3
+ * /volume1/legacy-bucket/key4
+ * /volume1/legacy-bucket/key5
+ * /volume1/legacy-bucket/key6
+ * For FSO Bucket
+ *
+ * /volume1/fso-bucket/dir1/dir2/dir3
+ * /volume1/fso-bucket/dir1/testfile
+ * /volume1/fso-bucket/dir1/file1
+ * /volume1/fso-bucket/dir1/dir2/testfile
+ * /volume1/fso-bucket/dir1/dir2/file1
+ * /volume1/fso-bucket/dir1/dir2/dir3/testfile
+ * /volume1/fso-bucket/dir1/dir2/dir3/file1
+ * Input Request for OBS bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/obs-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/obs-bucket",
+ * "replicatedDataSize": 20971520,
+ * "unReplicatedDataSize": 20971520,
+ * "keyCount": 2,
+ * "lastKey": "/volume1/obs-bucket/key1/key2",
+ * "keys": [
+ * {
+ * "key": "/volume1/obs-bucket/key1",
+ * "path": "key1",
+ * "inStateSince": 1715174266126,
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715174266126,
+ * "modificationTime": 1715174267480,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key1/key2",
+ * "path": "key1/key2",
+ * "inStateSince": 1715174269510,
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715174269510,
+ * "modificationTime": 1715174270410,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ * Input Request for FSO bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/fso-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/fso-bucket",
+ * "replicatedDataSize": 62914560,
+ * "unReplicatedDataSize": 20971520,
+ * "keyCount": 2,
+ * "lastKey":
"/-9223372036854775552/-9223372036854775040/-9223372036854774525/testfile",
+ * "keys": [
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854775040/-9223372036854774525/file1",
+ * "path": "file1",
+ * "inStateSince": 1715174237440,
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715174237440,
+ * "modificationTime": 1715174238161,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854775040/-9223372036854774525/testfile",
+ * "path": "testfile",
+ * "inStateSince": 1715174234840,
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715174234840,
+ * "modificationTime": 1715174235562,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ * Input Request for Legacy bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/legacy-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/legacy-bucket",
+ * "replicatedDataSize": 52428800,
+ * "unReplicatedDataSize": 52428800,
+ * "keyCount": 2,
+ * "lastKey": "/volume1/legacy-bucket/key1/key2",
+ * "keys": [
+ * {
+ * "key": "/volume1/legacy-bucket/key1",
+ * "path": "key1",
+ * "inStateSince": 1715174303702,
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715174303702,
+ * "modificationTime": 1715174304619,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/legacy-bucket/key1/key2",
+ * "path": "key1/key2",
+ * "inStateSince": 1715174306641,
+ * "size": 41943040,
+ * "replicatedSize": 41943040,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715174306641,
+ * "modificationTime": 1715174307994,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ * ********************************************************
+ * @throws IOException
+ */
+ @GET
+ @Path("/listKeys")
+ @SuppressWarnings("methodlength")
+ public Response listKeys(@QueryParam("replicationType") String
replicationType,
+ @QueryParam("creationDate") String creationDate,
+ @DefaultValue(DEFAULT_KEY_SIZE)
@QueryParam("keySize") long keySize,
+ @DefaultValue(OM_KEY_PREFIX)
@QueryParam("startPrefix") String startPrefix,
+ @DefaultValue(StringUtils.EMPTY)
@QueryParam(RECON_QUERY_PREVKEY) String prevKey,
+ @DefaultValue(DEFAULT_FETCH_COUNT)
@QueryParam("limit") int limit) {
+
+
+ // This API supports startPrefix from bucket level.
+ if (startPrefix == null || startPrefix.length() == 0) {
+ return Response.status(Response.Status.BAD_REQUEST).build();
+ }
+ String[] names = startPrefix.split(OM_KEY_PREFIX);
Review Comment:
> it must have atleast 3 part, volume, bucket and key. what if need list all
keys in a bucket?
Here assumption is that client will provide `startPrefix` path beginning
with `"/"`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
