devmadhuu commented on code in PR #6658:
URL: https://github.com/apache/ozone/pull/6658#discussion_r1611497309
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java:
##########
@@ -674,6 +685,624 @@ public Response getDeletedDirectorySummary() {
return Response.ok(dirSummary).build();
}
+ /**
+ * This API will list out limited 'count' number of keys after applying
below filters in API parameters:
+ * Default Values of API param filters:
+ * -- replicationType - empty string and filter will not be applied, so
list out all keys irrespective of
+ * replication type.
+ * -- creationTime - empty string and filter will not be applied, so list
out keys irrespective of age.
+ * -- keySize - 0 bytes, which means all keys greater than zero bytes
will be listed, effectively all.
+ * -- startPrefix - /, API assumes that startPrefix path always starts
with /. E.g. /volume/bucket
+ * -- prevKey - ""
+ * -- limit - 1000
+ *
+ * @param replicationType Filter for RATIS or EC replication keys
+ * @param creationDate Filter for keys created after creationDate in
"MM-dd-yyyy HH:mm:ss" string format.
+ * @param keySize Filter for Keys greater than keySize in bytes.
+ * @param startPrefix Filter for startPrefix path.
+ * @param prevKey rocksDB last key of page requested.
+ * @param limit Filter for limited count of keys.
+ *
+ * @return the list of keys in JSON structured format as per respective
bucket layout.
+ *
+ * Now lets consider, we have following OBS, LEGACY and FSO bucket key/files
namespace tree structure
+ *
+ * For OBS Bucket
+ *
+ * /volume1/obs-bucket/key1
+ * /volume1/obs-bucket/key1/key2
+ * /volume1/obs-bucket/key1/key2/key3
+ * /volume1/obs-bucket/key4
+ * /volume1/obs-bucket/key5
+ * /volume1/obs-bucket/key6
+ * For LEGACY Bucket
+ *
+ * /volume1/legacy-bucket/key
+ * /volume1/legacy-bucket/key1/key2
+ * /volume1/legacy-bucket/key1/key2/key3
+ * /volume1/legacy-bucket/key4
+ * /volume1/legacy-bucket/key5
+ * /volume1/legacy-bucket/key6
+ * For FSO Bucket
+ *
+ * /volume1/fso-bucket/dir1/dir2/dir3
+ * /volume1/fso-bucket/dir1/testfile
+ * /volume1/fso-bucket/dir1/file1
+ * /volume1/fso-bucket/dir1/dir2/testfile
+ * /volume1/fso-bucket/dir1/dir2/file1
+ * /volume1/fso-bucket/dir1/dir2/dir3/testfile
+ * /volume1/fso-bucket/dir1/dir2/dir3/file1
+ * Input Request for OBS bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/obs-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/obs-bucket",
+ * "replicatedDataSize": 62914560,
+ * "unReplicatedDataSize": 62914560,
+ * "lastKey": "/volume1/obs-bucket/key6",
+ * "keys": [
+ * {
+ * "key": "/volume1/obs-bucket/key1",
+ * "path": "volume1/obs-bucket/key1",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781418742,
+ * "modificationTime": 1715781419762,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key1/key2",
+ * "path": "volume1/obs-bucket/key1/key2",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781421716,
+ * "modificationTime": 1715781422723,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key1/key2/key3",
+ * "path": "volume1/obs-bucket/key1/key2/key3",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781424718,
+ * "modificationTime": 1715781425598,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key4",
+ * "path": "volume1/obs-bucket/key4",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781427561,
+ * "modificationTime": 1715781428407,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key5",
+ * "path": "volume1/obs-bucket/key5",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781430347,
+ * "modificationTime": 1715781431185,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key6",
+ * "path": "volume1/obs-bucket/key6",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781433154,
+ * "modificationTime": 1715781433962,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ * Input Request for FSO bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/fso-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/fso-bucket",
+ * "replicatedDataSize": 188743680,
+ * "unReplicatedDataSize": 62914560,
+ * "lastKey":
"/-9223372036854775552/-9223372036854774016/-9223372036854773503/testfile",
+ * "keys": [
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773501/file1",
+ * "path": "volume1/fso-bucket/dir1/dir2/dir3/file1",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781411785,
+ * "modificationTime": 1715781415119,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773501/testfile",
+ * "path": "volume1/fso-bucket/dir1/dir2/dir3/testfile",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781409146,
+ * "modificationTime": 1715781409882,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773502/file1",
+ * "path": "volume1/fso-bucket/dir1/dir2/file1",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781406333,
+ * "modificationTime": 1715781407140,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773502/testfile",
+ * "path": "volume1/fso-bucket/dir1/dir2/testfile",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781403655,
+ * "modificationTime": 1715781404460,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773503/file1",
+ * "path": "volume1/fso-bucket/dir1/file1",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781400980,
+ * "modificationTime": 1715781401768,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773503/testfile",
+ * "path": "volume1/fso-bucket/dir1/testfile",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781397636,
+ * "modificationTime": 1715781398919,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ *
+ * ********************************************************
+ * @throws IOException
+ */
+ @GET
+ @Path("/listKeys")
+ @SuppressWarnings("methodlength")
+ public Response listKeys(@QueryParam("replicationType") String
replicationType,
+ @QueryParam("creationDate") String creationDate,
+ @DefaultValue(DEFAULT_KEY_SIZE)
@QueryParam("keySize") long keySize,
+ @DefaultValue(OM_KEY_PREFIX)
@QueryParam("startPrefix") String startPrefix,
+ @DefaultValue(StringUtils.EMPTY)
@QueryParam(RECON_QUERY_PREVKEY) String prevKey,
+ @DefaultValue(DEFAULT_FETCH_COUNT)
@QueryParam("limit") int limit) {
+
+
+ // This API supports startPrefix from bucket level.
+ if (startPrefix == null || startPrefix.length() == 0) {
+ return Response.status(Response.Status.BAD_REQUEST).build();
+ }
+ String[] names = startPrefix.split(OM_KEY_PREFIX);
+ if (names.length < 3) {
+ return Response.status(Response.Status.BAD_REQUEST).build();
+ }
+ ListKeysResponse listKeysResponse = new ListKeysResponse();
+ if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
+ listKeysResponse.setStatus(ResponseStatus.INITIALIZING);
+ return Response.ok(listKeysResponse).build();
+ }
+ ParamInfo paramInfo = new ParamInfo(replicationType, creationDate,
keySize, startPrefix, prevKey,
+ limit, false, "");
+ Response response = getListKeysResponse(paramInfo);
+ if (response.getEntity() instanceof ListKeysResponse) {
+ listKeysResponse = (ListKeysResponse) response.getEntity();
+ }
+
+ List<KeyEntityInfo> keyInfoList = listKeysResponse.getKeys();
+ if (!keyInfoList.isEmpty()) {
+ listKeysResponse.setLastKey(keyInfoList.get(keyInfoList.size() -
1).getKey());
+ }
+ return Response.ok(listKeysResponse).build();
+ }
+
+ private Response getListKeysResponse(ParamInfo paramInfo) {
+ try {
+ paramInfo.setLimit(Math.max(0, paramInfo.getLimit())); // Ensure limit
is non-negative
+ ListKeysResponse listKeysResponse = new ListKeysResponse();
+ listKeysResponse.setPath(paramInfo.getStartPrefix());
+ long replicatedTotal = 0;
+ long unreplicatedTotal = 0;
+ boolean keysFound = false; // Flag to track if any keys are found
+
+ // Search keys from non-FSO layout.
+ Map<String, OmKeyInfo> obsKeys;
+ Table<String, OmKeyInfo> keyTable =
+ omMetadataManager.getKeyTable(BucketLayout.LEGACY);
+ obsKeys = retrieveKeysFromTable(keyTable, paramInfo);
+ for (Map.Entry<String, OmKeyInfo> entry : obsKeys.entrySet()) {
+ keysFound = true;
+ KeyEntityInfo keyEntityInfo =
+ createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
+
+ listKeysResponse.getKeys().add(keyEntityInfo);
+ replicatedTotal += entry.getValue().getReplicatedSize();
+ unreplicatedTotal += entry.getValue().getDataSize();
+ }
+
+ // Search keys from FSO layout.
+ Map<String, OmKeyInfo> fsoKeys = searchKeysInFSO(paramInfo);
+ for (Map.Entry<String, OmKeyInfo> entry : fsoKeys.entrySet()) {
+ keysFound = true;
+ KeyEntityInfo keyEntityInfo =
+ createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
+
+ listKeysResponse.getKeys().add(keyEntityInfo);
+ replicatedTotal += entry.getValue().getReplicatedSize();
+ unreplicatedTotal += entry.getValue().getDataSize();
+ }
+
+ // If no keys were found, return a response indicating that no keys
matched
+ if (!keysFound) {
+ return
ReconResponseUtils.noMatchedKeysResponse(paramInfo.getStartPrefix());
+ }
+
+ // Set the aggregated totals in the response
+ listKeysResponse.setReplicatedDataSize(replicatedTotal);
+ listKeysResponse.setUnReplicatedDataSize(unreplicatedTotal);
+
+ return Response.ok(listKeysResponse).build();
+ } catch (IOException e) {
+ return ReconResponseUtils.createInternalServerErrorResponse(
+ "Error listing keys from OM DB: " + e.getMessage());
+ } catch (RuntimeException e) {
+ return ReconResponseUtils.createInternalServerErrorResponse(
+ "Unexpected runtime error while searching keys in OM DB: " +
e.getMessage());
+ } catch (Exception e) {
+ return ReconResponseUtils.createInternalServerErrorResponse(
+ "Error listing keys from OM DB: " + e.getMessage());
+ }
+ }
+
+ public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
+ throws IOException {
+ int originalLimit = paramInfo.getLimit();
+ Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
+ // Convert the search prefix to an object path for FSO buckets
+ String startPrefixObjectPath =
convertStartPrefixPathToObjectIdPath(paramInfo.getStartPrefix());
+ String[] names = parseRequestPath(startPrefixObjectPath);
+ Table<String, OmKeyInfo> fileTable =
+ omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+
+ // If names.length > 2, then the search prefix is at the volume or bucket
level hence
+ // no need to find parent or extract id's or find subpaths as the
fileTable is
+ // suitable for volume and bucket level search
+ if (names.length > 2) {
+ // Fetch the parent ID to search for
+ long parentId = Long.parseLong(names[names.length - 1]);
+
+ // Fetch the nameSpaceSummary for the parent ID
+ NSSummary parentSummary =
+ reconNamespaceSummaryManager.getNSSummary(parentId);
+ if (parentSummary == null) {
+ return matchedKeys;
+ }
+ List<String> subPaths = new ArrayList<>();
+ // Add the initial search prefix object path because it can have both
files and subdirectories with files.
+ subPaths.add(startPrefixObjectPath);
+
+ // Recursively gather all subpaths
+ gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]),
Long.parseLong(names[1]));
+ // Iterate over the subpaths and retrieve the files
+ for (String subPath : subPaths) {
+ paramInfo.setStartPrefix(subPath);
+ matchedKeys.putAll(
+ retrieveKeysFromTable(fileTable, paramInfo));
+ paramInfo.setLimit(originalLimit - matchedKeys.size());
+ if (matchedKeys.size() >= originalLimit) {
+ break;
+ }
+ }
+ return matchedKeys;
+ }
+
+ paramInfo.setStartPrefix(startPrefixObjectPath);
Review Comment:
Yes, it will hit , if startPrefix path is till bucket. Pls check and run
`testListKeysFSOBucket`
##########
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java:
##########
@@ -674,6 +685,624 @@ public Response getDeletedDirectorySummary() {
return Response.ok(dirSummary).build();
}
+ /**
+ * This API will list out limited 'count' number of keys after applying
below filters in API parameters:
+ * Default Values of API param filters:
+ * -- replicationType - empty string and filter will not be applied, so
list out all keys irrespective of
+ * replication type.
+ * -- creationTime - empty string and filter will not be applied, so list
out keys irrespective of age.
+ * -- keySize - 0 bytes, which means all keys greater than zero bytes
will be listed, effectively all.
+ * -- startPrefix - /, API assumes that startPrefix path always starts
with /. E.g. /volume/bucket
+ * -- prevKey - ""
+ * -- limit - 1000
+ *
+ * @param replicationType Filter for RATIS or EC replication keys
+ * @param creationDate Filter for keys created after creationDate in
"MM-dd-yyyy HH:mm:ss" string format.
+ * @param keySize Filter for Keys greater than keySize in bytes.
+ * @param startPrefix Filter for startPrefix path.
+ * @param prevKey rocksDB last key of page requested.
+ * @param limit Filter for limited count of keys.
+ *
+ * @return the list of keys in JSON structured format as per respective
bucket layout.
+ *
+ * Now lets consider, we have following OBS, LEGACY and FSO bucket key/files
namespace tree structure
+ *
+ * For OBS Bucket
+ *
+ * /volume1/obs-bucket/key1
+ * /volume1/obs-bucket/key1/key2
+ * /volume1/obs-bucket/key1/key2/key3
+ * /volume1/obs-bucket/key4
+ * /volume1/obs-bucket/key5
+ * /volume1/obs-bucket/key6
+ * For LEGACY Bucket
+ *
+ * /volume1/legacy-bucket/key
+ * /volume1/legacy-bucket/key1/key2
+ * /volume1/legacy-bucket/key1/key2/key3
+ * /volume1/legacy-bucket/key4
+ * /volume1/legacy-bucket/key5
+ * /volume1/legacy-bucket/key6
+ * For FSO Bucket
+ *
+ * /volume1/fso-bucket/dir1/dir2/dir3
+ * /volume1/fso-bucket/dir1/testfile
+ * /volume1/fso-bucket/dir1/file1
+ * /volume1/fso-bucket/dir1/dir2/testfile
+ * /volume1/fso-bucket/dir1/dir2/file1
+ * /volume1/fso-bucket/dir1/dir2/dir3/testfile
+ * /volume1/fso-bucket/dir1/dir2/dir3/file1
+ * Input Request for OBS bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/obs-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/obs-bucket",
+ * "replicatedDataSize": 62914560,
+ * "unReplicatedDataSize": 62914560,
+ * "lastKey": "/volume1/obs-bucket/key6",
+ * "keys": [
+ * {
+ * "key": "/volume1/obs-bucket/key1",
+ * "path": "volume1/obs-bucket/key1",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781418742,
+ * "modificationTime": 1715781419762,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key1/key2",
+ * "path": "volume1/obs-bucket/key1/key2",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781421716,
+ * "modificationTime": 1715781422723,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key1/key2/key3",
+ * "path": "volume1/obs-bucket/key1/key2/key3",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781424718,
+ * "modificationTime": 1715781425598,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key4",
+ * "path": "volume1/obs-bucket/key4",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781427561,
+ * "modificationTime": 1715781428407,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key5",
+ * "path": "volume1/obs-bucket/key5",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781430347,
+ * "modificationTime": 1715781431185,
+ * "isKey": true
+ * },
+ * {
+ * "key": "/volume1/obs-bucket/key6",
+ * "path": "volume1/obs-bucket/key6",
+ * "size": 10485760,
+ * "replicatedSize": 10485760,
+ * "replicationInfo": {
+ * "replicationFactor": "ONE",
+ * "requiredNodes": 1,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781433154,
+ * "modificationTime": 1715781433962,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ * Input Request for FSO bucket:
+ *
+ *
`api/v1/keys/listKeys?startPrefix=/volume1/fso-bucket&limit=2&replicationType=RATIS`
+ * Output Response:
+ *
+ * {
+ * "status": "OK",
+ * "path": "/volume1/fso-bucket",
+ * "replicatedDataSize": 188743680,
+ * "unReplicatedDataSize": 62914560,
+ * "lastKey":
"/-9223372036854775552/-9223372036854774016/-9223372036854773503/testfile",
+ * "keys": [
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773501/file1",
+ * "path": "volume1/fso-bucket/dir1/dir2/dir3/file1",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781411785,
+ * "modificationTime": 1715781415119,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773501/testfile",
+ * "path": "volume1/fso-bucket/dir1/dir2/dir3/testfile",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781409146,
+ * "modificationTime": 1715781409882,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773502/file1",
+ * "path": "volume1/fso-bucket/dir1/dir2/file1",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781406333,
+ * "modificationTime": 1715781407140,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773502/testfile",
+ * "path": "volume1/fso-bucket/dir1/dir2/testfile",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781403655,
+ * "modificationTime": 1715781404460,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773503/file1",
+ * "path": "volume1/fso-bucket/dir1/file1",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781400980,
+ * "modificationTime": 1715781401768,
+ * "isKey": true
+ * },
+ * {
+ * "key":
"/-9223372036854775552/-9223372036854774016/-9223372036854773503/testfile",
+ * "path": "volume1/fso-bucket/dir1/testfile",
+ * "size": 10485760,
+ * "replicatedSize": 31457280,
+ * "replicationInfo": {
+ * "replicationFactor": "THREE",
+ * "requiredNodes": 3,
+ * "replicationType": "RATIS"
+ * },
+ * "creationTime": 1715781397636,
+ * "modificationTime": 1715781398919,
+ * "isKey": true
+ * }
+ * ]
+ * }
+ *
+ * ********************************************************
+ * @throws IOException
+ */
+ @GET
+ @Path("/listKeys")
+ @SuppressWarnings("methodlength")
+ public Response listKeys(@QueryParam("replicationType") String
replicationType,
+ @QueryParam("creationDate") String creationDate,
+ @DefaultValue(DEFAULT_KEY_SIZE)
@QueryParam("keySize") long keySize,
+ @DefaultValue(OM_KEY_PREFIX)
@QueryParam("startPrefix") String startPrefix,
+ @DefaultValue(StringUtils.EMPTY)
@QueryParam(RECON_QUERY_PREVKEY) String prevKey,
+ @DefaultValue(DEFAULT_FETCH_COUNT)
@QueryParam("limit") int limit) {
+
+
+ // This API supports startPrefix from bucket level.
+ if (startPrefix == null || startPrefix.length() == 0) {
+ return Response.status(Response.Status.BAD_REQUEST).build();
+ }
+ String[] names = startPrefix.split(OM_KEY_PREFIX);
+ if (names.length < 3) {
+ return Response.status(Response.Status.BAD_REQUEST).build();
+ }
+ ListKeysResponse listKeysResponse = new ListKeysResponse();
+ if (!ReconUtils.isInitializationComplete(omMetadataManager)) {
+ listKeysResponse.setStatus(ResponseStatus.INITIALIZING);
+ return Response.ok(listKeysResponse).build();
+ }
+ ParamInfo paramInfo = new ParamInfo(replicationType, creationDate,
keySize, startPrefix, prevKey,
+ limit, false, "");
+ Response response = getListKeysResponse(paramInfo);
+ if (response.getEntity() instanceof ListKeysResponse) {
+ listKeysResponse = (ListKeysResponse) response.getEntity();
+ }
+
+ List<KeyEntityInfo> keyInfoList = listKeysResponse.getKeys();
+ if (!keyInfoList.isEmpty()) {
+ listKeysResponse.setLastKey(keyInfoList.get(keyInfoList.size() -
1).getKey());
+ }
+ return Response.ok(listKeysResponse).build();
+ }
+
+ private Response getListKeysResponse(ParamInfo paramInfo) {
+ try {
+ paramInfo.setLimit(Math.max(0, paramInfo.getLimit())); // Ensure limit
is non-negative
+ ListKeysResponse listKeysResponse = new ListKeysResponse();
+ listKeysResponse.setPath(paramInfo.getStartPrefix());
+ long replicatedTotal = 0;
+ long unreplicatedTotal = 0;
+ boolean keysFound = false; // Flag to track if any keys are found
+
+ // Search keys from non-FSO layout.
+ Map<String, OmKeyInfo> obsKeys;
+ Table<String, OmKeyInfo> keyTable =
+ omMetadataManager.getKeyTable(BucketLayout.LEGACY);
+ obsKeys = retrieveKeysFromTable(keyTable, paramInfo);
+ for (Map.Entry<String, OmKeyInfo> entry : obsKeys.entrySet()) {
+ keysFound = true;
+ KeyEntityInfo keyEntityInfo =
+ createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
+
+ listKeysResponse.getKeys().add(keyEntityInfo);
+ replicatedTotal += entry.getValue().getReplicatedSize();
+ unreplicatedTotal += entry.getValue().getDataSize();
+ }
+
+ // Search keys from FSO layout.
+ Map<String, OmKeyInfo> fsoKeys = searchKeysInFSO(paramInfo);
+ for (Map.Entry<String, OmKeyInfo> entry : fsoKeys.entrySet()) {
+ keysFound = true;
+ KeyEntityInfo keyEntityInfo =
+ createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
+
+ listKeysResponse.getKeys().add(keyEntityInfo);
+ replicatedTotal += entry.getValue().getReplicatedSize();
+ unreplicatedTotal += entry.getValue().getDataSize();
+ }
+
+ // If no keys were found, return a response indicating that no keys
matched
+ if (!keysFound) {
+ return
ReconResponseUtils.noMatchedKeysResponse(paramInfo.getStartPrefix());
+ }
+
+ // Set the aggregated totals in the response
+ listKeysResponse.setReplicatedDataSize(replicatedTotal);
+ listKeysResponse.setUnReplicatedDataSize(unreplicatedTotal);
+
+ return Response.ok(listKeysResponse).build();
+ } catch (IOException e) {
+ return ReconResponseUtils.createInternalServerErrorResponse(
+ "Error listing keys from OM DB: " + e.getMessage());
+ } catch (RuntimeException e) {
+ return ReconResponseUtils.createInternalServerErrorResponse(
+ "Unexpected runtime error while searching keys in OM DB: " +
e.getMessage());
+ } catch (Exception e) {
+ return ReconResponseUtils.createInternalServerErrorResponse(
+ "Error listing keys from OM DB: " + e.getMessage());
+ }
+ }
+
+ public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
+ throws IOException {
+ int originalLimit = paramInfo.getLimit();
+ Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
+ // Convert the search prefix to an object path for FSO buckets
+ String startPrefixObjectPath =
convertStartPrefixPathToObjectIdPath(paramInfo.getStartPrefix());
+ String[] names = parseRequestPath(startPrefixObjectPath);
+ Table<String, OmKeyInfo> fileTable =
+ omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+
+ // If names.length > 2, then the search prefix is at the volume or bucket
level hence
Review Comment:
ok done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
