This is an automated email from the ASF dual-hosted git repository.
devesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f98eac2099 HDDS-11561. Refactor Open Key Search Endpoint and
Consolidate with OmDBInsightEndpoint Using StartPrefix Parameter. (#7336)
f98eac2099 is described below
commit f98eac2099ff979364a501bdb219bc5f9be4a595
Author: Arafat2198 <[email protected]>
AuthorDate: Tue Nov 26 20:36:39 2024 +0530
HDDS-11561. Refactor Open Key Search Endpoint and Consolidate with
OmDBInsightEndpoint Using StartPrefix Parameter. (#7336)
---
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 99 ++++++
.../ozone/recon/api/OMDBInsightEndpoint.java | 227 +++++++------
.../ozone/recon/api/OMDBInsightSearchEndpoint.java | 350 ---------------------
.../ozone/recon/api/TestOmDBInsightEndPoint.java | 57 ++--
...dpoint.java => TestOpenKeysSearchEndpoint.java} | 170 ++++++----
5 files changed, 367 insertions(+), 536 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
index 407694bf85..12139e1772 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmUtils;
import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
+import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher;
import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.hdds.utils.db.Table;
@@ -78,8 +79,12 @@ import static org.jooq.impl.DSL.using;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler;
import org.apache.hadoop.ozone.recon.api.ServiceNotReadyException;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
import org.apache.hadoop.ozone.recon.api.types.DUResponse;
@@ -392,6 +397,100 @@ public class ReconUtils {
return fullPath;
}
+ /**
+ * Converts a key prefix into an object path for FSO buckets, using IDs.
+ *
+ * This method transforms a user-provided path (e.g., "volume/bucket/dir1")
into
+ * a database-friendly format ("/volumeID/bucketID/ParentId/") by replacing
names
+ * with their corresponding IDs. It simplifies database queries for FSO
bucket operations.
+ * <pre>
+ * {@code
+ * Examples:
+ * - Input: "volume/bucket/key" -> Output:
"/volumeID/bucketID/parentDirID/key"
+ * - Input: "volume/bucket/dir1" -> Output: "/volumeID/bucketID/dir1ID/"
+ * - Input: "volume/bucket/dir1/key1" -> Output:
"/volumeID/bucketID/dir1ID/key1"
+ * - Input: "volume/bucket/dir1/dir2" -> Output: "/volumeID/bucketID/dir2ID/"
+ * }
+ * </pre>
+ * @param prevKeyPrefix The path to be converted.
+ * @return The object path as "/volumeID/bucketID/ParentId/" or an empty
string if an error occurs.
+ * @throws IOException If database access fails.
+ * @throws IllegalArgumentException If the provided path is invalid or
cannot be converted.
+ */
+ public static String convertToObjectPathForOpenKeySearch(String
prevKeyPrefix,
+
ReconOMMetadataManager omMetadataManager,
+
ReconNamespaceSummaryManager reconNamespaceSummaryManager,
+
OzoneStorageContainerManager reconSCM)
+ throws IOException {
+ try {
+ String[] names =
EntityHandler.parseRequestPath(EntityHandler.normalizePath(
+ prevKeyPrefix, BucketLayout.FILE_SYSTEM_OPTIMIZED));
+ Table<String, OmKeyInfo> openFileTable =
omMetadataManager.getOpenKeyTable(
+ BucketLayout.FILE_SYSTEM_OPTIMIZED);
+
+ // Root-Level: Return the original path
+ if (names.length == 0 || names[0].isEmpty()) {
+ return prevKeyPrefix;
+ }
+
+ // Volume-Level: Fetch the volumeID
+ String volumeName = names[0];
+ validateNames(volumeName);
+ String volumeKey = omMetadataManager.getVolumeKey(volumeName);
+ long volumeId =
omMetadataManager.getVolumeTable().getSkipCache(volumeKey).getObjectID();
+ if (names.length == 1) {
+ return constructObjectPathWithPrefix(volumeId);
+ }
+
+ // Bucket-Level: Fetch the bucketID
+ String bucketName = names[1];
+ validateNames(bucketName);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName,
bucketName);
+ OmBucketInfo bucketInfo =
omMetadataManager.getBucketTable().getSkipCache(bucketKey);
+ long bucketId = bucketInfo.getObjectID();
+ if (names.length == 2 || bucketInfo.getBucketLayout() !=
BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ return constructObjectPathWithPrefix(volumeId, bucketId);
+ }
+
+ // Directory or Key-Level: Check both key and directory
+ BucketHandler handler =
+ BucketHandler.getBucketHandler(reconNamespaceSummaryManager,
omMetadataManager, reconSCM, bucketInfo);
+
+ if (names.length >= 3) {
+ String lastEntiry = names[names.length - 1];
+
+ // Check if the directory exists
+ OmDirectoryInfo dirInfo = handler.getDirInfo(names);
+ if (dirInfo != null && dirInfo.getName().equals(lastEntiry)) {
+ return constructObjectPathWithPrefix(volumeId, bucketId,
dirInfo.getObjectID()) + OM_KEY_PREFIX;
+ }
+
+ // Check if the key exists
+ long dirID = handler.getDirObjectId(names, names.length);
+ String keyKey = constructObjectPathWithPrefix(volumeId, bucketId,
dirID) +
+ OM_KEY_PREFIX + lastEntiry;
+ OmKeyInfo keyInfo = openFileTable.getSkipCache(keyKey);
+ if (keyInfo != null && keyInfo.getFileName().equals(lastEntiry)) {
+ return constructObjectPathWithPrefix(volumeId, bucketId,
+ keyInfo.getParentObjectID()) + OM_KEY_PREFIX + lastEntiry;
+ }
+
+ return prevKeyPrefix;
+ }
+ } catch (IllegalArgumentException e) {
+ log.error(
+ "IllegalArgumentException encountered while converting key prefix to
object path: {}",
+ prevKeyPrefix, e);
+ throw e;
+ } catch (RuntimeException e) {
+ log.error(
+ "RuntimeException encountered while converting key prefix to object
path: {}",
+ prevKeyPrefix, e);
+ return prevKeyPrefix;
+ }
+ return prevKeyPrefix;
+ }
+
private static void triggerRebuild(ReconNamespaceSummaryManager
reconNamespaceSummaryManager,
ReconOMMetadataManager omMetadataManager)
{
ExecutorService executor = Executors.newSingleThreadExecutor(r -> {
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
index d7cb691253..64da15db41 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
@@ -56,9 +56,10 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.IOException;
+import java.util.Collections;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -80,8 +81,6 @@ import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_QUERY_PREVKEY;
import static
org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse;
import static
org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse;
import static
org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse;
-import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable;
-import static org.apache.hadoop.ozone.recon.ReconUtils.validateStartPrefix;
import static
org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler;
import static
org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath;
import static
org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath;
@@ -179,102 +178,133 @@ public class OMDBInsightEndpoint {
@Path("/open")
public Response getOpenKeyInfo(
@DefaultValue(DEFAULT_FETCH_COUNT) @QueryParam(RECON_QUERY_LIMIT)
- int limit,
+ int limit,
@DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_PREVKEY)
- String prevKey,
- @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_FSO)
- @QueryParam(RECON_OPEN_KEY_INCLUDE_FSO)
- boolean includeFso,
- @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_NON_FSO)
- @QueryParam(RECON_OPEN_KEY_INCLUDE_NON_FSO)
- boolean includeNonFso) {
+ String prevKey,
+ @DefaultValue(StringUtils.EMPTY) @QueryParam(RECON_QUERY_START_PREFIX)
+ String startPrefix,
+ @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_FSO)
@QueryParam(RECON_OPEN_KEY_INCLUDE_FSO)
+ boolean includeFso,
+ @DefaultValue(DEFAULT_OPEN_KEY_INCLUDE_NON_FSO)
@QueryParam(RECON_OPEN_KEY_INCLUDE_NON_FSO)
+ boolean includeNonFso) {
+
KeyInsightInfoResponse openKeyInsightInfo = new KeyInsightInfoResponse();
- List<KeyEntityInfo> nonFSOKeyInfoList =
- openKeyInsightInfo.getNonFSOKeyInfoList();
-
- boolean skipPrevKeyDone = false;
- boolean isLegacyBucketLayout = true;
- boolean recordsFetchedLimitReached = false;
-
- String lastKey = "";
- List<KeyEntityInfo> fsoKeyInfoList =
openKeyInsightInfo.getFsoKeyInfoList();
- for (BucketLayout layout : Arrays.asList(
- BucketLayout.LEGACY, BucketLayout.FILE_SYSTEM_OPTIMIZED)) {
- isLegacyBucketLayout = (layout == BucketLayout.LEGACY);
- // Skip bucket iteration based on parameters includeFso and includeNonFso
- if ((!includeFso && !isLegacyBucketLayout) ||
- (!includeNonFso && isLegacyBucketLayout)) {
- continue;
+
+ try {
+ long replicatedTotal = 0;
+ long unreplicatedTotal = 0;
+ boolean skipPrevKeyDone = false; // Tracks if prevKey was used earlier
+ boolean keysFound = false; // Flag to track if any keys are found
+ String lastKey = null;
+ Map<String, OmKeyInfo> obsKeys = Collections.emptyMap();
+ Map<String, OmKeyInfo> fsoKeys = Collections.emptyMap();
+
+ // Validate startPrefix if it's provided
+ if (isNotBlank(startPrefix) && !validateStartPrefix(startPrefix)) {
+ return createBadRequestResponse("Invalid startPrefix: Path must be at
the bucket level or deeper.");
}
- Table<String, OmKeyInfo> openKeyTable =
- omMetadataManager.getOpenKeyTable(layout);
- try (
- TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- keyIter = openKeyTable.iterator()) {
- boolean skipPrevKey = false;
- String seekKey = prevKey;
- if (!skipPrevKeyDone && isNotBlank(prevKey)) {
- skipPrevKey = true;
- Table.KeyValue<String, OmKeyInfo> seekKeyValue =
- keyIter.seek(seekKey);
- // check if RocksDB was able to seek correctly to the given key
prefix
- // if not, then return empty result
- // In case of an empty prevKeyPrefix, all the keys are returned
- if (seekKeyValue == null ||
- (isNotBlank(prevKey) &&
- !seekKeyValue.getKey().equals(prevKey))) {
- continue;
- }
+ // Use searchOpenKeys logic with adjustments for FSO and Non-FSO
filtering
+ if (includeNonFso) {
+ // Search for non-FSO keys in KeyTable
+ Table<String, OmKeyInfo> openKeyTable =
omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY);
+ obsKeys = ReconUtils.extractKeysFromTable(openKeyTable, startPrefix,
limit, prevKey);
+ for (Map.Entry<String, OmKeyInfo> entry : obsKeys.entrySet()) {
+ keysFound = true;
+ skipPrevKeyDone = true; // Don't use the prevKey for the file table
+ KeyEntityInfo keyEntityInfo =
createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
+ openKeyInsightInfo.getNonFSOKeyInfoList().add(keyEntityInfo); // Add
to non-FSO list
+ replicatedTotal += entry.getValue().getReplicatedSize();
+ unreplicatedTotal += entry.getValue().getDataSize();
+ lastKey = entry.getKey(); // Update lastKey
}
- while (keyIter.hasNext()) {
- Table.KeyValue<String, OmKeyInfo> kv = keyIter.next();
- String key = kv.getKey();
- lastKey = key;
- OmKeyInfo omKeyInfo = kv.getValue();
- // skip the prev key if prev key is present
- if (skipPrevKey && key.equals(prevKey)) {
- skipPrevKeyDone = true;
- continue;
- }
- KeyEntityInfo keyEntityInfo = new KeyEntityInfo();
- keyEntityInfo.setIsKey(omKeyInfo.isFile());
- keyEntityInfo.setKey(key);
- keyEntityInfo.setPath(omKeyInfo.getKeyName());
- keyEntityInfo.setInStateSince(omKeyInfo.getCreationTime());
- keyEntityInfo.setSize(omKeyInfo.getDataSize());
- keyEntityInfo.setReplicatedSize(omKeyInfo.getReplicatedSize());
- keyEntityInfo.setReplicationConfig(omKeyInfo.getReplicationConfig());
- openKeyInsightInfo.setUnreplicatedDataSize(
- openKeyInsightInfo.getUnreplicatedDataSize() +
- keyEntityInfo.getSize());
- openKeyInsightInfo.setReplicatedDataSize(
- openKeyInsightInfo.getReplicatedDataSize() +
- keyEntityInfo.getReplicatedSize());
- boolean added =
- isLegacyBucketLayout ? nonFSOKeyInfoList.add(keyEntityInfo) :
- fsoKeyInfoList.add(keyEntityInfo);
- if ((nonFSOKeyInfoList.size() + fsoKeyInfoList.size()) == limit) {
- recordsFetchedLimitReached = true;
- break;
- }
+ }
+
+ if (includeFso) {
+ // Search for FSO keys in FileTable
+ // If prevKey was used for non-FSO keys, skip it for FSO keys.
+ String effectivePrevKey = skipPrevKeyDone ? "" : prevKey;
+ // If limit = -1 then we need to fetch all keys without limit
+ int effectiveLimit = limit == -1 ? limit : limit - obsKeys.size();
+ fsoKeys = searchOpenKeysInFSO(startPrefix, effectiveLimit,
effectivePrevKey);
+ for (Map.Entry<String, OmKeyInfo> entry : fsoKeys.entrySet()) {
+ keysFound = true;
+ KeyEntityInfo keyEntityInfo =
createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
+ openKeyInsightInfo.getFsoKeyInfoList().add(keyEntityInfo); // Add to
FSO list
+ replicatedTotal += entry.getValue().getReplicatedSize();
+ unreplicatedTotal += entry.getValue().getDataSize();
+ lastKey = entry.getKey(); // Update lastKey
}
- } catch (IOException ex) {
- throw new WebApplicationException(ex,
- Response.Status.INTERNAL_SERVER_ERROR);
- } catch (IllegalArgumentException e) {
- throw new WebApplicationException(e, Response.Status.BAD_REQUEST);
- } catch (Exception ex) {
- throw new WebApplicationException(ex,
- Response.Status.INTERNAL_SERVER_ERROR);
}
- if (recordsFetchedLimitReached) {
- break;
+
+ // If no keys were found, return a response indicating that no keys
matched
+ if (!keysFound) {
+ return noMatchedKeysResponse(startPrefix);
}
+
+ // Set the aggregated totals in the response
+ openKeyInsightInfo.setReplicatedDataSize(replicatedTotal);
+ openKeyInsightInfo.setUnreplicatedDataSize(unreplicatedTotal);
+ openKeyInsightInfo.setLastKey(lastKey);
+
+ // Return the response with the matched keys and their data sizes
+ return Response.ok(openKeyInsightInfo).build();
+ } catch (IOException e) {
+ // Handle IO exceptions and return an internal server error response
+ return createInternalServerErrorResponse("Error searching open keys in
OM DB: " + e.getMessage());
+ } catch (IllegalArgumentException e) {
+ // Handle illegal argument exceptions and return a bad request response
+ return createBadRequestResponse("Invalid argument: " + e.getMessage());
}
+ }
- openKeyInsightInfo.setLastKey(lastKey);
- return Response.ok(openKeyInsightInfo).build();
+ public Map<String, OmKeyInfo> searchOpenKeysInFSO(String startPrefix,
+ int limit, String prevKey)
+ throws IOException, IllegalArgumentException {
+ Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
+ // Convert the search prefix to an object path for FSO buckets
+ String startPrefixObjectPath =
ReconUtils.convertToObjectPathForOpenKeySearch(
+ startPrefix, omMetadataManager, reconNamespaceSummaryManager,
reconSCM);
+ String[] names = parseRequestPath(startPrefixObjectPath);
+ Table<String, OmKeyInfo> openFileTable =
+ omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+
+ // If names.length <= 2, then the search prefix is at the volume or bucket
level hence
+ // no need to find parent or extract id's or find subpaths as the
openFileTable is
+ // suitable for volume and bucket level search
+ if (names.length > 2 && startPrefixObjectPath.endsWith(OM_KEY_PREFIX)) {
+ // Fetch the parent ID to search for
+ long parentId = Long.parseLong(names[names.length - 1]);
+
+ // Fetch the nameSpaceSummary for the parent ID
+ NSSummary parentSummary =
reconNamespaceSummaryManager.getNSSummary(parentId);
+ if (parentSummary == null) {
+ return matchedKeys;
+ }
+ List<String> subPaths = new ArrayList<>();
+ // Add the initial search prefix object path because it can have both
openFiles
+ // and subdirectories with openFiles
+ subPaths.add(startPrefixObjectPath);
+
+ // Recursively gather all subpaths
+ ReconUtils.gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]),
Long.parseLong(names[1]),
+ reconNamespaceSummaryManager);
+
+ // Iterate over the subpaths and retrieve the open files
+ for (String subPath : subPaths) {
+ matchedKeys.putAll(
+ ReconUtils.extractKeysFromTable(openFileTable, subPath, limit -
matchedKeys.size(), prevKey));
+ if (matchedKeys.size() >= limit) {
+ break;
+ }
+ }
+ return matchedKeys;
+ }
+
+ // If the search level is at the volume, bucket or key level, directly
search the openFileTable
+ matchedKeys.putAll(
+ ReconUtils.extractKeysFromTable(openFileTable, startPrefixObjectPath,
limit, prevKey));
+ return matchedKeys;
}
/**
@@ -481,7 +511,7 @@ public class OMDBInsightEndpoint {
// Search for deleted keys in DeletedTable
Table<String, RepeatedOmKeyInfo> deletedTable =
omMetadataManager.getDeletedTable();
Map<String, RepeatedOmKeyInfo> deletedKeys =
- extractKeysFromTable(deletedTable, startPrefix, limit, prevKey);
+ ReconUtils.extractKeysFromTable(deletedTable, startPrefix, limit,
prevKey);
// Iterate over the retrieved keys and populate the response
for (Map.Entry<String, RepeatedOmKeyInfo> entry : deletedKeys.entrySet()) {
@@ -1265,8 +1295,7 @@ public class OMDBInsightEndpoint {
KeyEntityInfo keyEntityInfo = new KeyEntityInfo();
keyEntityInfo.setKey(dbKey); // Set the DB key
keyEntityInfo.setIsKey(keyInfo.isFile());
- keyEntityInfo.setPath(ReconUtils.constructFullPath(keyInfo,
reconNamespaceSummaryManager,
- omMetadataManager));
+ keyEntityInfo.setPath(ReconUtils.constructFullPath(keyInfo,
reconNamespaceSummaryManager, omMetadataManager));
keyEntityInfo.setSize(keyInfo.getDataSize());
keyEntityInfo.setCreationTime(keyInfo.getCreationTime());
keyEntityInfo.setModificationTime(keyInfo.getModificationTime());
@@ -1284,6 +1313,20 @@ public class OMDBInsightEndpoint {
dirSummary.put("totalDeletedDirectories", deletedDirCount);
}
+ private boolean validateStartPrefix(String startPrefix) {
+
+ // Ensure startPrefix starts with '/' for non-empty values
+ startPrefix = startPrefix.startsWith("/") ? startPrefix : "/" +
startPrefix;
+
+ // Split the path to ensure it's at least at the bucket level
(volume/bucket).
+ String[] pathComponents = startPrefix.split("/");
+ if (pathComponents.length < 3 || pathComponents[2].isEmpty()) {
+ return false; // Invalid if not at bucket level or deeper
+ }
+
+ return true;
+ }
+
private String createPath(OmKeyInfo omKeyInfo) {
return omKeyInfo.getVolumeName() + OM_KEY_PREFIX +
omKeyInfo.getBucketName() + OM_KEY_PREFIX + omKeyInfo.getKeyName();
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java
deleted file mode 100644
index fcd73fbe72..0000000000
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightSearchEndpoint.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.recon.api;
-
-import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
-import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler;
-import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo;
-import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse;
-import org.apache.hadoop.ozone.recon.api.types.NSSummary;
-import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
-import org.apache.hadoop.ozone.recon.spi.impl.ReconNamespaceSummaryManagerImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.util.Map;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.ArrayList;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
-import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_START_PREFIX;
-import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT;
-import static
org.apache.hadoop.ozone.recon.ReconConstants.RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY;
-import static
org.apache.hadoop.ozone.recon.ReconResponseUtils.noMatchedKeysResponse;
-import static
org.apache.hadoop.ozone.recon.ReconResponseUtils.createBadRequestResponse;
-import static
org.apache.hadoop.ozone.recon.ReconResponseUtils.createInternalServerErrorResponse;
-import static org.apache.hadoop.ozone.recon.ReconUtils.validateStartPrefix;
-import static
org.apache.hadoop.ozone.recon.ReconUtils.constructObjectPathWithPrefix;
-import static org.apache.hadoop.ozone.recon.ReconUtils.extractKeysFromTable;
-import static org.apache.hadoop.ozone.recon.ReconUtils.gatherSubPaths;
-import static org.apache.hadoop.ozone.recon.ReconUtils.validateNames;
-import static
org.apache.hadoop.ozone.recon.api.handlers.BucketHandler.getBucketHandler;
-import static
org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.normalizePath;
-import static
org.apache.hadoop.ozone.recon.api.handlers.EntityHandler.parseRequestPath;
-
-/**
- * REST endpoint for search implementation in OM DB Insight.
- *
- * This class provides endpoints for searching keys in the Ozone Manager
database.
- * It supports searching for both open and deleted keys across File System
Optimized (FSO)
- * and Object Store (non-FSO) bucket layouts. The results include matching
keys and their
- * data sizes.
- */
-@Path("/keys")
-@Produces(MediaType.APPLICATION_JSON)
-@AdminOnly
-public class OMDBInsightSearchEndpoint {
-
- private OzoneStorageContainerManager reconSCM;
- private final ReconOMMetadataManager omMetadataManager;
- private static final Logger LOG =
- LoggerFactory.getLogger(OMDBInsightSearchEndpoint.class);
- private ReconNamespaceSummaryManagerImpl reconNamespaceSummaryManager;
-
-
- @Inject
- public OMDBInsightSearchEndpoint(OzoneStorageContainerManager reconSCM,
- ReconOMMetadataManager omMetadataManager,
- ReconNamespaceSummaryManagerImpl
reconNamespaceSummaryManager) {
- this.reconSCM = reconSCM;
- this.omMetadataManager = omMetadataManager;
- this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
- }
-
-
- /**
- * Performs a search for open keys in the Ozone Manager OpenKey and OpenFile
table using a specified search prefix.
- * This endpoint searches across both File System Optimized (FSO) and Object
Store (non-FSO) layouts,
- * compiling a list of keys that match the given prefix along with their
data sizes.
- *
- * The search prefix must start from the bucket level
('/volumeName/bucketName/') or any specific directory
- * or key level (e.g., '/volA/bucketA/dir1' for everything under 'dir1'
inside 'bucketA' of 'volA').
- * The search operation matches the prefix against the start of keys' names
within the OM DB.
- *
- * Example Usage:
- * 1. A startPrefix of "/volA/bucketA/" retrieves every key under bucket
'bucketA' in volume 'volA'.
- * 2. Specifying "/volA/bucketA/dir1" focuses the search within 'dir1'
inside 'bucketA' of 'volA'.
- *
- * @param startPrefix The prefix for searching keys, starting from the
bucket level or any specific path.
- * @param limit Limits the number of returned keys.
- * @param prevKey The key to start after for the next set of records.
- * @return A KeyInsightInfoResponse, containing matching keys and their data
sizes.
- * @throws IOException On failure to access the OM database or process the
operation.
- * @throws IllegalArgumentException If the provided startPrefix or other
arguments are invalid.
- */
- @GET
- @Path("/open/search")
- public Response searchOpenKeys(
- @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_START_PREFIX)
@QueryParam("startPrefix")
- String startPrefix,
- @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_LIMIT)
@QueryParam("limit")
- int limit,
- @DefaultValue(RECON_OM_INSIGHTS_DEFAULT_SEARCH_PREV_KEY)
@QueryParam("prevKey")
- String prevKey) throws IOException {
-
- try {
- // Validate the request parameters
- if (!validateStartPrefix(startPrefix)) {
- return createBadRequestResponse("Invalid startPrefix: Path must be at
the bucket level or deeper.");
- }
-
- // Ensure the limit is non-negative
- limit = Math.max(0, limit);
-
- // Initialize response object
- KeyInsightInfoResponse insightResponse = new KeyInsightInfoResponse();
- long replicatedTotal = 0;
- long unreplicatedTotal = 0;
- boolean keysFound = false; // Flag to track if any keys are found
- String lastKey = null;
-
- // Search for non-fso keys in KeyTable
- Table<String, OmKeyInfo> openKeyTable =
- omMetadataManager.getOpenKeyTable(BucketLayout.LEGACY);
- Map<String, OmKeyInfo> obsKeys =
- extractKeysFromTable(openKeyTable, startPrefix, limit, prevKey);
- for (Map.Entry<String, OmKeyInfo> entry : obsKeys.entrySet()) {
- keysFound = true;
- KeyEntityInfo keyEntityInfo =
- createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
- insightResponse.getNonFSOKeyInfoList()
- .add(keyEntityInfo); // Add to non-FSO list
- replicatedTotal += entry.getValue().getReplicatedSize();
- unreplicatedTotal += entry.getValue().getDataSize();
- lastKey = entry.getKey(); // Update lastKey
- }
-
- // Search for fso keys in FileTable
- Map<String, OmKeyInfo> fsoKeys = searchOpenKeysInFSO(startPrefix, limit,
prevKey);
- for (Map.Entry<String, OmKeyInfo> entry : fsoKeys.entrySet()) {
- keysFound = true;
- KeyEntityInfo keyEntityInfo =
- createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());
- insightResponse.getFsoKeyInfoList()
- .add(keyEntityInfo); // Add to FSO list
- replicatedTotal += entry.getValue().getReplicatedSize();
- unreplicatedTotal += entry.getValue().getDataSize();
- lastKey = entry.getKey(); // Update lastKey
- }
-
- // If no keys were found, return a response indicating that no keys
matched
- if (!keysFound) {
- return noMatchedKeysResponse(startPrefix);
- }
-
- // Set the aggregated totals in the response
- insightResponse.setReplicatedDataSize(replicatedTotal);
- insightResponse.setUnreplicatedDataSize(unreplicatedTotal);
- insightResponse.setLastKey(lastKey);
-
- // Return the response with the matched keys and their data sizes
- return Response.ok(insightResponse).build();
- } catch (IOException e) {
- // Handle IO exceptions and return an internal server error response
- return createInternalServerErrorResponse(
- "Error searching open keys in OM DB: " + e.getMessage());
- } catch (IllegalArgumentException e) {
- // Handle illegal argument exceptions and return a bad request response
- return createBadRequestResponse(
- "Invalid startPrefix: " + e.getMessage());
- }
- }
-
- public Map<String, OmKeyInfo> searchOpenKeysInFSO(String startPrefix,
- int limit, String prevKey)
- throws IOException, IllegalArgumentException {
- Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
- // Convert the search prefix to an object path for FSO buckets
- String startPrefixObjectPath = convertToObjectPath(startPrefix);
- String[] names = parseRequestPath(startPrefixObjectPath);
- Table<String, OmKeyInfo> openFileTable =
- omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
-
- // If names.length <= 2, then the search prefix is at the volume or bucket
level hence
- // no need to find parent or extract id's or find subpaths as the
openFileTable is
- // suitable for volume and bucket level search
- if (names.length > 2 && startPrefixObjectPath.endsWith(OM_KEY_PREFIX)) {
- // Fetch the parent ID to search for
- long parentId = Long.parseLong(names[names.length - 1]);
-
- // Fetch the nameSpaceSummary for the parent ID
- NSSummary parentSummary =
reconNamespaceSummaryManager.getNSSummary(parentId);
- if (parentSummary == null) {
- return matchedKeys;
- }
- List<String> subPaths = new ArrayList<>();
- // Add the initial search prefix object path because it can have both
openFiles
- // and subdirectories with openFiles
- subPaths.add(startPrefixObjectPath);
-
- // Recursively gather all subpaths
- gatherSubPaths(parentId, subPaths, Long.parseLong(names[0]),
Long.parseLong(names[1]),
- reconNamespaceSummaryManager);
-
- // Iterate over the subpaths and retrieve the open files
- for (String subPath : subPaths) {
- matchedKeys.putAll(
- extractKeysFromTable(openFileTable, subPath, limit -
matchedKeys.size(), prevKey));
- if (matchedKeys.size() >= limit) {
- break;
- }
- }
- return matchedKeys;
- }
-
- // If the search level is at the volume, bucket or key level, directly
search the openFileTable
- matchedKeys.putAll(
- extractKeysFromTable(openFileTable, startPrefixObjectPath, limit,
prevKey));
- return matchedKeys;
- }
-
- /**
- * Converts a key prefix into an object path for FSO buckets, using IDs.
- *
- * This method transforms a user-provided path (e.g., "volume/bucket/dir1")
into
- * a database-friendly format ("/volumeID/bucketID/ParentId/") by replacing
names
- * with their corresponding IDs. It simplifies database queries for FSO
bucket operations.
- * <pre>
- * {@code
- * Examples:
- * - Input: "volume/bucket/key" -> Output:
"/volumeID/bucketID/parentDirID/key"
- * - Input: "volume/bucket/dir1" -> Output: "/volumeID/bucketID/dir1ID/"
- * - Input: "volume/bucket/dir1/key1" -> Output:
"/volumeID/bucketID/dir1ID/key1"
- * - Input: "volume/bucket/dir1/dir2" -> Output: "/volumeID/bucketID/dir2ID/"
- * }
- * </pre>
- * @param prevKeyPrefix The path to be converted.
- * @return The object path as "/volumeID/bucketID/ParentId/" or an empty
string if an error occurs.
- * @throws IOException If database access fails.
- * @throws IllegalArgumentException If the provided path is invalid or
cannot be converted.
- */
- public String convertToObjectPath(String prevKeyPrefix) throws IOException {
- try {
- String[] names = parseRequestPath(normalizePath(prevKeyPrefix,
BucketLayout.FILE_SYSTEM_OPTIMIZED));
- Table<String, OmKeyInfo> openFileTable =
omMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
-
- // Root-Level: Return the original path
- if (names.length == 0) {
- return prevKeyPrefix;
- }
-
- // Volume-Level: Fetch the volumeID
- String volumeName = names[0];
- validateNames(volumeName);
- String volumeKey = omMetadataManager.getVolumeKey(volumeName);
- long volumeId =
omMetadataManager.getVolumeTable().getSkipCache(volumeKey).getObjectID();
- if (names.length == 1) {
- return constructObjectPathWithPrefix(volumeId);
- }
-
- // Bucket-Level: Fetch the bucketID
- String bucketName = names[1];
- validateNames(bucketName);
- String bucketKey = omMetadataManager.getBucketKey(volumeName,
bucketName);
- OmBucketInfo bucketInfo =
omMetadataManager.getBucketTable().getSkipCache(bucketKey);
- long bucketId = bucketInfo.getObjectID();
- if (names.length == 2 || bucketInfo.getBucketLayout() !=
BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- return constructObjectPathWithPrefix(volumeId, bucketId);
- }
-
- // Directory or Key-Level: Check both key and directory
- BucketHandler handler =
- getBucketHandler(reconNamespaceSummaryManager, omMetadataManager,
reconSCM, bucketInfo);
-
- if (names.length >= 3) {
- String lastEntiry = names[names.length - 1];
-
- // Check if the directory exists
- OmDirectoryInfo dirInfo = handler.getDirInfo(names);
- if (dirInfo != null && dirInfo.getName().equals(lastEntiry)) {
- return constructObjectPathWithPrefix(volumeId, bucketId,
dirInfo.getObjectID()) + OM_KEY_PREFIX;
- }
-
- // Check if the key exists
- long dirID = handler.getDirObjectId(names, names.length);
- String keyKey = constructObjectPathWithPrefix(volumeId, bucketId,
dirID) +
- OM_KEY_PREFIX + lastEntiry;
- OmKeyInfo keyInfo = openFileTable.getSkipCache(keyKey);
- if (keyInfo != null && keyInfo.getFileName().equals(lastEntiry)) {
- return constructObjectPathWithPrefix(volumeId, bucketId,
- keyInfo.getParentObjectID()) + OM_KEY_PREFIX + lastEntiry;
- }
-
- return prevKeyPrefix;
- }
- } catch (IllegalArgumentException e) {
- LOG.error(
- "IllegalArgumentException encountered while converting key prefix to
object path: {}",
- prevKeyPrefix, e);
- throw e;
- } catch (RuntimeException e) {
- LOG.error(
- "RuntimeException encountered while converting key prefix to object
path: {}",
- prevKeyPrefix, e);
- return prevKeyPrefix;
- }
- return prevKeyPrefix;
- }
-
- /**
- * Creates a KeyEntityInfo object from an OmKeyInfo object and the
corresponding key.
- *
- * @param dbKey The key in the database corresponding to the OmKeyInfo
object.
- * @param keyInfo The OmKeyInfo object to create the KeyEntityInfo from.
- * @return The KeyEntityInfo object created from the OmKeyInfo object and
the key.
- */
- private KeyEntityInfo createKeyEntityInfoFromOmKeyInfo(String dbKey,
- OmKeyInfo keyInfo) {
- KeyEntityInfo keyEntityInfo = new KeyEntityInfo();
- keyEntityInfo.setKey(dbKey); // Set the DB key
- keyEntityInfo.setIsKey(keyInfo.isFile());
- keyEntityInfo.setPath(keyInfo.getKeyName()); // Assuming path is the same
as key name
- keyEntityInfo.setInStateSince(keyInfo.getCreationTime());
- keyEntityInfo.setSize(keyInfo.getDataSize());
- keyEntityInfo.setReplicatedSize(keyInfo.getReplicatedSize());
- keyEntityInfo.setReplicationConfig(keyInfo.getReplicationConfig());
- return keyEntityInfo;
- }
-
-}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java
index ed4b82fa3e..61a9711876 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOmDBInsightEndPoint.java
@@ -90,6 +90,7 @@ import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyT
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -897,11 +898,11 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
.get("/sampleVol/bucketOne/key_one");
assertEquals("key_one", omKeyInfo1.getKeyName());
Response openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(-1, "", true, true);
+ omdbInsightEndpoint.getOpenKeyInfo(-1, "", "", true, true);
KeyInsightInfoResponse keyInsightInfoResp =
(KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
- assertEquals("key_one",
+ assertEquals("sampleVol/bucketOne/key_one",
keyInsightInfoResp.getNonFSOKeyInfoList().get(0).getPath());
}
@@ -1044,7 +1045,7 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
reconOMMetadataManager.getOpenKeyTable(getBucketLayout())
.put("/sampleVol/bucketOne/key_three", omKeyInfo3);
Response openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(2, "", true, true);
+ omdbInsightEndpoint.getOpenKeyInfo(2, "", "", true, true);
KeyInsightInfoResponse keyInsightInfoResp =
(KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
@@ -1053,10 +1054,10 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
assertEquals(0, keyInsightInfoResp.getFsoKeyInfoList().size());
assertEquals(2, keyInsightInfoResp.getFsoKeyInfoList().size() +
keyInsightInfoResp.getNonFSOKeyInfoList().size());
- assertEquals("key_three",
+ assertEquals("sampleVol/bucketOne/key_three",
keyInsightInfoResp.getNonFSOKeyInfoList().get(1).getPath());
- openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(3, "", true, true);
+ openKeyInfoResp = omdbInsightEndpoint.getOpenKeyInfo(3, "", "", true,
true);
keyInsightInfoResp =
(KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
@@ -1065,7 +1066,7 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
assertEquals(1, keyInsightInfoResp.getFsoKeyInfoList().size());
assertEquals(3, keyInsightInfoResp.getFsoKeyInfoList().size() +
keyInsightInfoResp.getNonFSOKeyInfoList().size());
- assertEquals("key_three",
+ assertEquals("sampleVol/bucketOne/key_three",
keyInsightInfoResp.getNonFSOKeyInfoList().get(1).getPath());
}
@@ -1107,7 +1108,7 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
// CASE 1 :- Display only FSO keys in response
// includeFsoKeys=true, includeNonFsoKeys=false
Response openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(10, "", true, false);
+ omdbInsightEndpoint.getOpenKeyInfo(10, "", "", true, false);
KeyInsightInfoResponse keyInsightInfoResp =
(KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
@@ -1119,7 +1120,7 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
// CASE 2 :- Display only Non-FSO keys in response
// includeFsoKeys=false, includeNonFsoKeys=true
openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(10, "", false, true);
+ omdbInsightEndpoint.getOpenKeyInfo(10, "", "", false, true);
keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
assertEquals(0,
@@ -1130,7 +1131,7 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
// CASE 3 :- Display both FSO and Non-FSO keys in response
// includeFsoKeys=true, includeNonFsoKeys=true
openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(10, "", true, true);
+ omdbInsightEndpoint.getOpenKeyInfo(10, "", "", true, true);
keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
assertEquals(4,
@@ -1141,45 +1142,39 @@ public class TestOmDBInsightEndPoint extends
AbstractReconSqlDBTest {
// CASE 4 :- Don't Display both FSO and Non-FSO keys in response
// includeFsoKeys=false, includeNonFsoKeys=false
openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(10, "", false, false);
- keyInsightInfoResp = (KeyInsightInfoResponse) openKeyInfoResp.getEntity();
- assertNotNull(keyInsightInfoResp);
- assertEquals(0,
- keyInsightInfoResp.getFsoKeyInfoList().size());
- assertEquals(0,
- keyInsightInfoResp.getNonFSOKeyInfoList().size());
+ omdbInsightEndpoint.getOpenKeyInfo(10, "", "", false, false);
+ assertEquals(204, openKeyInfoResp.getStatus());
+ String entity = (String) openKeyInfoResp.getEntity();
+ assertTrue(entity.contains("No keys matched the search prefix"),
+ "Expected a message indicating no keys were found");
}
@Test
public void testGetOpenKeyInfoPrevKeyParam() throws Exception {
OmKeyInfo omKeyInfo1 =
- getOmKeyInfo("sampleVol", "bucketOne", "key_one", true);
+ getOmKeyInfo("sampleVol", "bucketOne", "key_1", true);
OmKeyInfo omKeyInfo2 =
- getOmKeyInfo("sampleVol", "bucketOne", "key_two", true);
+ getOmKeyInfo("sampleVol", "bucketOne", "key_2", true);
OmKeyInfo omKeyInfo3 =
- getOmKeyInfo("sampleVol", "bucketOne", "key_three", true);
+ getOmKeyInfo("sampleVol", "bucketOne", "key_3", true);
reconOMMetadataManager.getOpenKeyTable(getBucketLayout())
- .put("/sampleVol/bucketOne/key_one", omKeyInfo1);
+ .put("/sampleVol/bucketOne/key_1", omKeyInfo1);
reconOMMetadataManager.getOpenKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED)
- .put("/sampleVol/bucketOne/key_two", omKeyInfo2);
+ .put("/sampleVol/bucketOne/key_2", omKeyInfo2);
reconOMMetadataManager.getOpenKeyTable(getBucketLayout())
- .put("/sampleVol/bucketOne/key_three", omKeyInfo3);
+ .put("/sampleVol/bucketOne/key_3", omKeyInfo3);
Response openKeyInfoResp =
- omdbInsightEndpoint.getOpenKeyInfo(-1, "/sampleVol/bucketOne/key_one",
+ omdbInsightEndpoint.getOpenKeyInfo(-1, "/sampleVol/bucketOne/key_1",
"",
true, true);
KeyInsightInfoResponse keyInsightInfoResp =
(KeyInsightInfoResponse) openKeyInfoResp.getEntity();
assertNotNull(keyInsightInfoResp);
- assertEquals(1,
- keyInsightInfoResp.getNonFSOKeyInfoList().size());
+ assertEquals(1, keyInsightInfoResp.getNonFSOKeyInfoList().size());
assertEquals(1, keyInsightInfoResp.getFsoKeyInfoList().size());
- assertEquals(2, keyInsightInfoResp.getFsoKeyInfoList().size() +
- keyInsightInfoResp.getNonFSOKeyInfoList().size());
- assertEquals("key_three",
- keyInsightInfoResp.getNonFSOKeyInfoList().get(0).getPath());
- assertEquals("key_two",
- keyInsightInfoResp.getFsoKeyInfoList().get(0).getPath());
+ assertEquals(2, keyInsightInfoResp.getFsoKeyInfoList().size() +
keyInsightInfoResp.getNonFSOKeyInfoList().size());
+ assertEquals("sampleVol/bucketOne/key_3",
keyInsightInfoResp.getNonFSOKeyInfoList().get(0).getPath());
+ assertEquals("sampleVol/bucketOne/key_2",
keyInsightInfoResp.getFsoKeyInfoList().get(0).getPath());
}
@Test
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenKeysSearchEndpoint.java
similarity index 83%
rename from
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java
rename to
hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenKeysSearchEndpoint.java
index c3c2fe5deb..f55d988cfe 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOMDBInsightSearchEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestOpenKeysSearchEndpoint.java
@@ -81,12 +81,12 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
* 11. Test Search Open Keys with Pagination: Verifies paginated search
results.
* 12. Test Search in Empty Bucket: Checks the response for searching within
an empty bucket.
*/
-public class TestOMDBInsightSearchEndpoint extends AbstractReconSqlDBTest {
+public class TestOpenKeysSearchEndpoint extends AbstractReconSqlDBTest {
@TempDir
private Path temporaryFolder;
private ReconOMMetadataManager reconOMMetadataManager;
- private OMDBInsightSearchEndpoint omdbInsightSearchEndpoint;
+ private OMDBInsightEndpoint omdbInsightEndpoint;
private OzoneConfiguration ozoneConfiguration;
private static final String ROOT_PATH = "/";
private static final String TEST_USER = "TestUser";
@@ -116,11 +116,8 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
.addBinding(OMDBInsightEndpoint.class)
.addBinding(ContainerHealthSchemaManager.class)
.build();
- reconNamespaceSummaryManager =
- reconTestInjector.getInstance(ReconNamespaceSummaryManager.class);
- omdbInsightSearchEndpoint = reconTestInjector.getInstance(
- OMDBInsightSearchEndpoint.class);
-
+ reconNamespaceSummaryManager =
reconTestInjector.getInstance(ReconNamespaceSummaryManager.class);
+ omdbInsightEndpoint =
reconTestInjector.getInstance(OMDBInsightEndpoint.class);
// populate OM DB and reprocess into Recon RocksDB
populateOMDB();
NSSummaryTaskWithFSO nSSummaryTaskWithFso =
@@ -150,26 +147,19 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
public void testRootLevelSearchRestriction() throws IOException {
// Test with root level path
String rootPath = "/";
- Response response = omdbInsightSearchEndpoint.searchOpenKeys(rootPath, 20,
"");
+ Response response =
+ omdbInsightEndpoint.getOpenKeyInfo(-1, "", rootPath, true, true);
assertEquals(Response.Status.BAD_REQUEST.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("Invalid startPrefix: Path must be at the
bucket level or deeper"),
"Expected a message indicating the path must be at the bucket level or
deeper");
-
- // Test with root level path without trailing slash
- rootPath = "";
- response = omdbInsightSearchEndpoint.searchOpenKeys(rootPath, 20, "");
- assertEquals(Response.Status.BAD_REQUEST.getStatusCode(),
response.getStatus());
- entity = (String) response.getEntity();
- assertTrue(entity.contains("Invalid startPrefix: Path must be at the
bucket level or deeper"),
- "Expected a message indicating the path must be at the bucket level or
deeper");
}
@Test
public void testVolumeLevelSearchRestriction() throws IOException {
// Test with volume level path
String volumePath = "/vola";
- Response response = omdbInsightSearchEndpoint.searchOpenKeys(volumePath,
20, "");
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(20, "", volumePath,
true, true);
assertEquals(Response.Status.BAD_REQUEST.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("Invalid startPrefix: Path must be at the
bucket level or deeper"),
@@ -177,7 +167,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
// Test with another volume level path
volumePath = "/volb";
- response = omdbInsightSearchEndpoint.searchOpenKeys(volumePath, 20, "");
+ response = omdbInsightEndpoint.getOpenKeyInfo(20, "", volumePath, true,
true);
assertEquals(Response.Status.BAD_REQUEST.getStatusCode(),
response.getStatus());
entity = (String) response.getEntity();
assertTrue(entity.contains("Invalid startPrefix: Path must be at the
bucket level or deeper"),
@@ -188,7 +178,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
public void testBucketLevelSearch() throws IOException {
// Search inside FSO bucket
Response response =
- omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1", 20, "");
+ omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1", true,
true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result =
(KeyInsightInfoResponse) response.getEntity();
@@ -200,7 +190,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
// Search inside OBS bucket
response =
- omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1", 20, "");
+ omdbInsightEndpoint.getOpenKeyInfo(20, "", "/volb/bucketb1", true,
true);
assertEquals(200, response.getStatus());
result =
(KeyInsightInfoResponse) response.getEntity();
@@ -212,13 +202,13 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
// Search Inside LEGACY bucket
response =
- omdbInsightSearchEndpoint.searchOpenKeys("/volc/bucketc1", 20, "");
+ omdbInsightEndpoint.getOpenKeyInfo(20, "", "/volc/bucketc1", true,
true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(7, result.getNonFSOKeyInfoList().size());
// Test with bucket that does not exist
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/nonexistentbucket", 20, "");
+ response = omdbInsightEndpoint.getOpenKeyInfo(20, "",
"/vola/nonexistentbucket", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
@@ -228,7 +218,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testDirectoryLevelSearch() throws IOException {
Response response =
- omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1", 20,
"");
+ omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/dira1",
true, true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result =
(KeyInsightInfoResponse) response.getEntity();
@@ -239,7 +229,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(1000 * 3, result.getReplicatedDataSize());
response =
- omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2", 20,
"");
+ omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/dira2",
true, true);
assertEquals(200, response.getStatus());
result =
(KeyInsightInfoResponse) response.getEntity();
@@ -250,7 +240,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(1000 * 3, result.getReplicatedDataSize());
response =
- omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3", 20,
"");
+ omdbInsightEndpoint.getOpenKeyInfo(20, "", "/vola/bucketa1/dira3",
true, true);
assertEquals(200, response.getStatus());
result =
(KeyInsightInfoResponse) response.getEntity();
@@ -261,7 +251,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(10000 * 3, result.getReplicatedDataSize());
// Test with non-existent directory
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentdir", 20,
"");
+ response = omdbInsightEndpoint.getOpenKeyInfo(20, "",
"/vola/bucketa1/nonexistentdir", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
@@ -271,7 +261,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testKeyLevelSearch() throws IOException {
// FSO Bucket key-level search
- Response response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/filea1", 10, "");
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(10, "",
"/vola/bucketa1/filea1", true, true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
assertEquals(1, result.getFsoKeyInfoList().size());
@@ -280,7 +270,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(1000, result.getUnreplicatedDataSize());
assertEquals(1000 * 3, result.getReplicatedDataSize());
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/filea2", 10, "");
+ response = omdbInsightEndpoint.getOpenKeyInfo(10, "",
"/vola/bucketa1/filea2", true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(1, result.getFsoKeyInfoList().size());
@@ -290,7 +280,8 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(1000 * 3, result.getReplicatedDataSize());
// OBS Bucket key-level search
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/fileb1", 10, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/volb/bucketb1/fileb1", true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(0, result.getFsoKeyInfoList().size());
@@ -299,7 +290,8 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(1000, result.getUnreplicatedDataSize());
assertEquals(1000 * 3, result.getReplicatedDataSize());
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/fileb2", 10, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/volb/bucketb1/fileb2", true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(0, result.getFsoKeyInfoList().size());
@@ -309,13 +301,15 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertEquals(1000 * 3, result.getReplicatedDataSize());
// Test with non-existent key
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/nonexistentfile", 1,
"");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/volb/bucketb1/nonexistentfile", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
"Expected a message indicating no keys were found");
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1/nonexistentfile", 1,
"");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/volb/bucketb1/nonexistentfile", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
@@ -326,29 +320,31 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testKeyLevelSearchUnderDirectory() throws IOException {
// FSO Bucket key-level search
- Response response =
-
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/innerfile", 10,
"");
+ Response response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/vola/bucketa1/dira1/innerfile", true, true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
assertEquals(1, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
- response =
-
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/innerfile", 10,
"");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/vola/bucketa1/dira2/innerfile", true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(1, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
// Test for unknown file in fso bucket
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira1/unknownfile",
10, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/vola/bucketa1/dira1/unknownfile", true,
true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
"Expected a message indicating no keys were found");
// Test for unknown file in fso bucket
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira2/unknownfile",
10, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(10, "", "/vola/bucketa1/dira2/unknownfile", true,
true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
@@ -358,55 +354,55 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testSearchUnderNestedDirectory() throws IOException {
- Response response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3", 20,
- "");
+ Response response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3", true, true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
assertEquals(10, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
// Search under dira31
- response =
omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1/dira3/dira31",
- 20, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31", true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(6, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
// Search under dira32
- response = omdbInsightSearchEndpoint.searchOpenKeys(
- "/vola/bucketa1/dira3/dira31/dira32", 20, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32", true,
true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(3, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
// Search under dira33
- response = omdbInsightSearchEndpoint.searchOpenKeys(
- "/vola/bucketa1/dira3/dira31/dira32/dira33", 20, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "", "/vola/bucketa1/dira3/dira31/dira32/dira33",
true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(1, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
// Search for the exact file under dira33
- response = omdbInsightSearchEndpoint.searchOpenKeys(
- "/vola/bucketa1/dira3/dira31/dira32/dira33/file33_1", 20, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "",
"/vola/bucketa1/dira3/dira31/dira32/dira33/file33_1", true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(1, result.getFsoKeyInfoList().size());
assertEquals(0, result.getNonFSOKeyInfoList().size());
// Search for a non existant file under each nested directory
- response = omdbInsightSearchEndpoint.searchOpenKeys(
- "/vola/bucketa1/dira3/dira31/dira32/dira33/nonexistentfile", 20, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "",
"/vola/bucketa1/dira3/dira31/dira32/dira33/nonexistentfile", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
"Expected a message indicating no keys were found");
- response = omdbInsightSearchEndpoint.searchOpenKeys(
- "/vola/bucketa1/dira3/dira31/dira32/nonexistentfile", 20, "");
+ response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "",
"/vola/bucketa1/dira3/dira31/dira32/nonexistentfile", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
@@ -416,7 +412,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testLimitSearch() throws IOException {
Response response =
- omdbInsightSearchEndpoint.searchOpenKeys("/vola/bucketa1", 2, "");
+ omdbInsightEndpoint.getOpenKeyInfo(2, "", "/vola/bucketa1", true,
true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result =
(KeyInsightInfoResponse) response.getEntity();
@@ -428,8 +424,8 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
public void testSearchOpenKeysWithBadRequest() throws IOException {
// Give a negative limit
int negativeLimit = -1;
- Response response = omdbInsightSearchEndpoint.searchOpenKeys("@323232",
negativeLimit, "");
-
+ Response response = omdbInsightEndpoint
+ .getOpenKeyInfo(negativeLimit, "", "@323232", true, true);
// Then the response should indicate that the request was bad
assertEquals(Response.Status.BAD_REQUEST.getStatusCode(),
response.getStatus(), "Expected a 400 BAD REQUEST status");
@@ -438,7 +434,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertTrue(entity.contains("Invalid startPrefix: Path must be at the
bucket level or deeper"),
"Expected a message indicating the path must be at the bucket level or
deeper");
- response = omdbInsightSearchEndpoint.searchOpenKeys("///", 20, "");
+ response = omdbInsightEndpoint.getOpenKeyInfo(20, "", "///", true, true);
assertEquals(Response.Status.BAD_REQUEST.getStatusCode(),
response.getStatus());
entity = (String) response.getEntity();
assertTrue(entity.contains("Invalid startPrefix: Path must be at the
bucket level or deeper"),
@@ -447,8 +443,8 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testLastKeyInResponse() throws IOException {
- Response response =
- omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb1", 20, "");
+ Response response = omdbInsightEndpoint
+ .getOpenKeyInfo(20, "", "/volb/bucketb1", true, true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result =
(KeyInsightInfoResponse) response.getEntity();
@@ -470,7 +466,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
String prevKey = "";
// Perform the first search request
- Response response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix,
limit, prevKey);
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(limit, prevKey,
startPrefix, true, true);
assertEquals(200, response.getStatus());
KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
assertEquals(2, result.getNonFSOKeyInfoList().size());
@@ -481,7 +477,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertNotNull(prevKey, "Last key should not be null");
// Perform the second search request using the last key
- response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit,
prevKey);
+ response = omdbInsightEndpoint.getOpenKeyInfo(limit, prevKey, startPrefix,
true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(2, result.getNonFSOKeyInfoList().size());
@@ -492,7 +488,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
assertNotNull(prevKey, "Last key should not be null");
// Perform the third search request using the last key
- response = omdbInsightSearchEndpoint.searchOpenKeys(startPrefix, limit,
prevKey);
+ response = omdbInsightEndpoint.getOpenKeyInfo(limit, prevKey, startPrefix,
true, true);
assertEquals(200, response.getStatus());
result = (KeyInsightInfoResponse) response.getEntity();
assertEquals(1, result.getNonFSOKeyInfoList().size());
@@ -504,13 +500,61 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
@Test
public void testSearchInEmptyBucket() throws IOException {
// Search in empty bucket bucketb2
- Response response =
omdbInsightSearchEndpoint.searchOpenKeys("/volb/bucketb2", 20, "");
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(20, "",
"/volb/bucketb2", true, true);
assertEquals(Response.Status.NO_CONTENT.getStatusCode(),
response.getStatus());
String entity = (String) response.getEntity();
assertTrue(entity.contains("No keys matched the search prefix"),
"Expected a message indicating no keys were found");
}
+ @Test
+ public void testSearchWithPrevKeyOnly() throws IOException {
+ String prevKey = "/volb/bucketb1/fileb1"; // Key exists in volb/bucketb1
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(4, prevKey, "",
true, true);
+
+ assertEquals(200, response.getStatus());
+
+ KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
+ assertEquals(4, result.getNonFSOKeyInfoList().size(), "Expected 4
remaining keys after 'fileb1'");
+ assertEquals("/volb/bucketb1/fileb5", result.getLastKey(), "Expected last
key to be 'fileb5'");
+ }
+
+ @Test
+ public void testSearchWithEmptyPrevKeyAndStartPrefix() throws IOException {
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(-1, "", "", true,
true);
+
+ assertEquals(200, response.getStatus());
+
+ KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
+ // Assert all the keys are returned
+ assertEquals(12, result.getNonFSOKeyInfoList().size(), "Expected all keys
to be returned");
+ }
+
+ @Test
+ public void testSearchWithStartPrefixOnly() throws IOException {
+ String startPrefix = "/volb/bucketb1/";
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(10, "",
startPrefix, true, true);
+
+ assertEquals(200, response.getStatus());
+
+ KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
+ assertEquals(5, result.getNonFSOKeyInfoList().size(), "Expected 5 keys
starting with 'fileb1'");
+ assertEquals("/volb/bucketb1/fileb5", result.getLastKey(), "Expected last
key to be 'fileb5'");
+ }
+
+ @Test
+ public void testSearchWithPrevKeyAndStartPrefix() throws IOException {
+ String startPrefix = "/volb/bucketb1/";
+ String prevKey = "/volb/bucketb1/fileb1";
+ Response response = omdbInsightEndpoint.getOpenKeyInfo(10, prevKey,
startPrefix, true, true);
+
+ assertEquals(200, response.getStatus());
+
+ KeyInsightInfoResponse result = (KeyInsightInfoResponse)
response.getEntity();
+ assertEquals(4, result.getNonFSOKeyInfoList().size(), "Expected 4 keys
after 'fileb1'");
+ assertEquals("/volb/bucketb1/fileb5", result.getLastKey(), "Expected last
key to be 'fileb5'");
+ }
+
/**
* Tests the NSSummaryEndpoint for a given volume, bucket, and directory
structure.
* The test setup mimics the following filesystem structure with specified
sizes:
@@ -566,7 +610,7 @@ public class TestOMDBInsightSearchEndpoint extends
AbstractReconSqlDBTest {
// Create Bucket in volb
createBucket("volb", "bucketb1", 1000 + 1000 + 1000 + 1000 + 1000,
- getOBSBucketLayout());
+ getOBSBucketLayout());
createBucket("volb", "bucketb2", 0, getOBSBucketLayout()); // Empty Bucket
// Create Bucket in volc
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
