smengcl commented on a change in pull request #108: HDDS-1987. Fix listStatus
API
URL: https://github.com/apache/hadoop-ozone/pull/108#discussion_r350245710
##########
File path:
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
##########
@@ -770,6 +776,214 @@ public void testLookupKeyWithLocation() throws
IOException {
.getLocationList().get(0).getPipeline().getClosestNode());
}
+ @Test
+ public void testListStatusWithTableCache() throws Exception {
+ // Inspired by TestOmMetadataManager#testListKeys
+ String prefixKeyInDB = "key-d";
+ String prefixKeyInCache = "key-c";
+
+ // Add a total of 100 key entries to DB and TableCache (50 entries each)
+ for (int i = 1; i <= 100; i++) {
+ if (i % 2 == 0) { // Add to DB
+ TestOMRequestUtils.addKeyToTable(false,
+ VOLUME_NAME, BUCKET_NAME, prefixKeyInDB + i,
+ 1000L, HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.ONE, metadataManager);
+ } else { // Add to TableCache
+ TestOMRequestUtils.addKeyToTableCache(
+ VOLUME_NAME, BUCKET_NAME, prefixKeyInCache + i,
+ HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
+ metadataManager);
+ }
+ }
+
+ OmKeyArgs rootDirArgs = createKeyArgs("");
+ // Get entries in both TableCache and DB
+ List<OzoneFileStatus> fileStatuses =
+ keyManager.listStatus(rootDirArgs, true, "", 1000);
+ Assert.assertEquals(100, fileStatuses.size());
+
+ // Get entries with startKey=prefixKeyInDB
+ fileStatuses =
+ keyManager.listStatus(rootDirArgs, true, prefixKeyInDB, 1000);
+ Assert.assertEquals(50, fileStatuses.size());
+
+ // Get entries with startKey=prefixKeyInCache
+ fileStatuses =
+ keyManager.listStatus(rootDirArgs, true, prefixKeyInCache, 1000);
+ Assert.assertEquals(100, fileStatuses.size());
+
+ // Clean up cache by marking those keys in cache as deleted
+ for (int i = 1; i <= 100; i += 2) {
+ String key = metadataManager.getOzoneKey(
+ VOLUME_NAME, BUCKET_NAME, prefixKeyInCache + i);
+ metadataManager.getKeyTable().addCacheEntry(new CacheKey<>(key),
+ new CacheValue<>(Optional.absent(), 2L));
+ }
+ }
+
+ @Test
+ public void testListStatusWithTableCacheRecursive() throws Exception {
+ String keyNameDir1 = "dir1";
+ OmKeyArgs keyArgsDir1 =
+ createBuilder().setKeyName(keyNameDir1).build();
+ keyManager.createDirectory(keyArgsDir1);
+
+ String keyNameDir1Subdir1 = "dir1" + OZONE_URI_DELIMITER + "subdir1";
+ OmKeyArgs keyArgsDir1Subdir1 =
+ createBuilder().setKeyName(keyNameDir1Subdir1).build();
+ keyManager.createDirectory(keyArgsDir1Subdir1);
+
+ String keyNameDir2 = "dir2";
+ OmKeyArgs keyArgsDir2 =
+ createBuilder().setKeyName(keyNameDir2).build();
+ keyManager.createDirectory(keyArgsDir2);
+
+ OmKeyArgs rootDirArgs = createKeyArgs("");
+ // Test listStatus with recursive=false, should only have dirs under root
+ List<OzoneFileStatus> fileStatuses =
+ keyManager.listStatus(rootDirArgs, false, "", 1000);
+ Assert.assertEquals(2, fileStatuses.size());
+
+ // Test listStatus with recursive=true, should have dirs under root and
+ fileStatuses =
+ keyManager.listStatus(rootDirArgs, true, "", 1000);
+ Assert.assertEquals(3, fileStatuses.size());
+
+ // Add a total of 10 key entries to DB and TableCache under dir1
+ String prefixKeyInDB = "key-d";
+ String prefixKeyInCache = "key-c";
+ for (int i = 1; i <= 10; i++) {
+ if (i % 2 == 0) { // Add to DB
+ TestOMRequestUtils.addKeyToTable(false,
+ VOLUME_NAME, BUCKET_NAME,
+ keyNameDir1Subdir1 + OZONE_URI_DELIMITER + prefixKeyInDB + i,
+ 1000L, HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.ONE, metadataManager);
+ } else { // Add to TableCache
+ TestOMRequestUtils.addKeyToTableCache(
+ VOLUME_NAME, BUCKET_NAME,
+ keyNameDir1Subdir1 + OZONE_URI_DELIMITER + prefixKeyInCache + i,
+ HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
+ metadataManager);
+ }
+ }
+
+ // Test non-recursive, should return the dir under root
+ fileStatuses =
+ keyManager.listStatus(rootDirArgs, false, "", 1000);
+ Assert.assertEquals(2, fileStatuses.size());
+
+ // Test recursive, should return the dir and the keys in it
+ fileStatuses =
+ keyManager.listStatus(rootDirArgs, true, "", 1000);
+ Assert.assertEquals(10 + 3, fileStatuses.size());
+
+ // Clean up
+ for (int i = 1; i <= 10; i += 2) {
+ // Mark TableCache entries as deleted
+ // Note that DB entry clean up is handled by cleanupTest()
+ String key = metadataManager.getOzoneKey(
+ VOLUME_NAME, BUCKET_NAME,
+ keyNameDir1Subdir1 + OZONE_URI_DELIMITER + prefixKeyInCache + i);
+ metadataManager.getKeyTable().addCacheEntry(new CacheKey<>(key),
+ new CacheValue<>(Optional.absent(), 2L));
+ }
+ }
+
+ @Test
+ public void testListStatusWithDeletedEntriesInCache() throws Exception {
+ String prefixKey = "key-";
+ TreeSet<String> existKeySet = new TreeSet<>();
+ TreeSet<String> deletedKeySet = new TreeSet<>();
+
+ for (int i = 1; i <= 100; i++) {
+ if (i % 2 == 0) {
+ TestOMRequestUtils.addKeyToTable(false,
+ VOLUME_NAME, BUCKET_NAME, prefixKey + i,
+ 1000L, HddsProtos.ReplicationType.RATIS,
+ HddsProtos.ReplicationFactor.ONE, metadataManager);
+ existKeySet.add(prefixKey + i);
+ } else {
+ TestOMRequestUtils.addKeyToTableCache(
+ VOLUME_NAME, BUCKET_NAME, prefixKey + i,
+ HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE,
+ metadataManager);
+
+ String key = metadataManager.getOzoneKey(
+ VOLUME_NAME, BUCKET_NAME, prefixKey + i);
+ // Mark as deleted in cache.
+ metadataManager.getKeyTable().addCacheEntry(new CacheKey<>(key),
+ new CacheValue<>(Optional.absent(), 2L));
+ deletedKeySet.add(key);
+ }
+ }
+
+ OmKeyArgs rootDirArgs = createKeyArgs("");
+ List<OzoneFileStatus> fileStatuses =
+ keyManager.listStatus(rootDirArgs, true, "", 1000);
+ // Should only get entries that are not marked as deleted.
Review comment:
Sure. Added pagination check to existing test (with cache).
Also `testListStatus()` has pagination test (without cache) with two-level
deep dir.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: ozone-issues-h...@hadoop.apache.org
