This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4e364001ac HDDS-7211. Remove dead code from KeyManagerImpl (#3747)
4e364001ac is described below
commit 4e364001ac752e8764d2b9c4d8b6880d3fb78135
Author: Maxim Myskov <[email protected]>
AuthorDate: Thu Oct 6 21:46:12 2022 +0300
HDDS-7211. Remove dead code from KeyManagerImpl (#3747)
---
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 659 +--------------------
.../org/apache/hadoop/ozone/om/OzoneManager.java | 2 +-
.../ozone/om/request/key/TestOMKeyRequest.java | 2 +-
4 files changed, 19 insertions(+), 648 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index 8c89f7c4e2..492173b71c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -1336,7 +1336,7 @@ public class TestKeyManagerImpl {
omKeyInfo.appendNewBlocks(omKeyLocationInfoList, false);
KeyManagerImpl keyManagerImpl =
- new KeyManagerImpl(ozoneManager, scmClientMock, conf, "om1", metrics);
+ new KeyManagerImpl(ozoneManager, scmClientMock, conf, metrics);
keyManagerImpl.refresh(omKeyInfo);
@@ -1377,7 +1377,7 @@ public class TestKeyManagerImpl {
omKeyInfo.appendNewBlocks(omKeyLocationInfoList, false);
KeyManagerImpl keyManagerImpl =
- new KeyManagerImpl(ozoneManager, scmClientMock, conf, "om1", metrics);
+ new KeyManagerImpl(ozoneManager, scmClientMock, conf, metrics);
try {
keyManagerImpl.refresh(omKeyInfo);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 3de7053287..4ea8c52bf4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -37,7 +37,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.Stack;
import java.util.TreeMap;
-import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -105,19 +104,14 @@ import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX;
-import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT;
@@ -127,7 +121,6 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketLayout;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
-import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KMS_PROVIDER;
@@ -155,19 +148,13 @@ public class KeyManagerImpl implements KeyManager {
private final ScmClient scmClient;
private final OMMetadataManager metadataManager;
private final long scmBlockSize;
- private final boolean useRatis;
-
- private final int preallocateBlocksMax;
private final int listTrashKeysMax;
- private final String omId;
private final OzoneBlockTokenSecretManager secretManager;
private final boolean grpcBlockTokenEnabled;
private BackgroundService keyDeletingService;
private final KeyProviderCryptoExtension kmsProvider;
- private final PrefixManager prefixManager;
-
private final boolean enableFileSystemPaths;
private BackgroundService dirDeletingService;
private final OMPerformanceMetrics metrics;
@@ -176,11 +163,11 @@ public class KeyManagerImpl implements KeyManager {
@VisibleForTesting
public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
- OMMetadataManager metadataManager, OzoneConfiguration conf, String omId,
+ OMMetadataManager metadataManager, OzoneConfiguration conf,
OzoneBlockTokenSecretManager secretManager,
OMPerformanceMetrics metrics) {
this(null, new ScmClient(scmBlockClient, null, conf), metadataManager,
- conf, omId, secretManager, null, null, metrics);
+ conf, secretManager, null, metrics);
}
@VisibleForTesting
@@ -190,31 +177,24 @@ public class KeyManagerImpl implements KeyManager {
OzoneBlockTokenSecretManager secretManager,
OMPerformanceMetrics metrics) {
this(null, new ScmClient(scmBlockClient, scmContainerClient, conf),
- metadataManager, conf, omId, secretManager, null, null,
+ metadataManager, conf, secretManager, null,
metrics);
}
public KeyManagerImpl(OzoneManager om, ScmClient scmClient,
- OzoneConfiguration conf, String omId, OMPerformanceMetrics metrics) {
- this (om, scmClient, om.getMetadataManager(), conf, omId,
- om.getBlockTokenMgr(), om.getKmsProvider(), om.getPrefixManager(),
- metrics);
+ OzoneConfiguration conf, OMPerformanceMetrics metrics) {
+ this (om, scmClient, om.getMetadataManager(), conf,
+ om.getBlockTokenMgr(), om.getKmsProvider(), metrics);
}
- @SuppressWarnings("parameternumber")
public KeyManagerImpl(OzoneManager om, ScmClient scmClient,
- OMMetadataManager metadataManager, OzoneConfiguration conf, String omId,
+ OMMetadataManager metadataManager, OzoneConfiguration conf,
OzoneBlockTokenSecretManager secretManager,
- KeyProviderCryptoExtension kmsProvider, PrefixManager prefixManager,
+ KeyProviderCryptoExtension kmsProvider,
OMPerformanceMetrics metrics) {
this.scmBlockSize = (long) conf
.getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
StorageUnit.BYTES);
- this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
- DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
- this.preallocateBlocksMax = conf.getInt(
- OZONE_KEY_PREALLOCATION_BLOCKS_MAX,
- OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT);
this.grpcBlockTokenEnabled = conf.getBoolean(
HDDS_BLOCK_TOKEN_ENABLED,
HDDS_BLOCK_TOKEN_ENABLED_DEFAULT);
@@ -226,14 +206,11 @@ public class KeyManagerImpl implements KeyManager {
OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
this.ozoneManager = om;
- this.omId = omId;
this.scmClient = scmClient;
this.metadataManager = metadataManager;
- this.prefixManager = prefixManager;
this.secretManager = secretManager;
this.kmsProvider = kmsProvider;
this.metrics = metrics;
-
}
@Override
@@ -309,26 +286,6 @@ public class KeyManagerImpl implements KeyManager {
return metadataManager.getBucketTable().get(bucketKey);
}
- /**
- * Check S3 bucket exists or not.
- * @param volumeName
- * @param bucketName
- * @throws IOException
- */
- private OmBucketInfo validateS3Bucket(String volumeName, String bucketName)
- throws IOException {
-
- String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
- OmBucketInfo omBucketInfo = metadataManager.getBucketTable().
- get(bucketKey);
- //Check if bucket already exists
- if (omBucketInfo == null) {
- LOG.error("bucket not found: {}/{} ", volumeName, bucketName);
- throw new OMException("Bucket not found",
- BUCKET_NOT_FOUND);
- }
- return omBucketInfo;
- }
/* Optimize ugi lookup for RPC operations to avoid a trip through
* UGI.getCurrentUser which is synch'ed.
*/
@@ -560,16 +517,6 @@ public class KeyManagerImpl implements KeyManager {
}
}
- private boolean isKeyEmpty(OmKeyInfo keyInfo) {
- for (OmKeyLocationInfoGroup keyLocationList : keyInfo
- .getKeyLocationVersions()) {
- if (keyLocationList.getLocationListCount() != 0) {
- return false;
- }
- }
- return true;
- }
-
@Override
public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix,
@@ -784,8 +731,6 @@ public class KeyManagerImpl implements KeyManager {
"ReplicationConfig can't be identified");
if (partKeyInfoMapIterator.hasNext()) {
- Map.Entry<Integer, PartKeyInfo> partKeyInfoEntry =
- partKeyInfoMapIterator.next();
isTruncated = true;
} else {
isTruncated = false;
@@ -1551,21 +1496,15 @@ public class KeyManagerImpl implements KeyManager {
return fileStatusList;
}
- boolean useNewIterator = true;
if (isBucketFSOptimized(volName, buckName)) {
Preconditions.checkArgument(!recursive);
- if (useNewIterator) {
- OzoneListStatusHelper statusHelper =
- new OzoneListStatusHelper(metadataManager, scmBlockSize,
- this::getOzoneFileStatusFSO);
- Collection<OzoneFileStatus> statuses =
- statusHelper.listStatusFSO(args, startKey, numEntries,
- clientAddress, allowPartialPrefixes);
- return buildFinalStatusList(statuses, args, clientAddress);
- } else {
- return listStatusFSO(args, recursive, startKey, numEntries,
- clientAddress);
- }
+ OzoneListStatusHelper statusHelper =
+ new OzoneListStatusHelper(metadataManager, scmBlockSize,
+ this::getOzoneFileStatusFSO);
+ Collection<OzoneFileStatus> statuses =
+ statusHelper.listStatusFSO(args, startKey, numEntries,
+ clientAddress, allowPartialPrefixes);
+ return buildFinalStatusList(statuses, args, clientAddress);
}
String volumeName = args.getVolumeName();
@@ -1726,270 +1665,6 @@ public class KeyManagerImpl implements KeyManager {
}
}
- @SuppressWarnings("methodlength")
- public List<OzoneFileStatus> listStatusFSO(OmKeyArgs args, boolean recursive,
- String startKey, long numEntries, String clientAddress)
- throws IOException {
- Preconditions.checkNotNull(args, "Key args can not be null");
-
- // unsorted OMKeyInfo list contains combine results from TableCache and DB.
-
- if (numEntries <= 0) {
- return new ArrayList<>();
- }
-
- /**
- * A map sorted by OmKey to combine results from TableCache and DB for
- * each entity - Dir & File.
- *
- * Two separate maps are required because the order of seek -> (1)Seek
- * files in fileTable (2)Seek dirs in dirTable.
- *
- * StartKey should be added to the final listStatuses, so if we combine
- * files and dirs into a single map then directory with lower precedence
- * will appear at the top of the list even if the startKey is given as
- * fileName.
- *
- * For example, startKey="a/file1". As per the seek order, first fetches
- * all the files and then it will start seeking all the directories.
- * Assume a directory name exists "a/b". With one map, the sorted list will
- * be ["a/b", "a/file1"]. But the expected list is: ["a/file1", "a/b"],
- * startKey element should always be at the top of the listStatuses.
- */
- TreeMap<String, OzoneFileStatus> cacheFileMap = new TreeMap<>();
- TreeMap<String, OzoneFileStatus> cacheDirMap = new TreeMap<>();
-
- final String volumeName = args.getVolumeName();
- final String bucketName = args.getBucketName();
- final String keyName = args.getKeyName();
- final long volumeId = metadataManager.getVolumeId(volumeName);
- final long bucketId = metadataManager.getBucketId(volumeName, bucketName);
- String seekFileInDB;
- String seekDirInDB;
- long prefixKeyInDB;
- String prefixPath = keyName;
- int countEntries = 0;
-
- // TODO: recursive flag=true will be handled in HDDS-4360 jira.
-
-
- Set<String> deletedKeySet = new TreeSet<>();
- TreeMap<String, OzoneFileStatus> tempCacheDirMap = new TreeMap<>();
-
- TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- iterator = null;
-
- try {
- if (Strings.isNullOrEmpty(startKey)) {
- OzoneFileStatus fileStatus = getFileStatus(args, clientAddress);
- if (fileStatus.isFile()) {
- return Collections.singletonList(fileStatus);
- }
-
- // Not required to search in DeletedTable because all the deleted
- // keys will be marked directly in dirTable or in keyTable by
- // breaking the pointer to its sub-dirs and sub-files. So, there is no
- // issue of inconsistency.
-
- /*
- * keyName is a directory.
- * Say, "/a" is the dir name and its objectID is 1024, then seek
- * will be doing with "1024/" to get all immediate descendants.
- */
- if (fileStatus.getKeyInfo() != null) {
- prefixKeyInDB = fileStatus.getKeyInfo().getObjectID();
- } else {
- // list root directory.
- prefixKeyInDB = bucketId;
- }
- seekFileInDB = metadataManager.getOzonePathKey(
- volumeId, bucketId, prefixKeyInDB, "");
- seekDirInDB = metadataManager.getOzonePathKey(
- volumeId, bucketId, prefixKeyInDB, "");
-
- // Order of seek ->
- // (1)Seek files in fileTable
- // (2)Seek dirs in dirTable
-
- // First under lock obtain both entries from dir/file cache and
generate
- // entries marked for delete.
- metadataManager.getLock()
- .acquireReadLock(BUCKET_LOCK, volumeName, bucketName);
- try {
- BucketLayout bucketLayout = getBucketLayout(
- metadataManager, volumeName, bucketName);
- iterator = metadataManager.getKeyTable(bucketLayout).iterator();
- countEntries =
- getFilesAndDirsFromCacheWithBucket(volumeName, bucketName,
- cacheFileMap, tempCacheDirMap, deletedKeySet, prefixKeyInDB,
- seekFileInDB, seekDirInDB, prefixPath, startKey,
countEntries,
- numEntries);
-
- } finally {
- metadataManager.getLock()
- .releaseReadLock(BUCKET_LOCK, volumeName, bucketName);
- }
- countEntries =
- getFilesFromDirectory(cacheFileMap, seekFileInDB, prefixPath,
- prefixKeyInDB, countEntries, numEntries, deletedKeySet,
- iterator);
-
- } else {
- /*
- * startKey will be used in iterator seek and sets the beginning point
- * for key traversal.
- * keyName will be used as parentID where the user has requested to
- * list the keys from.
- *
- * When recursive flag=false, parentID won't change between two pages.
- * For example: OM has a namespace like,
- * /a/1...1M files and /a/b/1...1M files.
- * /a/1...1M directories and /a/b/1...1M directories.
- * Listing "/a", will always have the parentID as "a" irrespective of
- * the startKey value.
- */
-
- // Check startKey is an immediate child of keyName. For example,
- // keyName=/a/ and expected startKey=/a/b. startKey can't be /xyz/b.
- if (StringUtils.isNotBlank(keyName) && !OzoneFSUtils
- .isImmediateChild(keyName, startKey)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("StartKey {} is not an immediate child of keyName {}. "
- + "Returns empty list", startKey, keyName);
- }
- return Collections.emptyList();
- }
-
- // assign startKeyPath if prefixPath is empty string.
- if (StringUtils.isBlank(prefixPath)) {
- prefixPath = OzoneFSUtils.getParentDir(startKey);
- }
-
- OmKeyArgs startKeyArgs = args.toBuilder()
- .setKeyName(startKey)
- .setSortDatanodesInPipeline(false)
- .build();
- OzoneFileStatus fileStatusInfo = getOzoneFileStatusFSO(startKeyArgs,
- null, true);
-
- if (fileStatusInfo != null) {
- prefixKeyInDB = fileStatusInfo.getKeyInfo().getParentObjectID();
-
- if (fileStatusInfo.isDirectory()) {
- seekDirInDB = metadataManager.getOzonePathKey(
- volumeId, bucketId, prefixKeyInDB,
- fileStatusInfo.getKeyInfo().getFileName());
-
- // Order of seek -> (1) Seek dirs only in dirTable. In OM, always
- // the order of search is, first seek into fileTable and then
- // dirTable. So, its not required to search again in the fileTable.
-
- // Seek the given key in dirTable.
- metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
- bucketName);
- try {
- listStatusFindDirsInTableCache(tempCacheDirMap,
- metadataManager.getDirectoryTable(),
- prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName,
- bucketName, countEntries, numEntries, deletedKeySet);
- } finally {
- metadataManager.getLock().releaseReadLock(BUCKET_LOCK,
volumeName,
- bucketName);
- }
-
- } else {
- seekFileInDB = metadataManager.getOzonePathKey(
- volumeId, bucketId, prefixKeyInDB,
- fileStatusInfo.getKeyInfo().getFileName());
- // begins from the first sub-dir under the parent dir
- seekDirInDB = metadataManager.getOzonePathKey(
- volumeId, bucketId, prefixKeyInDB, "");
-
- // First under lock obtain both entries from dir/file cache and
- // generate entries marked for delete.
- metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
- bucketName);
- try {
- BucketLayout bucketLayout =
- getBucketLayout(metadataManager, volumeName, bucketName);
- iterator = metadataManager.getKeyTable(bucketLayout)
- .iterator();
- countEntries = getFilesAndDirsFromCacheWithBucket(volumeName,
- bucketName, cacheFileMap, tempCacheDirMap, deletedKeySet,
- prefixKeyInDB, seekFileInDB, seekDirInDB, prefixPath,
- startKey, countEntries, numEntries);
- } finally {
- metadataManager.getLock().releaseReadLock(BUCKET_LOCK,
volumeName,
- bucketName);
- }
-
- // 1. Seek the given key in key table.
- countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB,
- prefixPath, prefixKeyInDB, countEntries, numEntries,
- deletedKeySet, iterator);
- }
- } else {
- // TODO: HDDS-4364: startKey can be a non-existed key
- if (LOG.isDebugEnabled()) {
- LOG.debug("StartKey {} is a non-existed key and returning empty "
- + "list", startKey);
- }
- return Collections.emptyList();
- }
- }
- } finally {
- if (iterator != null) {
- iterator.close();
- }
- }
-
- // Now first add all entries in the dir cache, if numEntries required
- // is less than countEntries.
- for (Map.Entry<String, OzoneFileStatus> dirEntry :
- tempCacheDirMap.entrySet()) {
- if (countEntries < numEntries) {
- cacheDirMap.put(dirEntry.getKey(), dirEntry.getValue());
- countEntries++;
- }
- }
-
- // 2. Seek the given key in dir table.
- if (countEntries < numEntries) {
- getDirectories(cacheDirMap, seekDirInDB, prefixPath,
- prefixKeyInDB, countEntries, numEntries, recursive,
- volumeName, bucketName, deletedKeySet);
- }
-
-
- return buildFinalStatusList(cacheFileMap, cacheDirMap, args,
clientAddress);
-
- }
-
- /**
- * Build final OzoneFileStatus list to be returned to client.
- * @throws IOException
- */
- private List<OzoneFileStatus> buildFinalStatusList(
- Map<String, OzoneFileStatus> cacheFileMap,
- Map<String, OzoneFileStatus> cacheDirMap, OmKeyArgs omKeyArgs,
- String clientAddress)
- throws IOException {
- List<OmKeyInfo> keyInfoList = new ArrayList<>();
- List<OzoneFileStatus> fileStatusFinalList = new ArrayList<>();
-
- for (OzoneFileStatus fileStatus : cacheFileMap.values()) {
- fileStatusFinalList.add(fileStatus);
- keyInfoList.add(fileStatus.getKeyInfo());
- }
-
- for (OzoneFileStatus fileStatus : cacheDirMap.values()) {
- fileStatusFinalList.add(fileStatus);
- }
-
- return sortPipelineInfo(fileStatusFinalList, keyInfoList,
- omKeyArgs, clientAddress);
- }
-
private List<OzoneFileStatus> buildFinalStatusList(
Collection<OzoneFileStatus> statusesCollection, OmKeyArgs omKeyArgs,
String clientAddress)
@@ -2029,255 +1704,6 @@ public class KeyManagerImpl implements KeyManager {
return fileStatusFinalList;
}
-
- /***
- * Build files, directories and marked for deleted entries from dir/file
- * cache.
- */
- @SuppressWarnings("parameternumber")
- private int getFilesAndDirsFromCacheWithBucket(String volumeName,
- String bucketName, Map<String, OzoneFileStatus> cacheFileMap,
- Map<String, OzoneFileStatus> tempCacheDirMap,
- Set<String> deletedKeySet, long prefixKeyInDB,
- String seekFileInDB, String seekDirInDB, String prefixPath,
- String startKey, int countEntries, long numEntries) throws IOException {
-
-
- // First under lock obtain both entries from dir/file cache and generate
- // entries marked for delete.
- BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName,
- bucketName);
- countEntries = listStatusFindFilesInTableCache(cacheFileMap,
metadataManager
- .getKeyTable(bucketLayout),
- prefixKeyInDB, seekFileInDB, prefixPath, startKey, countEntries,
- numEntries, deletedKeySet);
-
- // Don't count entries from dir cache, as first we need to return all
- // files and then directories.
- listStatusFindDirsInTableCache(tempCacheDirMap,
- metadataManager.getDirectoryTable(),
- prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName,
- bucketName, countEntries, numEntries, deletedKeySet);
-
- return countEntries;
- }
-
- @SuppressWarnings("parameternumber")
- protected int getDirectories(
- TreeMap<String, OzoneFileStatus> cacheKeyMap,
- String seekDirInDB, String prefixPath, long prefixKeyInDB,
- int countEntries, long numEntries, boolean recursive,
- String volumeName, String bucketName, Set<String> deletedKeySet)
- throws IOException {
-
- Table dirTable = metadataManager.getDirectoryTable();
- try (TableIterator<String,
- ? extends Table.KeyValue<String, OmDirectoryInfo>>
- iterator = dirTable.iterator()) {
-
- return getDirectoriesWithIterator(cacheKeyMap, seekDirInDB, prefixPath,
- prefixKeyInDB, countEntries, numEntries, recursive, volumeName,
- bucketName, deletedKeySet, iterator);
- }
- }
-
- @SuppressWarnings("parameternumber")
- private int getDirectoriesWithIterator(
- TreeMap<String, OzoneFileStatus> cacheKeyMap, String seekDirInDB,
- String prefixPath, long prefixKeyInDB, int countEntries, long numEntries,
- boolean recursive, String volumeName, String bucketName,
- Set<String> deletedKeySet,
- TableIterator<String,
- ? extends Table.KeyValue<String, OmDirectoryInfo>> iterator)
- throws IOException {
- iterator.seek(seekDirInDB);
-
- while (iterator.hasNext() && numEntries - countEntries > 0) {
- Table.KeyValue<String, OmDirectoryInfo> entry = iterator.next();
- OmDirectoryInfo dirInfo = entry.getValue();
- if (deletedKeySet.contains(dirInfo.getPath())) {
- iterator.next(); // move to next entry in the table
- // entry is actually deleted in cache and can exists in DB
- continue;
- }
- if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(),
- prefixKeyInDB)) {
- break;
- }
-
- // TODO: recursive list will be handled in HDDS-4360 jira.
- if (!recursive) {
- String dirName = OMFileRequest.getAbsolutePath(prefixPath,
- dirInfo.getName());
- OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(volumeName,
- bucketName, dirInfo, dirName);
- cacheKeyMap.put(dirName, new OzoneFileStatus(omKeyInfo, scmBlockSize,
- true));
- countEntries++;
- }
- }
-
- return countEntries;
- }
-
- @SuppressWarnings("parameternumber")
- private int getFilesFromDirectory(
- TreeMap<String, OzoneFileStatus> cacheKeyMap,
- String seekKeyInDB, String prefixKeyPath, long prefixKeyInDB,
- int countEntries, long numEntries, Set<String> deletedKeySet,
- TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- iterator)
- throws IOException {
- iterator.seek(seekKeyInDB);
- while (iterator.hasNext() && numEntries - countEntries > 0) {
- Table.KeyValue<String, OmKeyInfo> entry = iterator.next();
- OmKeyInfo keyInfo = entry.getValue();
- if (!entry.getKey().startsWith(seekKeyInDB)) {
- break;
- }
- final long volumeId = metadataManager.getVolumeId(
- keyInfo.getVolumeName());
- final long bucketId = metadataManager.getBucketId(
- keyInfo.getVolumeName(), keyInfo.getBucketName());
- final String keyPath = metadataManager.getOzonePathKey(volumeId,
- bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName());
- if (deletedKeySet.contains(keyPath)) {
- // move to next entry in the table
- // entry is actually deleted in cache and can exist in DB
- continue;
- }
- if (!OMFileRequest.isImmediateChild(keyInfo.getParentObjectID(),
- prefixKeyInDB)) {
- break;
- }
-
- keyInfo.setFileName(keyInfo.getKeyName());
- String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath,
- keyInfo.getKeyName());
- keyInfo.setKeyName(fullKeyPath);
- cacheKeyMap.put(fullKeyPath,
- new OzoneFileStatus(keyInfo, scmBlockSize, false));
- countEntries++;
- }
- return countEntries;
- }
-
- /**
- * Helper function for listStatus to find key in FileTableCache.
- */
- @SuppressWarnings("parameternumber")
- private int listStatusFindFilesInTableCache(
- Map<String, OzoneFileStatus> cacheKeyMap, Table<String,
- OmKeyInfo> keyTable, long prefixKeyInDB, String seekKeyInDB,
- String prefixKeyPath, String startKey, int countEntries,
- long numEntries, Set<String> deletedKeySet) {
-
- Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>>
- cacheIter = keyTable.cacheIterator();
-
- // TODO: recursive list will be handled in HDDS-4360 jira.
- while (cacheIter.hasNext() && numEntries - countEntries > 0) {
- Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry =
- cacheIter.next();
- String cacheKey = entry.getKey().getCacheKey();
- OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue();
- // cacheOmKeyInfo is null if an entry is deleted in cache
- if (cacheOmKeyInfo == null) {
- deletedKeySet.add(cacheKey);
- continue;
- }
-
- // make OmKeyInfo local copy to reset keyname to "fullKeyPath".
- // In DB keyName stores only the leaf node but the list
- // returning to the user should have full path.
- OmKeyInfo omKeyInfo = cacheOmKeyInfo.copyObject();
-
- omKeyInfo.setFileName(omKeyInfo.getKeyName());
- String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath,
- omKeyInfo.getKeyName());
- omKeyInfo.setKeyName(fullKeyPath);
-
- countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB,
- seekKeyInDB, startKey, countEntries, cacheKey, omKeyInfo,
- false);
- }
- return countEntries;
- }
-
- /**
- * Helper function for listStatus to find key in DirTableCache.
- */
- @SuppressWarnings("parameternumber")
- private int listStatusFindDirsInTableCache(
- Map<String, OzoneFileStatus> cacheKeyMap, Table<String,
- OmDirectoryInfo> dirTable, long prefixKeyInDB, String seekKeyInDB,
- String prefixKeyPath, String startKey, String volumeName,
- String bucketName, int countEntries, long numEntries,
- Set<String> deletedKeySet) {
-
- Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>>
- cacheIter = dirTable.cacheIterator();
- // seekKeyInDB will have two type of values.
- // 1. "1024/" -> startKey is null or empty
- // 2. "1024/b" -> startKey exists
- // TODO: recursive list will be handled in HDDS-4360 jira.
- while (cacheIter.hasNext() && numEntries - countEntries > 0) {
- Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>> entry =
- cacheIter.next();
- String cacheKey = entry.getKey().getCacheKey();
- OmDirectoryInfo cacheOmDirInfo = entry.getValue().getCacheValue();
- // cacheOmKeyInfo is null if an entry is deleted in cache
- if (cacheOmDirInfo == null) {
- deletedKeySet.add(cacheKey);
- continue;
- }
- String fullDirPath = OMFileRequest.getAbsolutePath(prefixKeyPath,
- cacheOmDirInfo.getName());
- OmKeyInfo cacheDirKeyInfo = OMFileRequest.getOmKeyInfo(volumeName,
- bucketName, cacheOmDirInfo, fullDirPath);
-
- countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB,
- seekKeyInDB, startKey, countEntries, cacheKey, cacheDirKeyInfo,
- true);
- }
- return countEntries;
- }
-
- @SuppressWarnings("parameternumber")
- private int addKeyInfoToFileStatusList(
- Map<String, OzoneFileStatus> cacheKeyMap,
- long prefixKeyInDB, String seekKeyInDB, String startKey,
- int countEntries, String cacheKey, OmKeyInfo cacheOmKeyInfo,
- boolean isDirectory) {
- // seekKeyInDB will have two type of values.
- // 1. "1024/" -> startKey is null or empty
- // 2. "1024/b" -> startKey exists
- if (StringUtils.isBlank(startKey)) {
- // startKey is null or empty, then the seekKeyInDB="1024/"
- if (cacheKey.startsWith(seekKeyInDB)) {
- OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo,
- scmBlockSize, isDirectory);
- cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus);
- countEntries++;
- }
- } else {
- // startKey not empty, then the seekKeyInDB="1024/b" and
- // seekKeyInDBWithOnlyParentID = "1024/". This is to avoid case of
- // parentID with "102444" cache entries.
- // Here, it has to list all the keys after "1024/b" and requires >=0
- // string comparison.
- String seekKeyInDBWithOnlyParentID = prefixKeyInDB + OM_KEY_PREFIX;
- if (cacheKey.startsWith(seekKeyInDBWithOnlyParentID) &&
- cacheKey.compareTo(seekKeyInDB) >= 0) {
- OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo,
- scmBlockSize, isDirectory);
- cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus);
- countEntries++;
- }
- }
- return countEntries;
- }
-
private String getNextGreaterString(String volumeName, String bucketName,
String keyPrefix) throws IOException {
// Increment the last character of the string and return the new ozone key.
@@ -2291,61 +1717,6 @@ public class KeyManagerImpl implements KeyManager {
return metadataManager.getOzoneKey(volumeName, bucketName, nextPrefix);
}
- /**
- * Verify that none of the parent path exists as file in the filesystem.
- *
- * @param volumeName Volume name
- * @param bucketName Bucket name
- * @param path Directory path. This is the absolute path of the
- * directory for the ozone filesystem.
- * @param directoryMustExist throws exception if true and given path does not
- * exist as directory
- * @return OzoneFileStatus of the first directory found in path in reverse
- * order
- * @throws OMException if ancestor exists as file in the filesystem
- * if directoryMustExist flag is true and parent does
- * not exist
- * if bucket does not exist
- * @throws IOException if there is error in the db
- * invalid arguments
- */
- private OzoneFileStatus verifyNoFilesInPath(String volumeName,
- String bucketName, Path path, boolean directoryMustExist)
- throws IOException {
- OmKeyArgs.Builder argsBuilder = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName);
- while (path != null) {
- String keyName = path.toString();
- try {
- OzoneFileStatus fileStatus =
- getFileStatus(argsBuilder.setKeyName(keyName).build());
- if (fileStatus.isFile()) {
- LOG.error("Unable to create directory (File already exists): "
- + "volume: {} bucket: {} key: {}", volumeName, bucketName,
- keyName);
- throw new OMException(
- "Unable to create directory at : volume: " + volumeName
- + "bucket: " + bucketName + "key: " + keyName,
- ResultCodes.FILE_ALREADY_EXISTS);
- } else if (fileStatus.isDirectory()) {
- return fileStatus;
- }
- } catch (OMException ex) {
- if (ex.getResult() != FILE_NOT_FOUND) {
- throw ex;
- } else if (ex.getResult() == FILE_NOT_FOUND) {
- if (directoryMustExist) {
- throw new OMException("Parent directory does not exist",
- ex.getCause(), DIRECTORY_NOT_FOUND);
- }
- }
- }
- path = path.getParent();
- }
- return null;
- }
-
private FileEncryptionInfo getFileEncryptionInfo(OmBucketInfo bucketInfo)
throws IOException {
FileEncryptionInfo encInfo = null;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 388fc2910f..8d6ae64ddf 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -724,7 +724,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
prefixManager = new PrefixManagerImpl(metadataManager, isRatisEnabled);
perfMetrics = OMPerformanceMetrics.register();
keyManager = new KeyManagerImpl(this, scmClient, configuration,
- omStorage.getOmId(), perfMetrics);
+ perfMetrics);
if (withNewSnapshot) {
Integer layoutVersionInDB = getLayoutVersionInDB();
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index a4906d8f2e..2a6f17ad61 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -148,7 +148,7 @@ public class TestOMKeyRequest {
scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class);
metrics = Mockito.mock(OMPerformanceMetrics.class);
keyManager = new KeyManagerImpl(ozoneManager, scmClient,
ozoneConfiguration,
- "", metrics);
+ metrics);
when(ozoneManager.getScmClient()).thenReturn(scmClient);
when(ozoneManager.getBlockTokenSecretManager())
.thenReturn(ozoneBlockTokenSecretManager);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
