This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 94fb6f33a6eb696db7016ecc9fb027df0c79aa3b Merge: 0323530 6e278ba Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Tue Jun 8 17:31:28 2021 +0530 Merge remote-tracking branch 'origin/master' into HDDS-2939 .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 + .../common/src/main/resources/ozone-default.xml | 10 +- .../container/common/report/ReportPublisher.java | 8 +- .../common/statemachine/StateContext.java | 131 ++++++++++++------- .../states/endpoint/HeartbeatEndpointTask.java | 5 +- .../transport/server/ratis/XceiverServerRatis.java | 3 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 2 +- .../common/report/TestReportPublisher.java | 2 +- .../common/statemachine/TestStateContext.java | 140 ++++++++++----------- .../states/endpoint/TestHeartbeatEndpointTask.java | 14 ++- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 10 ++ .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 18 ++- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 17 +-- ...OzoneManagerProtocolClientSideTranslatorPB.java | 4 + .../commandhandler/TestBlockDeletion.java | 3 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 101 +++++++++++++++ .../src/main/proto/OmClientProtocol.proto | 3 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 59 +++++++-- .../protocolPB/OzoneManagerRequestHandler.java | 2 + .../ozone/recon/fsck/ContainerHealthStatus.java | 6 + .../ozone/recon/fsck/ContainerHealthTask.java | 51 +++++++- .../scm/ReconStorageContainerManagerFacade.java | 5 + .../ozone/recon/fsck/TestContainerHealthTask.java | 88 +++++++++++++ 23 files changed, 530 insertions(+), 157 deletions(-) diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml index 2dea94e,3a9133f..81d174c --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@@ -2864,35 -2863,10 +2863,42 @@@ </property> <property> + <name>ozone.client.key.latest.version.location</name> + <tag>OZONE, CLIENT</tag> + <value>true</value> + <description>Ozone client gets the latest version location. + </description> + </property> ++ ++ <property> + <name>ozone.om.metadata.layout</name> + <tag>OZONE, OM</tag> + <value>SIMPLE</value> + <description> + This property is used to define the metadata layout of file system + paths. If it is configured as PREFIX in combination with + ozone.om.enable.filesystem.paths to true then this allows to perform + atomic rename and delete of any directory at any level in the namespace. + Defaulting to SIMPLE. Supported values: SIMPLE and PREFIX. + </description> + </property> + <property> + <name>ozone.directory.deleting.service.interval</name> + <value>1m</value> + <tag>OZONE, PERFORMANCE, OM</tag> + <description>Time interval of the directory deleting service. It runs on OM + periodically and cleanup orphan directory and its sub-tree. For every + orphan directory it deletes the sub-path tree structure(dirs/files). It + sends sub-files to KeyDeletingService to deletes its blocks. Unit could + be defined with postfix (ns,ms,s,m,h,d) + </description> + </property> + <property> + <name>ozone.path.deleting.limit.per.task</name> + <value>10000</value> + <tag>OZONE, PERFORMANCE, OM</tag> + <description>A maximum number of paths(dirs/files) to be deleted by + directory deleting service per time interval. + </description> + </property> - </configuration> diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index 9a0bf6b,868da8b..7b298d4 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@@ -46,7 -46,7 +46,8 @@@ public final class OmKeyArgs implement private boolean refreshPipeline; private boolean sortDatanodesInPipeline; private List<OzoneAcl> acls; + private boolean latestVersionLocation; + private boolean recursive; @SuppressWarnings("parameternumber") private OmKeyArgs(String volumeName, String bucketName, String keyName, @@@ -54,7 -54,8 +55,8 @@@ List<OmKeyLocationInfo> locationInfoList, boolean isMultipart, String uploadID, int partNumber, Map<String, String> metadataMap, boolean refreshPipeline, - List<OzoneAcl> acls, boolean sortDatanode, boolean recursive) { + List<OzoneAcl> acls, boolean sortDatanode, - boolean latestVersionLocation) { ++ boolean latestVersionLocation, boolean recursive) { this.volumeName = volumeName; this.bucketName = bucketName; this.keyName = keyName; @@@ -68,7 -69,7 +70,8 @@@ this.refreshPipeline = refreshPipeline; this.acls = acls; this.sortDatanodesInPipeline = sortDatanode; + this.latestVersionLocation = latestVersionLocation; + this.recursive = recursive; } public boolean getIsMultipartKey() { @@@ -135,10 -136,10 +138,14 @@@ return sortDatanodesInPipeline; } + public boolean getLatestVersionLocation() { + return latestVersionLocation; + } + + public boolean isRecursive() { + return recursive; + } + @Override public Map<String, String> toAuditMap() { Map<String, String> auditMap = new LinkedHashMap<>(); @@@ -193,8 -195,8 +201,9 @@@ private Map<String, String> metadata = new HashMap<>(); private boolean refreshPipeline; private boolean sortDatanodesInPipeline; + private boolean latestVersionLocation; private List<OzoneAcl> acls; + private boolean recursive; public Builder setVolumeName(String volume) { this.volumeName = volume; @@@ -266,17 -268,17 +275,22 @@@ return this; } + public Builder setLatestVersionLocation(boolean latest) { + this.latestVersionLocation = latest; + return this; + } + + public Builder setRecursive(boolean isRecursive) { + this.recursive = isRecursive; + return this; + } + public OmKeyArgs build() { return new OmKeyArgs(volumeName, bucketName, keyName, dataSize, replicationConfig, locationInfoList, isMultipartKey, multipartUploadID, multipartUploadPartNumber, metadata, refreshPipeline, acls, - sortDatanodesInPipeline, recursive); - sortDatanodesInPipeline, latestVersionLocation); ++ sortDatanodesInPipeline, latestVersionLocation, recursive); } } diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 295d2ad,846ed9a..650e9bb --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -741,9 -735,7 +741,10 @@@ message KeyArgs // This will be set by leader OM in HA and update the original request. optional FileEncryptionInfoProto fileEncryptionInfo = 15; + optional bool latestVersionLocation = 16; + + // This will be set when user performs delete directory recursively. - optional bool recursive = 16; ++ optional bool recursive = 17; } message KeyLocation { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index ac65aab,6c28ab0..bcb0ccf --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@@ -673,9 -665,13 +673,13 @@@ public class KeyManagerImpl implements LOG.debug("volume:{} bucket:{} Key:{} not found", volumeName, bucketName, keyName); } - throw new OMException("Key not found", KEY_NOT_FOUND); + throw new OMException("Key:" + keyName + " not found", KEY_NOT_FOUND); } + if (args.getLatestVersionLocation()) { + slimLocationVersion(value); + } + // add block token for read. addBlockToken4Read(value); @@@ -1901,12 -1765,9 +1910,14 @@@ String bucketName = args.getBucketName(); String keyName = args.getKeyName(); + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { + return getOzoneFileStatusFSO(volumeName, bucketName, keyName, - args.getSortDatanodes(), clientAddress, false); ++ args.getSortDatanodes(), clientAddress, ++ args.getLatestVersionLocation(), false); + } return getOzoneFileStatus(volumeName, bucketName, keyName, - args.getRefreshPipeline(), args.getSortDatanodes(), clientAddress); + args.getRefreshPipeline(), args.getSortDatanodes(), + args.getLatestVersionLocation(), clientAddress); } private OzoneFileStatus getOzoneFileStatus(String volumeName, @@@ -1969,65 -1834,6 +1984,67 @@@ FILE_NOT_FOUND); } + + private OzoneFileStatus getOzoneFileStatusFSO(String volumeName, + String bucketName, String keyName, boolean sortDatanodes, - String clientAddress, boolean skipFileNotFoundError) throws IOException { ++ String clientAddress, boolean latestLocationVersion, ++ boolean skipFileNotFoundError) throws IOException { + OzoneFileStatus fileStatus = null; + metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, + bucketName); + try { + // Check if this is the root of the filesystem. + if (keyName.length() == 0) { + OMFileRequest.validateBucket(metadataManager, volumeName, bucketName); + return new OzoneFileStatus(); + } + + fileStatus = OMFileRequest.getOMKeyInfoIfExists(metadataManager, + volumeName, bucketName, keyName, scmBlockSize); + + } finally { + metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, + bucketName); + } + + if (fileStatus != null) { + // if the key is a file then do refresh pipeline info in OM by asking SCM + if (fileStatus.isFile()) { - + OmKeyInfo fileKeyInfo = fileStatus.getKeyInfo(); - ++ if (latestLocationVersion) { ++ slimLocationVersion(fileKeyInfo); ++ } + // refreshPipeline flag check has been removed as part of + // https://issues.apache.org/jira/browse/HDDS-3658. + // Please refer this jira for more details. + refresh(fileKeyInfo); + + if (sortDatanodes) { + sortDatanodes(clientAddress, fileKeyInfo); + } + return new OzoneFileStatus(fileKeyInfo, scmBlockSize, false); + } else { + return fileStatus; + } + } + + // Key not found. + if (LOG.isDebugEnabled()) { + LOG.debug("Unable to get file status for the key: volume: {}, bucket:" + + " {}, key: {}, with error: No such file exists.", + volumeName, bucketName, keyName); + } + + // don't throw exception if this flag is true. + if (skipFileNotFoundError) { + return fileStatus; + } + + throw new OMException("Unable to get file status: volume: " + + volumeName + " bucket: " + bucketName + " key: " + keyName, + FILE_NOT_FOUND); + } + /** * Ozone FS api to create a directory. Parent directories if do not exist * are created for the input directory. @@@ -2169,16 -1975,10 +2186,17 @@@ String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); String keyName = args.getKeyName(); - OzoneFileStatus fileStatus = getOzoneFileStatus(volumeName, bucketName, - keyName, args.getRefreshPipeline(), args.getSortDatanodes(), - args.getLatestVersionLocation(), clientAddress); - //if key is not of type file or if key is not found we throw an exception + OzoneFileStatus fileStatus; + if (OzoneManagerRatisUtils.isBucketFSOptimized()) { + fileStatus = getOzoneFileStatusFSO(volumeName, bucketName, keyName, - args.getSortDatanodes(), clientAddress, false); ++ args.getSortDatanodes(), clientAddress, ++ args.getLatestVersionLocation(),false); + } else { + fileStatus = getOzoneFileStatus(volumeName, bucketName, + keyName, args.getRefreshPipeline(), args.getSortDatanodes(), - clientAddress); ++ args.getLatestVersionLocation(), clientAddress); + } + //if key is not of type file or if key is not found we throw an exception if (fileStatus.isFile()) { // add block token for read. addBlockToken4Read(fileStatus.getKeyInfo()); @@@ -2418,389 -2216,6 +2439,392 @@@ return fileStatusList; } + @SuppressWarnings("methodlength") + public List<OzoneFileStatus> listStatusFSO(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, String clientAddress) + throws IOException { + Preconditions.checkNotNull(args, "Key args can not be null"); + + // unsorted OMKeyInfo list contains combine results from TableCache and DB. + List<OzoneFileStatus> fileStatusFinalList = new ArrayList<>(); + + if (numEntries <= 0) { + return fileStatusFinalList; + } + + /** + * A map sorted by OmKey to combine results from TableCache and DB for + * each entity - Dir & File. + * + * Two separate maps are required because the order of seek -> (1)Seek + * files in fileTable (2)Seek dirs in dirTable. + * + * StartKey should be added to the final listStatuses, so if we combine + * files and dirs into a single map then directory with lower precedence + * will appear at the top of the list even if the startKey is given as + * fileName. + * + * For example, startKey="a/file1". As per the seek order, first fetches + * all the files and then it will start seeking all the directories. + * Assume a directory name exists "a/b". With one map, the sorted list will + * be ["a/b", "a/file1"]. But the expected list is: ["a/file1", "a/b"], + * startKey element should always be at the top of the listStatuses. + */ + TreeMap<String, OzoneFileStatus> cacheFileMap = new TreeMap<>(); + TreeMap<String, OzoneFileStatus> cacheDirMap = new TreeMap<>(); + + String volumeName = args.getVolumeName(); + String bucketName = args.getBucketName(); + String keyName = args.getKeyName(); + String seekFileInDB; + String seekDirInDB; + long prefixKeyInDB; + String prefixPath = keyName; + int countEntries = 0; + + // TODO: recursive flag=true will be handled in HDDS-4360 jira. + metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, + bucketName); + try { + if (Strings.isNullOrEmpty(startKey)) { + OzoneFileStatus fileStatus = getFileStatus(args, clientAddress); + if (fileStatus.isFile()) { + return Collections.singletonList(fileStatus); + } + + // Not required to search in DeletedTable because all the deleted + // keys will be marked directly in dirTable or in keyTable by + // breaking the pointer to its sub-dirs and sub-files. So, there is no + // issue of inconsistency. + + /* + * keyName is a directory. + * Say, "/a" is the dir name and its objectID is 1024, then seek + * will be doing with "1024/" to get all immediate descendants. + */ + if (fileStatus.getKeyInfo() != null) { + prefixKeyInDB = fileStatus.getKeyInfo().getObjectID(); + } else { + // list root directory. + String bucketKey = metadataManager.getBucketKey(volumeName, + bucketName); + OmBucketInfo omBucketInfo = + metadataManager.getBucketTable().get(bucketKey); + prefixKeyInDB = omBucketInfo.getObjectID(); + } + seekFileInDB = metadataManager.getOzonePathKey(prefixKeyInDB, ""); + seekDirInDB = metadataManager.getOzonePathKey(prefixKeyInDB, ""); + + // Order of seek -> (1)Seek files in fileTable (2)Seek dirs in dirTable + // 1. Seek the given key in key table. + countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, + prefixPath, prefixKeyInDB, startKey, countEntries, numEntries); + // 2. Seek the given key in dir table. + getDirectories(cacheDirMap, seekDirInDB, prefixPath, prefixKeyInDB, + startKey, countEntries, numEntries, volumeName, bucketName, + recursive); + } else { + /* + * startKey will be used in iterator seek and sets the beginning point + * for key traversal. + * keyName will be used as parentID where the user has requested to + * list the keys from. + * + * When recursive flag=false, parentID won't change between two pages. + * For example: OM has a namespace like, + * /a/1...1M files and /a/b/1...1M files. + * /a/1...1M directories and /a/b/1...1M directories. + * Listing "/a", will always have the parentID as "a" irrespective of + * the startKey value. + */ + + // Check startKey is an immediate child of keyName. For example, + // keyName=/a/ and expected startKey=/a/b. startKey can't be /xyz/b. + if (StringUtils.isNotBlank(keyName) && + !OzoneFSUtils.isImmediateChild(keyName, startKey)) { + if (LOG.isDebugEnabled()) { + LOG.debug("StartKey {} is not an immediate child of keyName {}. " + + "Returns empty list", startKey, keyName); + } + return Collections.emptyList(); + } + + // assign startKeyPath if prefixPath is empty string. + if (StringUtils.isBlank(prefixPath)) { + prefixPath = OzoneFSUtils.getParentDir(startKey); + } + + OzoneFileStatus fileStatusInfo = getOzoneFileStatusFSO(volumeName, - bucketName, startKey, false, null, true); ++ bucketName, startKey, false, null, ++ args.getLatestVersionLocation(),true); + + if (fileStatusInfo != null) { + prefixKeyInDB = fileStatusInfo.getKeyInfo().getParentObjectID(); + if(fileStatusInfo.isDirectory()){ + seekDirInDB = metadataManager.getOzonePathKey(prefixKeyInDB, + fileStatusInfo.getKeyInfo().getFileName()); + + // Order of seek -> (1) Seek dirs only in dirTable. In OM, always + // the order of search is, first seek into fileTable and then + // dirTable. So, its not required to search again in the fileTable. + + // Seek the given key in dirTable. + getDirectories(cacheDirMap, seekDirInDB, prefixPath, + prefixKeyInDB, startKey, countEntries, numEntries, + volumeName, bucketName, recursive); + } else { + seekFileInDB = metadataManager.getOzonePathKey(prefixKeyInDB, + fileStatusInfo.getKeyInfo().getFileName()); + // begins from the first sub-dir under the parent dir + seekDirInDB = metadataManager.getOzonePathKey(prefixKeyInDB, ""); + + // 1. Seek the given key in key table. + countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, + prefixPath, prefixKeyInDB, startKey, countEntries, + numEntries); + // 2. Seek the given key in dir table. + getDirectories(cacheDirMap, seekDirInDB, prefixPath, + prefixKeyInDB, startKey, countEntries, numEntries, + volumeName, bucketName, recursive); + } + } else { + // TODO: HDDS-4364: startKey can be a non-existed key + if (LOG.isDebugEnabled()) { + LOG.debug("StartKey {} is a non-existed key and returning empty " + + "list", startKey); + } + return Collections.emptyList(); + } + } + } finally { + metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, + bucketName); + } + + List<OmKeyInfo> keyInfoList = new ArrayList<>(); + for (OzoneFileStatus fileStatus : cacheFileMap.values()) { + fileStatusFinalList.add(fileStatus); + keyInfoList.add(fileStatus.getKeyInfo()); + } + for (OzoneFileStatus fileStatus : cacheDirMap.values()) { + fileStatusFinalList.add(fileStatus); + } - ++ if (args.getLatestVersionLocation()) { ++ slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0])); ++ } + // refreshPipeline flag check has been removed as part of + // https://issues.apache.org/jira/browse/HDDS-3658. + // Please refer this jira for more details. + refreshPipeline(keyInfoList); + if (args.getSortDatanodes()) { + sortDatanodes(clientAddress, keyInfoList.toArray(new OmKeyInfo[0])); + } + return fileStatusFinalList; + } + + @SuppressWarnings("parameternumber") + protected int getDirectories( + TreeMap<String, OzoneFileStatus> cacheKeyMap, + String seekDirInDB, String prefixPath, long prefixKeyInDB, + String startKey, int countEntries, long numEntries, String volumeName, + String bucketName, boolean recursive) throws IOException { + + // A set to keep track of keys deleted in cache but not flushed to DB. + Set<String> deletedKeySet = new TreeSet<>(); + + Table dirTable = metadataManager.getDirectoryTable(); + countEntries = listStatusFindDirsInTableCache(cacheKeyMap, dirTable, + prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, + bucketName, countEntries, numEntries, deletedKeySet); + TableIterator<String, ? extends Table.KeyValue<String, OmDirectoryInfo>> + iterator = dirTable.iterator(); + + iterator.seek(seekDirInDB); + + while (iterator.hasNext() && numEntries - countEntries > 0) { + OmDirectoryInfo dirInfo = iterator.value().getValue(); + if (deletedKeySet.contains(dirInfo.getPath())) { + iterator.next(); // move to next entry in the table + // entry is actually deleted in cache and can exists in DB + continue; + } + if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(), + prefixKeyInDB)) { + break; + } + + // TODO: recursive list will be handled in HDDS-4360 jira. + if (!recursive) { + String dirName = OMFileRequest.getAbsolutePath(prefixPath, + dirInfo.getName()); + OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(volumeName, + bucketName, dirInfo, dirName); + cacheKeyMap.put(dirName, new OzoneFileStatus(omKeyInfo, scmBlockSize, + true)); + countEntries++; + } + // move to next entry in the DirTable + iterator.next(); + } + + return countEntries; + } + + private int getFilesFromDirectory( + TreeMap<String, OzoneFileStatus> cacheKeyMap, + String seekKeyInDB, String prefixKeyPath, long prefixKeyInDB, + String startKey, int countEntries, long numEntries) throws IOException { + + // A set to keep track of keys deleted in cache but not flushed to DB. + Set<String> deletedKeySet = new TreeSet<>(); + + Table<String, OmKeyInfo> keyTable = metadataManager.getKeyTable(); + countEntries = listStatusFindFilesInTableCache(cacheKeyMap, keyTable, + prefixKeyInDB, seekKeyInDB, prefixKeyPath, startKey, + countEntries, numEntries, deletedKeySet); + TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> + iterator = keyTable.iterator(); + iterator.seek(seekKeyInDB); + while (iterator.hasNext() && numEntries - countEntries > 0) { + OmKeyInfo keyInfo = iterator.value().getValue(); + if (deletedKeySet.contains(keyInfo.getPath())) { + iterator.next(); // move to next entry in the table + // entry is actually deleted in cache and can exists in DB + continue; + } + if (!OMFileRequest.isImmediateChild(keyInfo.getParentObjectID(), + prefixKeyInDB)) { + break; + } + + keyInfo.setFileName(keyInfo.getKeyName()); + String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath, + keyInfo.getKeyName()); + keyInfo.setKeyName(fullKeyPath); + cacheKeyMap.put(fullKeyPath, + new OzoneFileStatus(keyInfo, scmBlockSize, false)); + countEntries++; + iterator.next(); // move to next entry in the table + } + return countEntries; + } + + /** + * Helper function for listStatus to find key in FileTableCache. + */ + @SuppressWarnings("parameternumber") + private int listStatusFindFilesInTableCache( + TreeMap<String, OzoneFileStatus> cacheKeyMap, Table<String, + OmKeyInfo> keyTable, long prefixKeyInDB, String seekKeyInDB, + String prefixKeyPath, String startKey, int countEntries, + long numEntries, Set<String> deletedKeySet) { + + Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> + cacheIter = keyTable.cacheIterator(); + + // TODO: recursive list will be handled in HDDS-4360 jira. + while (cacheIter.hasNext() && numEntries - countEntries > 0) { + Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry = + cacheIter.next(); + String cacheKey = entry.getKey().getCacheKey(); + OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue(); + // cacheOmKeyInfo is null if an entry is deleted in cache + if(cacheOmKeyInfo == null){ + deletedKeySet.add(cacheKey); + continue; + } + + // make OmKeyInfo local copy to reset keyname to "fullKeyPath". + // In DB keyName stores only the leaf node but the list + // returning to the user should have full path. + OmKeyInfo omKeyInfo = cacheOmKeyInfo.copyObject(); + + omKeyInfo.setFileName(omKeyInfo.getKeyName()); + String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath, + omKeyInfo.getKeyName()); + omKeyInfo.setKeyName(fullKeyPath); + + countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB, + seekKeyInDB, startKey, countEntries, cacheKey, omKeyInfo, + false); + } + return countEntries; + } + + /** + * Helper function for listStatus to find key in DirTableCache. + */ + @SuppressWarnings("parameternumber") + private int listStatusFindDirsInTableCache( + TreeMap<String, OzoneFileStatus> cacheKeyMap, Table<String, + OmDirectoryInfo> dirTable, long prefixKeyInDB, String seekKeyInDB, + String prefixKeyPath, String startKey, String volumeName, + String bucketName, int countEntries, long numEntries, + Set<String> deletedKeySet) { + + Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>> + cacheIter = dirTable.cacheIterator(); + // seekKeyInDB will have two type of values. + // 1. "1024/" -> startKey is null or empty + // 2. "1024/b" -> startKey exists + // TODO: recursive list will be handled in HDDS-4360 jira. + while (cacheIter.hasNext() && numEntries - countEntries > 0) { + Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>> entry = + cacheIter.next(); + String cacheKey = entry.getKey().getCacheKey(); + OmDirectoryInfo cacheOmDirInfo = entry.getValue().getCacheValue(); + // cacheOmKeyInfo is null if an entry is deleted in cache + if(cacheOmDirInfo == null){ + deletedKeySet.add(cacheKey); + continue; + } + String fullDirPath = OMFileRequest.getAbsolutePath(prefixKeyPath, + cacheOmDirInfo.getName()); + OmKeyInfo cacheDirKeyInfo = OMFileRequest.getOmKeyInfo(volumeName, + bucketName, cacheOmDirInfo, fullDirPath); + + countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB, + seekKeyInDB, startKey, countEntries, cacheKey, cacheDirKeyInfo, + true); + } + return countEntries; + } + + @SuppressWarnings("parameternumber") + private int addKeyInfoToFileStatusList( + TreeMap<String, OzoneFileStatus> cacheKeyMap, + long prefixKeyInDB, String seekKeyInDB, String startKey, + int countEntries, String cacheKey, OmKeyInfo cacheOmKeyInfo, + boolean isDirectory) { + // seekKeyInDB will have two type of values. + // 1. "1024/" -> startKey is null or empty + // 2. "1024/b" -> startKey exists + if (StringUtils.isBlank(startKey)) { + // startKey is null or empty, then the seekKeyInDB="1024/" + if (cacheKey.startsWith(seekKeyInDB)) { + OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo, + scmBlockSize, isDirectory); + cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus); + countEntries++; + } + } else { + // startKey not empty, then the seekKeyInDB="1024/b" and + // seekKeyInDBWithOnlyParentID = "1024/". This is to avoid case of + // parentID with "102444" cache entries. + // Here, it has to list all the keys after "1024/b" and requires >=0 + // string comparison. + String seekKeyInDBWithOnlyParentID = prefixKeyInDB + OM_KEY_PREFIX; + if (cacheKey.startsWith(seekKeyInDBWithOnlyParentID) && + cacheKey.compareTo(seekKeyInDB) >= 0) { + OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo, + scmBlockSize, isDirectory); + cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus); + countEntries++; + } + } + return countEntries; + } + private String getNextGreaterString(String volumeName, String bucketName, String keyPrefix) throws IOException { // Increment the last character of the string and return the new ozone key. @@@ -2953,88 -2368,22 +2977,105 @@@ } return nodeSet; } - + private void slimLocationVersion(OmKeyInfo... keyInfos) { + if (keyInfos != null) { + for (OmKeyInfo keyInfo : keyInfos) { + OmKeyLocationInfoGroup key = keyInfo.getLatestVersionLocations(); + if (key == null) { + LOG.warn("No location version for key {}", keyInfo); + continue; + } + int keyLocationVersionLength = keyInfo.getKeyLocationVersions().size(); + if (keyLocationVersionLength <= 1) { + continue; + } + keyInfo.setKeyLocationVersions(keyInfo.getKeyLocationVersions() - .subList(keyLocationVersionLength-1, keyLocationVersionLength)); ++ .subList(keyLocationVersionLength - 1, keyLocationVersionLength)); + } + } + } + + @Override + public OmKeyInfo getPendingDeletionDir() throws IOException { + OmKeyInfo omKeyInfo = null; + try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> + deletedDirItr = metadataManager.getDeletedDirTable().iterator()) { + if (deletedDirItr.hasNext()) { + Table.KeyValue<String, OmKeyInfo> keyValue = deletedDirItr.next(); + if (keyValue != null) { + omKeyInfo = keyValue.getValue(); + } + } + } + return omKeyInfo; + } + + @Override + public List<OmKeyInfo> getPendingDeletionSubDirs(OmKeyInfo parentInfo, + long numEntries) throws IOException { + List<OmKeyInfo> directories = new ArrayList<>(); + String seekDirInDB = metadataManager.getOzonePathKey( + parentInfo.getObjectID(), ""); + long countEntries = 0; + + Table dirTable = metadataManager.getDirectoryTable(); + TableIterator<String, ? extends Table.KeyValue<String, OmDirectoryInfo>> + iterator = dirTable.iterator(); + + iterator.seek(seekDirInDB); + + while (iterator.hasNext() && numEntries - countEntries > 0) { + OmDirectoryInfo dirInfo = iterator.value().getValue(); + if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(), + parentInfo.getObjectID())) { + break; + } + String dirName = OMFileRequest.getAbsolutePath(parentInfo.getKeyName(), + dirInfo.getName()); + OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo( + parentInfo.getVolumeName(), parentInfo.getBucketName(), dirInfo, + dirName); + directories.add(omKeyInfo); + countEntries++; + + // move to next entry in the DirTable + iterator.next(); + } + + return directories; + } + + @Override + public List<OmKeyInfo> getPendingDeletionSubFiles(OmKeyInfo parentInfo, + long numEntries) throws IOException { + List<OmKeyInfo> files = new ArrayList<>(); + String seekFileInDB = metadataManager.getOzonePathKey( + parentInfo.getObjectID(), ""); + long countEntries = 0; + + Table fileTable = metadataManager.getKeyTable(); + TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> + iterator = fileTable.iterator(); + + iterator.seek(seekFileInDB); + + while (iterator.hasNext() && numEntries - countEntries > 0) { + OmKeyInfo fileInfo = iterator.value().getValue(); + if (!OMFileRequest.isImmediateChild(fileInfo.getParentObjectID(), + parentInfo.getObjectID())) { + break; + } + fileInfo.setFileName(fileInfo.getKeyName()); + String fullKeyPath = OMFileRequest.getAbsolutePath( + parentInfo.getKeyName(), fileInfo.getKeyName()); + fileInfo.setKeyName(fullKeyPath); + + files.add(fileInfo); + countEntries++; + // move to next entry in the KeyTable + iterator.next(); + } + + return files; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
