xichen01 commented on code in PR #9042:
URL: https://github.com/apache/ozone/pull/9042#discussion_r2471670747
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java:
##########
@@ -366,171 +371,302 @@ private void evaluateFSOBucket(OmVolumeArgs volume,
OmBucketInfo bucket, String
// skip this rule if some directory doesn't exist for this rule's
prefix
continue;
}
- // use last directory's object ID to iterate the keys
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- StringBuffer directoryPath = new StringBuffer();
+ StringBuffer lastDirPath = new StringBuffer();
+ OmDirectoryInfo lastDir = null;
if (!dirList.isEmpty()) {
- OmDirectoryInfo lastDir = dirList.get(dirList.size() - 1);
- prefix += lastDir.getObjectID();
- for (OmDirectoryInfo dir : dirList) {
- directoryPath.append(dir.getName()).append(OM_KEY_PREFIX);
+ lastDir = dirList.get(dirList.size() - 1);
+ for (int i = 0; i < dirList.size(); i++) {
+ lastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 1) {
+ lastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ if (lastDirPath.toString().startsWith(TRASH_PREFIX)) {
+ LOG.info("Skip evaluate trash directory {}", lastDirPath);
+ } else {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable,
lastDirPath.toString(), lastDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
}
- if ((directoryPath.toString().equals(rule.getEffectivePrefix()) ||
- directoryPath.toString().equals(rule.getEffectivePrefix() +
OM_KEY_PREFIX))
- && rule.match(lastDir, directoryPath.toString())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+
+ if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+ // if the prefix doesn't end with "/", then also search and
evaluate the directory itself
+ // for example, "dir1/dir2" matches both directory "dir1/dir2" and
"dir1/dir22"
+ // or "dir1" matches both directory "dir1" and "dir11"
+ long objID;
+ String objPrefix;
+ String objPath;
+ if (dirList.size() > 1) {
+ OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2);
+ objID = secondLastDir.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + secondLastDir.getObjectID();
+ StringBuffer secondLastDirPath = new StringBuffer();
+ for (int i = 0; i < dirList.size() - 1; i++) {
+ secondLastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 2) {
+ secondLastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ objPath = secondLastDirPath.toString();
+ } else {
+ objID = bucket.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + bucket.getObjectID();
+ objPath = "";
+ }
+ try {
+ SubDirectorySummary subDirSummary = getSubDirectory(objID,
objPrefix, omMetadataManager);
+ for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ String subDirPath = objPath.isEmpty() ? subDir.getName() :
objPath + OM_KEY_PREFIX + subDir.getName();
+ if (!subDir.getName().equals(TRASH_PREFIX) &&
subDirPath.startsWith(rule.getEffectivePrefix()) &&
+ (lastDir == null || subDir.getObjectID() !=
lastDir.getObjectID())) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(),
keyTable, subDirPath, subDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ }
+ }
+ } catch (IOException e) {
+ // log failure and continue the process
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
objPrefix,
+ bucket.getVolumeName(), bucket.getBucketName(), e);
+ return;
}
- expiredDirList.add(directoryPath.toString(), 0,
lastDir.getUpdateID());
}
+ } else {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
}
+ }
- LOG.info("Prefix {} for {}", prefix, bucketKey);
- evaluateKeyTable(keyTable, prefix, directoryPath.toString(), rule,
expiredKeyList, bucket);
- evaluateDirTable(directoryInfoTable, prefix, directoryPath.toString(),
rule,
- expiredDirList, bucket);
+ if (!noPrefixRuleList.isEmpty()) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ noPrefixRuleList, expiredKeyList, expiredDirList);
}
+ }
- for (OmLCRule rule : nonDirectoryStylePrefixRuleList) {
- // find the directory for the prefix, it may not exist
- OmDirectoryInfo dirInfo = getDirectory(volume, bucket,
rule.getEffectivePrefix(), bucketKey);
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- if (dirInfo != null) {
- if (!dirInfo.getName().equals(TRASH_PREFIX)) {
- prefix += dirInfo.getObjectID();
- if (dirInfo.getName().equals(rule.getEffectivePrefix()) &&
rule.match(dirInfo, dirInfo.getName())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+ @SuppressWarnings({"checkstyle:parameternumber",
"checkstyle:MethodLength"})
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId,
Table<String, OmKeyInfo> keyTable,
+ String directoryPath, @Nullable OmDirectoryInfo dir, List<OmLCRule>
ruleList, LimitedExpiredObjectList keyList,
+ LimitedExpiredObjectList dirList) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ LimitedSizeStack stack = new LimitedSizeStack(cachedDirMaxCount);
+ try {
+ if (dir != null) {
+ stack.push(new PendingEvaluateDirectory(dir, directoryPath, null));
+ } else {
+ // put a placeholder PendingEvaluateDirectory to stack for bucket
+ stack.push(new PendingEvaluateDirectory(null, "", null));
+ }
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName,
directoryPath != null ? directoryPath : "", e);
+ return;
+ }
+
+ HashSet<Long> deletedDirSet = new HashSet<>();
+ while (!stack.isEmpty()) {
+ PendingEvaluateDirectory item = stack.pop();
+ OmDirectoryInfo currentDir = item.getDirectoryInfo();
+ String currentDirPath = item.getDirPath();
+ long currentDirObjID = currentDir == null ? bucket.getObjectID() :
currentDir.getObjectID();
+
+ // use current directory's object ID to iterate the keys and
directories under it
+ String prefix =
+ OM_KEY_PREFIX + volumeObjId + OM_KEY_PREFIX + bucket.getObjectID()
+ OM_KEY_PREFIX + currentDirObjID;
+ LOG.debug("Prefix {} for {}/{}", prefix, bucket.getVolumeName(),
bucket.getBucketName());
+
+ // get direct sub directories
+ SubDirectorySummary subDirSummary = item.getSubDirSummary();
+ boolean newSubDirPushed = false;
+ long deletedDirCount = 0;
+ if (subDirSummary == null) {
+ try {
+ subDirSummary = getSubDirectory(currentDirObjID, prefix,
omMetadataManager);
+ } catch (IOException e) {
+ // log failure, continue to process other directories in stack
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
currentDirPath, volumeName, bucketName, e);
+ continue;
+ }
+
+ // filter sub directory list
+ if (!subDirSummary.getSubDirList().isEmpty()) {
+ Iterator<OmDirectoryInfo> iterator =
subDirSummary.getSubDirList().iterator();
+ while (iterator.hasNext()) {
+ OmDirectoryInfo subDir = iterator.next();
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ if (subDirPath.startsWith(TRASH_PREFIX)) {
+ iterator.remove();
+ }
+ boolean matched = false;
+ for (OmLCRule rule : ruleList) {
+ if (rule.getEffectivePrefix() != null &&
subDirPath.startsWith(rule.getEffectivePrefix())) {
+ matched = true;
+ break;
+ }
+ }
+ if (!matched) {
+ iterator.remove();
}
- expiredDirList.add(dirInfo.getName(), 0, dirInfo.getUpdateID());
}
- } else {
- dirInfo = null;
- LOG.info("Skip evaluate trash directory {}", TRASH_PREFIX);
+ }
+
+ if (!subDirSummary.getSubDirList().isEmpty()) {
+ item.setSubDirSummary(subDirSummary);
+ try {
+ stack.push(item);
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName,
currentDirPath, e);
+ return;
+ }
+
+ // depth first evaluation, push subDirs into stack
+ for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ try {
+ stack.push(new PendingEvaluateDirectory(subDir, subDirPath,
null));
+ } catch (CapacityFullException e) {
+ LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName,
subDirPath, e);
+ return;
+ }
+ }
+ newSubDirPushed = true;
+ }
+ } else {
+ // this item is a parent directory, check how many sub directories
are deleted.
+ for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ if (deletedDirSet.remove(subDir.getObjectID())) {
+ deletedDirCount++;
+ }
+ }
+ }
+
+ if (newSubDirPushed) {
+ continue;
+ }
+
+ // evaluate direct files, first check cache, then check table
+ // there are three cases:
+ // a. key is deleted in cache, while it's not deleted in table yet
+ // b. key is new added in cache, not in table yet
+ // c. key is updated in cache(rename), but not updated in table yet
+ // in this case, the fromKey is a deleted key in cache, and the
toKey is a newly added key in cache,
+ // and fromKey is also in table
+ long numKeysUnderDir = 0;
+ long numKeysExpired = 0;
+ HashSet<String> deletedKeySet = new HashSet();
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter
= keyTable.cacheIterator();
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry =
cacheIter.next();
+ OmKeyInfo key = entry.getValue().getCacheValue();
+ if (key == null) {
+ deletedKeySet.add(entry.getKey().getCacheKey());
+ continue;
+ }
+ numKeysUnderDir++;
Review Comment:
we should check whether this is current dir's file (`if
(key.getParentObjectID() == currentDirObjID)`) before add the `numKeysUnderDir`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
