Reidddddd commented on a change in pull request #549: HBASE-22912 [Backport]
HBASE-22867 to branch-1 to avoid ForkJoinPool to spawn thousands of threads
URL: https://github.com/apache/hbase/pull/549#discussion_r319344765
##########
File path:
hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/CleanerChore.java
##########
@@ -322,140 +330,154 @@ public boolean getEnabled() {
T act() throws IOException;
}
- /**
- * Attemps to clean up a directory, its subdirectories, and files. Return
value is true if
- * everything was deleted. false on partial / total failures.
- */
- private final class CleanerTask extends RecursiveTask<Boolean> {
- private static final long serialVersionUID = -1584635903138015418L;
- private final Path dir;
- private final boolean root;
+ private interface Callback {
+ void run(boolean val);
+ }
- CleanerTask(final FileStatus dir, final boolean root) {
- this(dir.getPath(), root);
+ private final class AsyncResult {
+
+ private Callback callback;
+ private boolean result;
+ private boolean resultSet = false;
+
+ AsyncResult(Callback callback) {
+ this.callback = callback;
}
- CleanerTask(final Path dir, final boolean root) {
- this.dir = dir;
- this.root = root;
+ AsyncResult() {
}
- @Override
- protected Boolean compute() {
- LOG.trace("Cleaning under " + dir);
- List<FileStatus> subDirs;
- List<FileStatus> tmpFiles;
- final List<FileStatus> files;
- try {
- // if dir doesn't exist, we'll get null back for both of these
- // which will fall through to succeeding.
- subDirs = FSUtils.listStatusWithStatusFilter(fs, dir, new
FileStatusFilter() {
- @Override
- public boolean accept(FileStatus f) {
- return f.isDirectory();
- }
- });
- if (subDirs == null) {
- subDirs = Collections.emptyList();
+ void set(boolean result) {
+ synchronized (this) {
+ this.result = result;
+ if (callback != null) {
+ callback.run(result);
}
- tmpFiles = FSUtils.listStatusWithStatusFilter(fs, dir, new
FileStatusFilter() {
- @Override
- public boolean accept(FileStatus f) {
- return f.isFile();
- }
- });
- files = tmpFiles == null ? Collections.<FileStatus>emptyList() :
tmpFiles;
- } catch (IOException ioe) {
- LOG.warn("failed to get FileStatus for contents of '" + dir + "'",
ioe);
- return false;
+ // Mark the result set process finished and notify the waiting get
method.
+ this.resultSet = true;
+ this.notify();
}
+ }
- boolean allFilesDeleted = true;
- if (!files.isEmpty()) {
- allFilesDeleted = deleteAction(new Action<Boolean>() {
- @Override
- public Boolean act() throws IOException {
- return checkAndDeleteFiles(files);
- }
- }, "files");
+ synchronized boolean get() throws Exception {
+ while (!resultSet) {
+ wait();
+ }
+ return result;
+ }
+ }
+
+ /**
+ * Attempts to clean up a directory(its subdirectories, and files) in a
+ * {@link java.util.concurrent.ThreadPoolExecutor} concurrently. We can get
the final result by
+ * calling result.get().
+ */
+ private void traverseAndDelete(final Path dir, final boolean root, final
AsyncResult result) {
+ try {
+ final Action<Boolean> curDirDeletion = new Action<Boolean>() {
+ @Override
+ public Boolean act() throws IOException {
+ return fs.delete(dir, false);
+ }
+ };
+
+ // Step.1: List all files under the given directory.
+ List<FileStatus> allPaths = Arrays.asList(fs.listStatus(dir));
+ final List<FileStatus> subDirs = new ArrayList<>();
+ final List<FileStatus> files = new ArrayList<>();
+ for (FileStatus status : allPaths) {
+ if (status.isDirectory()) {
+ subDirs.add(status);
+ } else if (status.isFile()) {
+ files.add(status);
+ }
}
- boolean allSubdirsDeleted = true;
- if (!subDirs.isEmpty()) {
- final List<CleanerTask> tasks =
Lists.newArrayListWithCapacity(subDirs.size());
- for (FileStatus subdir : subDirs) {
- CleanerTask task = new CleanerTask(subdir, false);
- tasks.add(task);
- task.fork();
+ // Step.2: Try to delete all the deletable files.
+ final boolean allFilesDeleted = files.isEmpty() || deleteAction(new
Action<Boolean>() {
+ @Override
+ public Boolean act() throws IOException {
+ return checkAndDeleteFiles(files);
}
- allSubdirsDeleted = deleteAction(new Action<Boolean>() {
- @Override
- public Boolean act() throws IOException {
- return getCleanResult(tasks);
- }
- }, "subdirs");
+ }, "files", dir);
+
+ // Step.3: Start to traverse and delete the sub-directories.
+ if (subDirs.isEmpty()) {
+ // If no sub-directories, then just try to delete the current dir and
finish the result.
+ boolean deleted = true;
+ if (allFilesDeleted && !root) {
+ deleted = deleteAction(curDirDeletion, "dir", dir);
+ }
+ result.set(deleted);
Review comment:
There's no more sub dir, but if there're files remaining, we still return
true? A bit confusing, i think it should be false.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
