This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 27b2c1e HDFS-15583. Backport DirectoryScanner improvements
HDFS-14476, HDFS-14751 and HDFS-15048 to branch 3.2 and 3.1. Contributed by
Stephen O'Donnell
27b2c1e is described below
commit 27b2c1ea7bbf0048aca2f58e3204ae2c6a3bc3f0
Author: S O'Donnell <[email protected]>
AuthorDate: Tue Sep 22 10:34:05 2020 +0100
HDFS-15583. Backport DirectoryScanner improvements HDFS-14476, HDFS-14751
and HDFS-15048 to branch 3.2 and 3.1. Contributed by Stephen O'Donnell
(cherry picked from commit 5f34e3214e42098c7515f0441697980303aab643)
---
.../dev-support/findbugsExcludeFile.xml | 7 ++++-
.../hdfs/server/datanode/DirectoryScanner.java | 34 ++++++++++++++++------
2 files changed, 31 insertions(+), 10 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index 746df69..34cfaef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -311,4 +311,9 @@
<Method name="setInteractiveFormat" />
<Bug pattern="ME_ENUM_FIELD_SETTER" />
</Match>
- </FindBugsFilter>
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.datanode.DirectoryScanner" />
+ <Method name="reconcile" />
+ <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
+ </Match>
+</FindBugsFilter>
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 40a4cb9..dc941a5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -66,7 +66,7 @@ public class DirectoryScanner implements Runnable {
+ " starting at %s with interval of %dms";
private static final String START_MESSAGE_WITH_THROTTLE = START_MESSAGE
+ " and throttle limit of %dms/s";
-
+ private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
private final FsDatasetSpi<?> dataset;
private final ExecutorService reportCompileThreadPool;
private final ScheduledExecutorService masterThread;
@@ -298,7 +298,9 @@ public class DirectoryScanner implements Runnable {
* Clear the current cache of diffs and statistics.
*/
private void clear() {
- diffs.clear();
+ synchronized (diffs) {
+ diffs.clear();
+ }
stats.clear();
}
@@ -371,13 +373,25 @@ public class DirectoryScanner implements Runnable {
*/
@VisibleForTesting
public void reconcile() throws IOException {
+ LOG.debug("reconcile start DirectoryScanning");
scan();
- for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
- String bpid = entry.getKey();
- LinkedList<ScanInfo> diff = entry.getValue();
-
- for (ScanInfo info : diff) {
- dataset.checkAndUpdate(bpid, info);
+ int loopCount = 0;
+ synchronized (diffs) {
+ for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
+ String bpid = entry.getKey();
+ LinkedList<ScanInfo> diff = entry.getValue();
+
+ for (ScanInfo info : diff) {
+ dataset.checkAndUpdate(bpid, info);
+ if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ }
+ loopCount++;
+ }
}
}
if (!retainDiffs) clear();
@@ -400,7 +414,9 @@ public class DirectoryScanner implements Runnable {
Stats statsRecord = new Stats(bpid);
stats.put(bpid, statsRecord);
LinkedList<ScanInfo> diffRecord = new LinkedList<ScanInfo>();
- diffs.put(bpid, diffRecord);
+ synchronized(diffs) {
+ diffs.put(bpid, diffRecord);
+ }
statsRecord.totalBlocks = blockpoolReport.length;
final List<ReplicaInfo> bl = dataset.getSortedFinalizedBlocks(bpid);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
