This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2d98a7bed7 HDDS-9641. Parallel loading datanode volume db store (#5555)
2d98a7bed7 is described below
commit 2d98a7bed7378cbca08e3ed6bf97f500fdb38af8
Author: hao guo <[email protected]>
AuthorDate: Wed Nov 8 17:17:45 2023 +0800
HDDS-9641. Parallel loading datanode volume db store (#5555)
---
.../container/common/utils/HddsVolumeUtil.java | 38 ++++++++++++++++------
1 file changed, 28 insertions(+), 10 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
index 0d4709392a..9b1da77b66 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/HddsVolumeUtil.java
@@ -29,9 +29,11 @@ import org.slf4j.Logger;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
import static
org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil.onFailure;
@@ -86,16 +88,32 @@ public final class HddsVolumeUtil {
// between each HddsVolume -> DbVolume.
mapDbVolumesToDataVolumesIfNeeded(hddsVolumeSet, dbVolumeSet);
- for (HddsVolume volume : StorageVolumeUtil.getHddsVolumesList(
- hddsVolumeSet.getVolumesList())) {
- try {
- volume.loadDbStore(readOnly);
- } catch (IOException e) {
- onFailure(volume);
- if (logger != null) {
- logger.error("Load db store for HddsVolume {} failed",
- volume.getStorageDir().getAbsolutePath(), e);
- }
+ List<CompletableFuture<Void>> futures = new ArrayList<>();
+ List<HddsVolume> hddsVolumes = StorageVolumeUtil.getHddsVolumesList(
+ hddsVolumeSet.getVolumesList());
+ long start = System.currentTimeMillis();
+ for (HddsVolume volume : hddsVolumes) {
+ futures.add(CompletableFuture.runAsync(
+ () -> loadVolume(volume, readOnly, logger)));
+ }
+ for (CompletableFuture<Void> future : futures) {
+ future.join();
+ }
+ if (logger != null) {
+ logger.info("Load {} volumes DbStore cost: {}ms", hddsVolumes.size(),
+ System.currentTimeMillis() - start);
+ }
+ }
+
+ private static void loadVolume(HddsVolume volume, boolean readOnly,
+ Logger logger) {
+ try {
+ volume.loadDbStore(readOnly);
+ } catch (IOException e) {
+ onFailure(volume);
+ if (logger != null) {
+ logger.error("Load db store for HddsVolume {} failed",
+ volume.getStorageDir().getAbsolutePath(), e);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
