This is an automated email from the ASF dual-hosted git repository.
msingh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c87b059 HDDS-6284. Add BlockDeletingService worker size config (#3056)
c87b059 is described below
commit c87b059b0f475c8af08f1ae7ead9f5e193e9df57
Author: hao guo <[email protected]>
AuthorDate: Thu Feb 17 17:13:22 2022 +0800
HDDS-6284. Add BlockDeletingService worker size config (#3056)
---
.../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 +++++
hadoop-hdds/common/src/main/resources/ozone-default.xml | 8 ++++++++
.../keyvalue/statemachine/background/BlockDeletingService.java | 9 ++++-----
.../apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java | 8 +++++++-
.../hadoop/ozone/container/common/TestBlockDeletingService.java | 4 ++--
.../common/impl/TestContainerDeletionChoosingPolicy.java | 2 +-
.../ozone/container/testutils/BlockDeletingServiceTestImpl.java | 2 +-
7 files changed, 28 insertions(+), 10 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 9c7d769..627c432 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -194,6 +194,11 @@ public final class OzoneConfigKeys {
public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT
= "300s"; // 300s for default
+ public static final String OZONE_BLOCK_DELETING_SERVICE_WORKERS =
+ "ozone.block.deleting.service.workers";
+ public static final int OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT
+ = 10;
+
public static final String OZONE_KEY_PREALLOCATION_BLOCKS_MAX =
"ozone.key.preallocation.max.blocks";
public static final int OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 92ebc6b..24f0c45 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -399,6 +399,14 @@
</description>
</property>
<property>
+ <name>ozone.block.deleting.service.workers</name>
+ <value>10</value>
+ <tag>OZONE, PERFORMANCE, SCM</tag>
+ <description>Number of workers executed of block deletion service. This
+ configuration should be set to greater than 0.
+ </description>
+ </property>
+ <property>
<name>ozone.UnsafeByteOperations.enabled</name>
<value>true</value>
<tag>OZONE, PERFORMANCE, CLIENT</tag>
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index d40afc5..c215ecf 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -87,14 +87,13 @@ public class BlockDeletingService extends BackgroundService
{
// Task priority is useful when a to-delete block has weight.
private static final int TASK_PRIORITY_DEFAULT = 1;
- // Core pool size for container tasks
- private static final int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
public BlockDeletingService(OzoneContainer ozoneContainer,
- long serviceInterval, long serviceTimeout, TimeUnit timeUnit,
- ConfigurationSource conf) {
+ long serviceInterval, long serviceTimeout,
+ TimeUnit timeUnit, int workerSize,
+ ConfigurationSource conf) {
super("BlockDeletingService", serviceInterval, timeUnit,
- BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
+ workerSize, serviceTimeout);
this.ozoneContainer = ozoneContainer;
try {
containerDeletionPolicy = conf.getClass(
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index d6c90d5..5eaa01e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -66,6 +66,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT;
import static
org.apache.hadoop.ozone.container.ozoneimpl.ContainerScrubberConfiguration.VOLUME_BYTES_PER_SECOND_KEY;
import org.apache.hadoop.util.Timer;
@@ -184,9 +186,13 @@ public class OzoneContainer {
.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
+
+ int serviceWorkerSize = config
+ .getInt(OZONE_BLOCK_DELETING_SERVICE_WORKERS,
+ OZONE_BLOCK_DELETING_SERVICE_WORKERS_DEFAULT);
blockDeletingService =
new BlockDeletingService(this, svcInterval.toMillis(), serviceTimeout,
- TimeUnit.MILLISECONDS, config);
+ TimeUnit.MILLISECONDS, serviceWorkerSize, config);
if (certClient != null && secConf.isGrpcTlsEnabled()) {
List<X509Certificate> x509Certificates =
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 5306eb0..7ac55f0 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -531,7 +531,7 @@ public class TestBlockDeletingService {
mockDependencies(containerSet, keyValueHandler);
BlockDeletingService svc = new BlockDeletingService(ozoneContainer,
TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
- conf);
+ 10, conf);
svc.start();
LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
@@ -553,7 +553,7 @@ public class TestBlockDeletingService {
timeout = 0;
svc = new BlockDeletingService(ozoneContainer,
TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.MILLISECONDS,
- conf);
+ 10, conf);
svc.start();
// get container meta data
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
index 92ebbca..94f2e3e 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java
@@ -232,7 +232,7 @@ public class TestContainerDeletionChoosingPolicy {
Mockito.when(ozoneContainer.getWriteChannel()).thenReturn(null);
blockDeletingService = new BlockDeletingService(ozoneContainer,
SERVICE_INTERVAL_IN_MILLISECONDS, SERVICE_TIMEOUT_IN_MILLISECONDS,
- TimeUnit.MILLISECONDS, conf);
+ TimeUnit.MILLISECONDS, 10, conf);
return blockDeletingService;
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
index ecb7af8..ee2bcfe 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
@@ -45,7 +45,7 @@ public class BlockDeletingServiceTestImpl
public BlockDeletingServiceTestImpl(OzoneContainer container,
int serviceInterval, ConfigurationSource conf) {
super(container, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
- TimeUnit.MILLISECONDS, conf);
+ TimeUnit.MILLISECONDS, 10, conf);
}
@VisibleForTesting
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
