This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-7593
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-7593 by this push:
new f244a67037 HDDS-10252. [hsync] Revisit configuration keys for
incremental chunk list after HDDS-9884 (#6289)
f244a67037 is described below
commit f244a67037ccee76f759bd5ff7871f192312d64f
Author: Wei-Chiu Chuang <[email protected]>
AuthorDate: Sun Mar 3 19:51:46 2024 -0800
HDDS-10252. [hsync] Revisit configuration keys for incremental chunk list
after HDDS-9884 (#6289)
---
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 2 +-
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 ++--
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 6 ++---
.../container/keyvalue/impl/BlockManagerImpl.java | 28 ++++++++++------------
.../containergenerator/GeneratorDatanode.java | 6 ++---
6 files changed, 23 insertions(+), 25 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index d1dcc654b1..0b7c8652d7 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -230,7 +230,7 @@ public class OzoneClientConfig {
"list rather than full chunk list to optimize performance. " +
"Critical to HBase.",
tags = ConfigTag.CLIENT)
- private boolean incrementalChunkList = false;
+ private boolean incrementalChunkList = true;
@PostConstruct
private void validate() {
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 42a74dd12c..c0a48efead 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -146,8 +146,8 @@ public final class ScmConfigKeys {
"32KB";
public static final String OZONE_CHUNK_LIST_INCREMENTAL =
- "ozone.chunk.list.incremental";
- public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = false;
+ "ozone.incremental.chunk.list";
+ public static final boolean OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT = true;
public static final String OZONE_SCM_CONTAINER_LAYOUT_KEY =
"ozone.scm.container.layout";
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index a0d4b59db1..555b180625 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -126,7 +126,7 @@ public final class OzoneConfigKeys {
public static final String OZONE_FS_HSYNC_ENABLED
= "ozone.fs.hsync.enabled";
public static final boolean OZONE_FS_HSYNC_ENABLED_DEFAULT
- = false;
+ = true;
/**
* hsync lease soft limit.
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 47067de5fe..d9a39d3772 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -856,8 +856,8 @@
</description>
</property>
<property>
- <name>ozone.chunk.list.incremental</name>
- <value>false</value>
+ <name>ozone.incremental.chunk.list</name>
+ <value>true</value>
<tag>OZONE, CLIENT, DATANODE, PERFORMANCE</tag>
<description>
By default, a writer client sends full chunk list of a block when it
@@ -4116,7 +4116,7 @@
<property>
<name>ozone.fs.hsync.enabled</name>
- <value>false</value>
+ <value>true</value>
<tag>OZONE, CLIENT</tag>
<description>
Enable hsync/hflush. By default they are disabled.
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
index e40434f508..4690565b0b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -42,7 +42,6 @@ import com.google.common.base.Preconditions;
import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.BCSID_MISMATCH;
import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNKNOWN_BCSID;
-import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNSUPPORTED_REQUEST;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_CHUNK_LIST_INCREMENTAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT;
@@ -66,6 +65,7 @@ public class BlockManagerImpl implements BlockManager {
// Default Read Buffer capacity when Checksum is not present
private final int defaultReadBufferCapacity;
private final int readMappedBufferThreshold;
+ private boolean incrementalEnabled;
/**
* Constructs a Block Manager.
@@ -81,6 +81,15 @@ public class BlockManagerImpl implements BlockManager {
this.readMappedBufferThreshold = config.getBufferSize(
ScmConfigKeys.OZONE_CHUNK_READ_MAPPED_BUFFER_THRESHOLD_KEY,
ScmConfigKeys.OZONE_CHUNK_READ_MAPPED_BUFFER_THRESHOLD_DEFAULT);
+ incrementalEnabled =
+ config.getBoolean(OZONE_CHUNK_LIST_INCREMENTAL,
+ OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT);
+ if (incrementalEnabled && !VersionedDatanodeFeatures.isFinalized(
+ HDDSLayoutFeature.HBASE_SUPPORT)) {
+ LOG.warn("DataNode has not finalized upgrading to a version that " +
+ "supports incremental chunk list. Fallback to full chunk list");
+ incrementalEnabled = false;
+ }
}
@Override
@@ -93,23 +102,12 @@ public class BlockManagerImpl implements BlockManager {
boolean endOfBlock) throws IOException {
return persistPutBlock(
(KeyValueContainer) container,
- data,
- config,
- endOfBlock);
+ data, endOfBlock);
}
- public static long persistPutBlock(KeyValueContainer container,
- BlockData data, ConfigurationSource config, boolean endOfBlock)
+ public long persistPutBlock(KeyValueContainer container,
+ BlockData data, boolean endOfBlock)
throws IOException {
- boolean incrementalEnabled =
- config.getBoolean(OZONE_CHUNK_LIST_INCREMENTAL,
- OZONE_CHUNK_LIST_INCREMENTAL_DEFAULT);
- if (incrementalEnabled && !VersionedDatanodeFeatures.isFinalized(
- HDDSLayoutFeature.HBASE_SUPPORT)) {
- throw new StorageContainerException("DataNode has not finalized " +
- "upgrading to a version that supports incremental chunk list.",
- UNSUPPORTED_REQUEST);
- }
Preconditions.checkNotNull(data, "BlockData cannot be null for put " +
"operation.");
Preconditions.checkState(data.getContainerID() >= 0, "Container Id " +
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java
index 3a43ddd8ab..dbca12c8b2 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java
@@ -60,7 +60,6 @@ import
org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory;
-import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import com.codahale.metrics.Timer;
@@ -111,6 +110,7 @@ public class GeneratorDatanode extends BaseGenerator {
private int overlap;
private ChunkManager chunkManager;
+ private BlockManagerImpl blockManager;
private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy;
@@ -133,7 +133,7 @@ public class GeneratorDatanode extends BaseGenerator {
config = createOzoneConfiguration();
- BlockManager blockManager = new BlockManagerImpl(config);
+ blockManager = new BlockManagerImpl(config);
chunkManager = ChunkManagerFactory
.createChunkManager(config, blockManager, null);
@@ -286,7 +286,7 @@ public class GeneratorDatanode extends BaseGenerator {
writtenBytes += currentChunkSize;
}
- BlockManagerImpl.persistPutBlock(container, blockData, config, true);
+ blockManager.persistPutBlock(container, blockData, true);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
