Repository: hadoop Updated Branches: refs/heads/ozone-0.3 db90350c9 -> 53d4aefae
HDDS-794. Add configs to set StateMachineData write timeout in ContainerStateMachine. Contributed by Shashikant Banerjee. (cherry picked from commit 408f59caa9321be8a55afe44b1811c5dacf23206) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53d4aefa Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53d4aefa Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53d4aefa Branch: refs/heads/ozone-0.3 Commit: 53d4aefae8490acbd3e64dd791ffbe17afaf91c4 Parents: db90350 Author: Arpit Agarwal <a...@apache.org> Authored: Mon Nov 5 10:10:10 2018 -0800 Committer: Arpit Agarwal <a...@apache.org> Committed: Mon Nov 5 10:10:17 2018 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++++++ .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 9 +++++++++ .../common/src/main/resources/ozone-default.xml | 7 +++++++ .../server/ratis/ContainerStateMachine.java | 18 ++++++++++++++++-- .../server/ratis/XceiverServerRatis.java | 14 ++++++++++++++ .../container/keyvalue/helpers/ChunkUtils.java | 2 ++ .../container/keyvalue/impl/ChunkManagerImpl.java | 3 ++- 7 files changed, 56 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index f95b748..11e6a23 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -74,6 +74,12 @@ public final class ScmConfigKeys { "dfs.container.ratis.segment.preallocated.size"; public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024; + public static final String + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT = + "dfs.container.ratis.statemachinedata.sync.timeout"; + public static final TimeDuration + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT = + TimeDuration.valueOf(10, TimeUnit.SECONDS); public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY = "dfs.ratis.client.request.timeout.duration"; public static final TimeDuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index c931dcf..5e9fe08 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -232,6 +232,15 @@ public final class OzoneConfigKeys { = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY; public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT; + + // config settings to enable stateMachineData write timeout + public static final String + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT = + ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT; + public static final TimeDuration + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT = + ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT; + public static final int DFS_CONTAINER_CHUNK_MAX_SIZE = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE; public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR = http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/common/src/main/resources/ozone-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 237f8d8..2e250fa 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -53,6 +53,13 @@ </description> </property> <property> + <name>dfs.container.ratis.statemachinedata.sync.timeout</name> + <value>10s</value> + <tag>OZONE, DEBUG, CONTAINER, RATIS</tag> + <description>Timeout for StateMachine data writes by Ratis. + </description> + </property> + <property> <name>dfs.container.ratis.datanode.storage.dir</name> <value/> <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag> http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index ac0833b..9c71829 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -307,9 +307,18 @@ public class ContainerStateMachine extends BaseStateMachine { () -> runCommand(requestProto), chunkExecutor); } writeChunkFutureMap.put(entryIndex, writeChunkFuture); + LOG.debug("writeChunk writeStateMachineData : blockId " + write.getBlockID() + + " logIndex " + entryIndex + " chunkName " + write.getChunkData() + .getChunkName()); // Remove the future once it finishes execution from the // writeChunkFutureMap. - writeChunkFuture.thenApply(r -> writeChunkFutureMap.remove(entryIndex)); + writeChunkFuture.thenApply(r -> { + writeChunkFutureMap.remove(entryIndex); + LOG.debug("writeChunk writeStateMachineData completed: blockId " + write + .getBlockID() + " logIndex " + entryIndex + " chunkName " + write + .getChunkData().getChunkName()); + return r; + }); return writeChunkFuture; } @@ -519,7 +528,12 @@ public class ContainerStateMachine extends BaseStateMachine { if (cmdType == Type.CreateContainer) { long containerID = requestProto.getContainerID(); future.thenApply( - r -> createContainerFutureMap.remove(containerID).complete(null)); + r -> { + createContainerFutureMap.remove(containerID).complete(null); + LOG.info("create Container Transaction completed for container " + + containerID + " log index " + index); + return r; + }); } future.thenAccept(m -> { http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 113633b..f45b531 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -186,6 +186,20 @@ public final class XceiverServerRatis implements XceiverServerSpi { RaftClientConfigKeys.Rpc .setRequestTimeout(properties, clientRequestTimeout); + // set the configs enable and set the stateMachineData sync timeout + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true); + timeUnit = OzoneConfigKeys. + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT.getUnit(); + duration = conf.getTimeDuration( + OzoneConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT, + OzoneConfigKeys. + DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT + .getDuration(), timeUnit); + final TimeDuration dataSyncTimeout = + TimeDuration.valueOf(duration, timeUnit); + RaftServerConfigKeys.Log.StateMachineData + .setSyncTimeout(properties, dataSyncTimeout); + // Set the server Request timeout timeUnit = OzoneConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_DEFAULT .getUnit(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java index 20598d9..492a286 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java @@ -138,6 +138,8 @@ public final class ChunkUtils { } } } + log.debug("Write Chunk completed for chunkFile: {}, size {}", chunkFile, + data.length); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/53d4aefa/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java index 6fd8d5f..8416494 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java @@ -82,7 +82,8 @@ public class ChunkManagerImpl implements ChunkManager { chunkFile, info); File tmpChunkFile = getTmpChunkFile(chunkFile, info); - LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file", + LOG.debug( + "writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file:{}", info.getChunkName(), stage, chunkFile, tmpChunkFile); switch (stage) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
