This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push: new a6f80c0 HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE a6f80c0 is described below commit a6f80c096b5320f50b6e9e9b4ba5f7c7e3544385 Author: Márton Elek <e...@apache.org> AuthorDate: Mon Nov 11 19:09:55 2019 +0100 HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE Closes #60 --- .../container/common/impl/HddsDispatcher.java | 5 ++ .../ozone/container/common/interfaces/Handler.java | 60 ++++++++-------- .../ozone/container/keyvalue/KeyValueHandler.java | 12 ++-- .../ozone/container/ozoneimpl/OzoneContainer.java | 59 ++++++++++------ .../container/common/impl/TestHddsDispatcher.java | 16 +++-- .../container/common/interfaces/TestHandler.java | 8 ++- .../container/keyvalue/TestKeyValueHandler.java | 10 ++- .../TestKeyValueHandlerWithUnhealthyContainer.java | 7 +- .../container/metrics/TestContainerMetrics.java | 9 ++- .../container/server/TestContainerServer.java | 7 +- .../server/TestSecureContainerServer.java | 7 +- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 82 ++++++++++++---------- .../org/apache/hadoop/ozone/genesis/Genesis.java | 3 +- 13 files changed, 169 insertions(+), 116 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 7f46707..1feb500 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -321,6 +321,11 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { // Once container is marked unhealthy, all the subsequent write // transactions will fail with UNHEALTHY_CONTAINER exception. + if (container == null) { + throw new NullPointerException( + "Error on creating containers " + result + " " + responseProto + .getMessage()); + } // For container to be moved to unhealthy state here, the container can // only be in open or closing state. State containerState = container.getContainerData().getState(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java index 8c3b981..c623e14 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java @@ -18,25 +18,19 @@ package org.apache.hadoop.ozone.container.common.interfaces; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.function.Consumer; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerType; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; -import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; @@ -54,41 +48,46 @@ public abstract class Handler { protected final VolumeSet volumeSet; protected String scmID; protected final ContainerMetrics metrics; + protected String datanodeId; + private Consumer<ContainerReplicaProto> icrSender; - private final StateContext context; - private final DatanodeDetails datanodeDetails; - - protected Handler(Configuration config, StateContext context, + protected Handler(Configuration config, String datanodeId, ContainerSet contSet, VolumeSet volumeSet, - ContainerMetrics containerMetrics) { + ContainerMetrics containerMetrics, + Consumer<ContainerReplicaProto> icrSender) { this.conf = config; - this.context = context; this.containerSet = contSet; this.volumeSet = volumeSet; this.metrics = containerMetrics; - this.datanodeDetails = context.getParent().getDatanodeDetails(); + this.datanodeId = datanodeId; + this.icrSender = icrSender; } public static Handler getHandlerForContainerType( final ContainerType containerType, final Configuration config, - final StateContext context, final ContainerSet contSet, - final VolumeSet volumeSet, final ContainerMetrics metrics) { + final String datanodeId, final ContainerSet contSet, + final VolumeSet volumeSet, final ContainerMetrics metrics, + Consumer<ContainerReplicaProto> icrSender) { switch (containerType) { case KeyValueContainer: - return new KeyValueHandler(config, context, contSet, volumeSet, metrics); + return new KeyValueHandler(config, + datanodeId, contSet, volumeSet, metrics, + icrSender); default: throw new IllegalArgumentException("Handler for ContainerType: " + - containerType + "doesn't exist."); + containerType + "doesn't exist."); } } /** * Returns the Id of this datanode. + * * @return datanode Id */ - protected DatanodeDetails getDatanodeDetails() { - return datanodeDetails; + protected String getDatanodeId() { + return datanodeId; } + /** * This should be called whenever there is state change. It will trigger * an ICR to SCM. @@ -97,12 +96,8 @@ public abstract class Handler { */ protected void sendICR(final Container container) throws StorageContainerException { - IncrementalContainerReportProto icr = IncrementalContainerReportProto - .newBuilder() - .addReport(container.getContainerReport()) - .build(); - context.addReport(icr); - context.getParent().triggerHeartbeat(); + ContainerReplicaProto containerReport = container.getContainerReport(); + icrSender.accept(containerReport); } public abstract ContainerCommandResponseProto handle( @@ -175,8 +170,9 @@ public abstract class Handler { * Deletes the given container. * * @param container container to be deleted - * @param force if this is set to true, we delete container without checking - * state of the container. + * @param force if this is set to true, we delete container without + * checking + * state of the container. * @throws IOException */ public abstract void deleteContainer(Container container, boolean force) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 995b913..5da0b8f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -27,6 +27,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.function.Function; import org.apache.hadoop.conf.Configuration; @@ -47,6 +48,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .PutSmallFileRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.hdds.scm.ByteStringConversion; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.common.helpers @@ -60,7 +62,6 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; -import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.transport.server.ratis .DispatcherContext; import org.apache.hadoop.ozone.container.common.transport.server.ratis @@ -109,9 +110,10 @@ public class KeyValueHandler extends Handler { private final AutoCloseableLock containerCreationLock; private final boolean doSyncWrite; - public KeyValueHandler(Configuration config, StateContext context, - ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics) { - super(config, context, contSet, volSet, metrics); + public KeyValueHandler(Configuration config, String datanodeId, + ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics, + Consumer<ContainerReplicaProto> icrSender) { + super(config, datanodeId, contSet, volSet, metrics, icrSender); containerType = ContainerType.KeyValueContainer; blockManager = new BlockManagerImpl(config); doSyncWrite = @@ -220,7 +222,7 @@ public class KeyValueHandler extends Handler { KeyValueContainerData newContainerData = new KeyValueContainerData( containerID, maxContainerSize, request.getPipelineID(), - getDatanodeDetails().getUuidString()); + getDatanodeId()); // TODO: Add support to add metadataList to ContainerData. Add metadata // to container during creation. KeyValueContainer newContainer = new KeyValueContainer( diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 6bce22c..328cd91 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -18,16 +18,21 @@ package org.apache.hadoop.ozone.container.ozoneimpl; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.datanode.proto - .ContainerProtos.ContainerType; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.PipelineReportsProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto; import org.apache.hadoop.hdds.security.token.BlockTokenVerifier; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; @@ -42,24 +47,20 @@ import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSp import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; - import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService; import org.apache.hadoop.ozone.container.replication.GrpcReplicationService; -import org.apache.hadoop.ozone.container.replication - .OnDemandContainerReplicationSource; +import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource; import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.ozone.OzoneConfigKeys.*; - /** * Ozone main class sets up the network servers and initializes the container * layer. @@ -100,10 +101,22 @@ public class OzoneContainer { buildContainerSet(); final ContainerMetrics metrics = ContainerMetrics.create(conf); this.handlers = Maps.newHashMap(); + + Consumer<ContainerReplicaProto> icrSender = containerReplicaProto -> { + IncrementalContainerReportProto icr = IncrementalContainerReportProto + .newBuilder() + .addReport(containerReplicaProto) + .build(); + context.addReport(icr); + context.getParent().triggerHeartbeat(); + }; + for (ContainerType containerType : ContainerType.values()) { handlers.put(containerType, Handler.getHandlerForContainerType( - containerType, conf, context, containerSet, volumeSet, metrics)); + containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, icrSender)); } SecurityConfig secConf = new SecurityConfig(conf); @@ -169,7 +182,6 @@ public class OzoneContainer { } - /** * Start background daemon thread for performing container integrity checks. */ @@ -240,13 +252,14 @@ public class OzoneContainer { ContainerMetrics.remove(); } - @VisibleForTesting public ContainerSet getContainerSet() { return containerSet; } + /** * Returns container report. + * * @return - container report. */ diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java index e9e8da8..9db11d0 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .WriteChunkRequestProto; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerAction; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto; import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; @@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.test.GenericTestUtils; + import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.junit.Assert; import org.junit.Test; @@ -57,6 +59,7 @@ import java.io.File; import java.io.IOException; import java.util.Map; import java.util.UUID; +import java.util.function.Consumer; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; @@ -69,6 +72,9 @@ import static org.mockito.Mockito.verify; */ public class TestHddsDispatcher { + public static final Consumer<ContainerReplicaProto> NO_OP_ICR_SENDER = + c -> {}; + @Test public void testContainerCloseActionWhenFull() throws IOException { String testDir = GenericTestUtils.getTempPath( @@ -98,8 +104,9 @@ public class TestHddsDispatcher { Map<ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerType containerType : ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, context, - containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, NO_OP_ICR_SENDER)); } HddsDispatcher hddsDispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, context, metrics, null); @@ -214,8 +221,9 @@ public class TestHddsDispatcher { Map<ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerType containerType : ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, context, - containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, NO_OP_ICR_SENDER)); } HddsDispatcher hddsDispatcher = new HddsDispatcher( diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java index cb40f95..086b0fe 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java @@ -19,16 +19,19 @@ package org.apache.hadoop.ozone.container.common.interfaces; import com.google.common.collect.Maps; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; + import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -69,7 +72,10 @@ public class TestHandler { ContainerProtos.ContainerType.values()) { handlers.put(containerType, Handler.getHandlerForContainerType( - containerType, conf, context, containerSet, volumeSet, metrics)); + containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, + TestHddsDispatcher.NO_OP_ICR_SENDER)); } this.dispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, null, metrics, null); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index 2c71fef..05b92cc 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@ -244,8 +244,10 @@ public class TestKeyValueHandler { Mockito.when(stateMachine.getDatanodeDetails()) .thenReturn(datanodeDetails); Mockito.when(context.getParent()).thenReturn(stateMachine); - KeyValueHandler keyValueHandler = new KeyValueHandler(conf, context, cset, - volumeSet, metrics); + KeyValueHandler keyValueHandler = new KeyValueHandler(conf, + context.getParent().getDatanodeDetails().getUuidString(), cset, + volumeSet, metrics, c -> { + }); assertEquals("org.apache.hadoop.ozone.container.common" + ".volume.RoundRobinVolumeChoosingPolicy", keyValueHandler.getVolumeChoosingPolicyForTesting() @@ -255,7 +257,9 @@ public class TestKeyValueHandler { conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY, "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher"); try { - new KeyValueHandler(conf, context, cset, volumeSet, metrics); + new KeyValueHandler(conf, + context.getParent().getDatanodeDetails().getUuidString(), + cset, volumeSet, metrics, c->{}); } catch (RuntimeException ex) { GenericTestUtils.assertExceptionContains("class org.apache.hadoop" + ".ozone.container.common.impl.HddsDispatcher not org.apache" + diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java index e3ae56a..26603b6 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java @@ -24,9 +24,11 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; + import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; @@ -41,7 +43,6 @@ import static org.junit.Assert.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; - /** * Test that KeyValueHandler fails certain operations when the * container is unhealthy. @@ -147,10 +148,10 @@ public class TestKeyValueHandlerWithUnhealthyContainer { return new KeyValueHandler( new OzoneConfiguration(), - context, + stateMachine.getDatanodeDetails().getUuidString(), mock(ContainerSet.class), mock(VolumeSet.class), - mock(ContainerMetrics.class)); + mock(ContainerMetrics.class), TestHddsDispatcher.NO_OP_ICR_SENDER); } private KeyValueContainer getMockUnhealthyContainer() { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 1d30653..0598f15 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import com.google.common.collect.Maps; + import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; @@ -51,6 +53,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; import org.apache.hadoop.ozone.container.replication.GrpcReplicationService; import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource; import org.apache.hadoop.test.GenericTestUtils; + import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -104,8 +107,10 @@ public class TestContainerMetrics { for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, context, - containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, + TestHddsDispatcher.NO_OP_ICR_SENDER)); } HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers, context, metrics, null); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index d71c0e7..e382734 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClie import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; @@ -224,8 +225,10 @@ public class TestContainerServer { for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, context, - containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, + context.getParent().getDatanodeDetails().getUuidString(), + containerSet, volumeSet, metrics, + TestHddsDispatcher.NO_OP_ICR_SENDER)); } HddsDispatcher dispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, context, metrics, null); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java index 2253fe2..33f93e5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Handler; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; @@ -155,8 +156,10 @@ public class TestSecureContainerServer { for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { handlers.put(containerType, - Handler.getHandlerForContainerType(containerType, conf, context, - containerSet, volumeSet, metrics)); + Handler.getHandlerForContainerType(containerType, conf, + dd.getUuid().toString(), + containerSet, volumeSet, metrics, + TestHddsDispatcher.NO_OP_ICR_SENDER)); } HddsDispatcher hddsDispatcher = new HddsDispatcher( conf, containerSet, volumeSet, handlers, context, metrics, diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java index 28da134..1d087f9 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java @@ -17,25 +17,42 @@ */ package org.apache.hadoop.ozone.genesis; -import com.google.common.collect.Maps; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsUtils; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumData; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkRequestProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Handler; -import org.apache.hadoop.ozone.container.common.statemachine - .DatanodeStateMachine.DatanodeStates; +import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; + +import com.google.common.collect.Maps; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; - +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Scope; @@ -43,28 +60,6 @@ import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.TearDown; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .PutBlockRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .GetBlockRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ReadChunkRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .WriteChunkRequestProto; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; - /** * Benchmarks DatanodeDispatcher class. */ @@ -83,6 +78,7 @@ public class BenchMarkDatanodeDispatcher { private static final int INIT_CONTAINERS = 100; private static final int INIT_KEYS = 50; private static final int INIT_CHUNKS = 100; + public static final int CHUNK_SIZE = 1048576; private List<Long> containers; private List<Long> keys; @@ -94,8 +90,8 @@ public class BenchMarkDatanodeDispatcher { datanodeUuid = UUID.randomUUID().toString(); // 1 MB of data - data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(1048576)); - random = new Random(); + data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(CHUNK_SIZE)); + random = new Random(); Configuration conf = new OzoneConfiguration(); baseDir = System.getProperty("java.io.tmpdir") + File.separator + datanodeUuid; @@ -103,6 +99,9 @@ public class BenchMarkDatanodeDispatcher { // data directory conf.set("dfs.datanode.data.dir", baseDir + File.separator + "data"); + //We need 100 * container size minimum space + conf.set("ozone.scm.container.size", "10MB"); + ContainerSet containerSet = new ContainerSet(); volumeSet = new VolumeSet(datanodeUuid, conf); StateContext context = new StateContext( @@ -111,9 +110,12 @@ public class BenchMarkDatanodeDispatcher { Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap(); for (ContainerProtos.ContainerType containerType : ContainerProtos.ContainerType.values()) { - handlers.put(containerType, - Handler.getHandlerForContainerType( - containerType, conf, context, containerSet, volumeSet, metrics)); + Handler handler = Handler.getHandlerForContainerType( + containerType, conf, "datanodeid", + containerSet, volumeSet, metrics, + c -> {}); + handler.setScmID("scm"); + handlers.put(containerType, handler); } dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers, context, metrics, null); @@ -217,11 +219,16 @@ public class BenchMarkDatanodeDispatcher { private ContainerProtos.ChunkInfo getChunkInfo( BlockID blockID, String chunkName) { ContainerProtos.ChunkInfo.Builder builder = - ContainerProtos.ChunkInfo.newBuilder() + ChunkInfo.newBuilder() .setChunkName( DigestUtils.md5Hex(chunkName) + "_stream_" + blockID.getContainerID() + "_block_" + blockID.getLocalID()) + .setChecksumData( + ChecksumData.newBuilder() + .setBytesPerChecksum(4) + .setType(ChecksumType.CRC32) + .build()) .setOffset(0).setLen(data.size()); return builder.build(); } @@ -245,7 +252,7 @@ public class BenchMarkDatanodeDispatcher { private ContainerCommandRequestProto getGetBlockCommand(BlockID blockID) { GetBlockRequestProto.Builder readBlockRequest = GetBlockRequestProto.newBuilder() - .setBlockID(blockID.getDatanodeBlockIDProtobuf()); + .setBlockID(blockID.getDatanodeBlockIDProtobuf()); ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto .newBuilder() .setCmdType(ContainerProtos.Type.GetBlock) @@ -274,7 +281,6 @@ public class BenchMarkDatanodeDispatcher { bmdd.containerCount.getAndIncrement(); } - @Benchmark public void writeChunk(BenchMarkDatanodeDispatcher bmdd) { bmdd.dispatcher.dispatch(getWriteChunkCommand( diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java index 2de9b0f..4361666 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java @@ -41,7 +41,8 @@ public final class Genesis { // After adding benchmark in genesis package add the benchmark name in the // description for this option. - @Option(names = "-benchmark", split = ",", description = + @Option(names = {"-b", "-benchmark", "--benchmark"}, + split = ",", description = "Option used for specifying benchmarks to run.\n" + "Ex. ozone genesis -benchmark BenchMarkContainerStateMap," + "BenchMarkOMKeyAllocation.\n" --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org