This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new ddcfc52 HDDS-6252. EC: Datanode Chunk Validator fails on encountering
EC pipeline (#3035)
ddcfc52 is described below
commit ddcfc521af0f25ee3c748b9c9c5831c83ea92ad2
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Thu Feb 3 18:02:36 2022 +0100
HDDS-6252. EC: Datanode Chunk Validator fails on encountering EC pipeline
(#3035)
---
.../hadoop/ozone/freon/BaseFreonGenerator.java | 12 ++++++--
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 ++--------------------
2 files changed, 11 insertions(+), 33 deletions(-)
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
index 0a639ec..53267fd 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -345,19 +346,24 @@ public class BaseFreonGenerator {
return HAUtils.getScmContainerClient(ozoneConf);
}
+ @SuppressWarnings("java:S3864") // Stream.peek (for debug)
public static Pipeline findPipelineForTest(String pipelineId,
StorageContainerLocationProtocol client, Logger log) throws IOException {
- List<Pipeline> pipelines = client.listPipelines();
+ Stream<Pipeline> pipelines = client.listPipelines().stream();
Pipeline pipeline;
+ if (log.isDebugEnabled()) {
+ pipelines = pipelines
+ .peek(p -> log.debug("Found pipeline {}", p.getId().getId()));
+ }
if (pipelineId != null && pipelineId.length() > 0) {
- pipeline = pipelines.stream()
+ pipeline = pipelines
.filter(p -> p.getId().toString().equals(pipelineId))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(
"Pipeline ID is defined, but there is no such pipeline: "
+ pipelineId));
} else {
- pipeline = pipelines.stream()
+ pipeline = pipelines
.filter(p -> p.getReplicationConfig().getRequiredNodes() == 3)
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java
index 5d86bc5..b290da2 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java
@@ -18,15 +18,12 @@ package org.apache.hadoop.ozone.freon;
import java.io.IOException;
import java.util.concurrent.Callable;
-import java.util.stream.Stream;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
@@ -79,7 +76,6 @@ public class DatanodeChunkValidator extends BaseFreonGenerator
@Override
- @SuppressWarnings("java:S3864") // Stream.peek (for debug)
public Void call() throws Exception {
init();
@@ -91,33 +87,9 @@ public class DatanodeChunkValidator extends
BaseFreonGenerator
);
}
- try (StorageContainerLocationProtocol scmLocationClient =
+ try (StorageContainerLocationProtocol scmClient =
createStorageContainerLocationClient(ozoneConf)) {
- Stream<Pipeline> pipelines = scmLocationClient.listPipelines().stream();
- if (LOG.isDebugEnabled()) {
- pipelines = pipelines
- .peek(p -> LOG.debug("Found pipeline {}", p.getId()));
- }
- Pipeline pipeline;
- if (pipelineId != null && pipelineId.length() > 0) {
- pipeline = pipelines
- .filter(p -> p.getId().getId().toString().equals(pipelineId))
- .findFirst()
- .orElseThrow(() -> new IllegalArgumentException(
- "Pipeline ID is defined, but there is no such pipeline: "
- + pipelineId));
-
- } else {
- pipeline = pipelines
- .filter(
- p ->
ReplicationConfig.getLegacyFactor(p.getReplicationConfig())
- == HddsProtos.ReplicationFactor.THREE)
- .findFirst()
- .orElseThrow(() -> new IllegalArgumentException(
- "Pipeline ID is NOT defined, and no pipeline " +
- "has been found with factor=THREE"));
- LOG.info("Using pipeline {}", pipeline.getId());
- }
+ Pipeline pipeline = findPipelineForTest(pipelineId, scmClient, LOG);
try (XceiverClientManager xceiverClientManager =
new XceiverClientManager(ozoneConf)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
