This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new cddc5719ae HDDS-9323. Apply expiry of excluded datanodes to writing
Ratis keys (#5530)
cddc5719ae is described below
commit cddc5719aec17f3252ea811f53233e7d709e0f76
Author: DaveTeng0 <[email protected]>
AuthorDate: Tue Nov 21 08:22:54 2023 -0800
HDDS-9323. Apply expiry of excluded datanodes to writing Ratis keys (#5530)
---
.../java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java | 2 +-
.../hdds/scm/container/common/helpers/ExcludeList.java | 3 +++
.../ozone/client/io/BlockDataStreamOutputEntryPool.java | 13 ++++++++++++-
.../hadoop/ozone/client/io/BlockOutputStreamEntryPool.java | 7 +++++--
.../ozone/client/rpc/TestFailureHandlingByClient.java | 4 ++++
5 files changed, 25 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index e2c5471be8..80f86a677a 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -184,7 +184,7 @@ public class OzoneClientConfig {
@Config(key = "exclude.nodes.expiry.time",
defaultValue = "600000",
description = "Time after which an excluded node is reconsidered for" +
- " writes in EC. If the value is zero, the node is excluded for the" +
+ " writes. If the value is zero, the node is excluded for the" +
" life of the client",
tags = ConfigTag.CLIENT)
private long excludeNodesExpiryTime = 10 * 60 * 1000;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
index 567e28e186..2577a1e5ea 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java
@@ -151,4 +151,7 @@ public class ExcludeList {
'}';
}
+ public long getExpiryTime() {
+ return expiryTime;
+ }
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
index d4bccd5535..8b8e8c474e 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockDataStreamOutputEntryPool.java
@@ -35,6 +35,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.time.Clock;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
@@ -83,7 +85,7 @@ public class BlockDataStreamOutputEntryPool implements
KeyMetadataAware {
.setMultipartUploadPartNumber(partNumber).build();
this.requestID = requestId;
this.openID = openID;
- this.excludeList = new ExcludeList();
+ this.excludeList = createExcludeList();
this.bufferList = new ArrayList<>();
}
@@ -289,6 +291,15 @@ public class BlockDataStreamOutputEntryPool implements
KeyMetadataAware {
return totalDataLen;
}
+ OzoneClientConfig getConfig() {
+ return config;
+ }
+
+ ExcludeList createExcludeList() {
+ return new ExcludeList(getConfig().getExcludeNodesExpiryTime(),
+ Clock.system(ZoneOffset.UTC));
+ }
+
public long getDataSize() {
return keyArgs.getDataSize();
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
index 65c1cd4caa..573e4a8dd3 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntryPool.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.ozone.client.io;
import java.io.IOException;
+import java.time.Clock;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
@@ -118,7 +120,8 @@ public class BlockOutputStreamEntryPool implements
KeyMetadataAware {
}
ExcludeList createExcludeList() {
- return new ExcludeList();
+ return new ExcludeList(getConfig().getExcludeNodesExpiryTime(),
+ Clock.system(ZoneOffset.UTC));
}
BlockOutputStreamEntryPool(ContainerClientMetrics clientMetrics) {
@@ -138,7 +141,7 @@ public class BlockOutputStreamEntryPool implements
KeyMetadataAware {
currentStreamIndex = 0;
openID = -1;
- excludeList = new ExcludeList();
+ excludeList = createExcludeList();
this.clientMetrics = clientMetrics;
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index a9e15ee607..911650390f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -174,6 +174,10 @@ public class TestFailureHandlingByClient {
// get the name of a valid container
Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
+ // assert that the exclude list's expire time equals to
+ // default value 600000 ms in OzoneClientConfig.java
+ Assert.assertEquals(((KeyOutputStream) key.getOutputStream())
+ .getExcludeList().getExpiryTime(), 600000);
KeyOutputStream groupOutputStream =
(KeyOutputStream) key.getOutputStream();
List<OmKeyLocationInfo> locationInfoList =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
