[hadoop] 13/50: HDFS-13610. [SBN read] Edit Tail Fast Path Part 4: Cleanup. Integration test, documentation, remove unnecessary dummy sync, minors fixups. Contributed by Erik Krogen.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 9976b7bcbe9085e77e9a80419508a7672efdb8d7
Author: Erik Krogen
AuthorDate: Fri May 11 13:23:38 2018 -0700
HDFS-13610. [SBN read] Edit Tail Fast Path Part 4: Cleanup. Integration
test, documentation, remove unnecessary dummy sync, minors fixups. Contributed
by Erik Krogen.
---
.../hdfs/qjournal/client/QuorumJournalManager.java | 15 +-
.../hdfs/qjournal/client/QuorumOutputStream.java | 13 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 6 +
.../hdfs/qjournal/server/JournaledEditsCache.java | 63 +---
.../site/markdown/HDFSHighAvailabilityWithQJM.md | 35 +
.../qjournal/client/TestQuorumJournalManager.java | 4 +
.../client/TestQuorumJournalManagerUnit.java | 2 +
.../hdfs/qjournal/server/JournalTestUtil.java | 48 ++
.../namenode/ha/TestStandbyInProgressTail.java | 167 +
9 files changed, 284 insertions(+), 69 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index f9d96b0..a14e6a0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -77,6 +77,8 @@ public class QuorumJournalManager implements JournalManager {
// Maximum number of transactions to fetch at a time when using the
// RPC edit fetch mechanism
private final int maxTxnsPerRpc;
+ // Whether or not in-progress tailing is enabled in the configuration
+ private final boolean inProgressTailingEnabled;
// Timeouts for which the QJM will wait for each of the following actions.
private final int startSegmentTimeoutMs;
private final int prepareRecoveryTimeoutMs;
@@ -139,6 +141,9 @@ public class QuorumJournalManager implements JournalManager
{
conf.getInt(QJM_RPC_MAX_TXNS_KEY, QJM_RPC_MAX_TXNS_DEFAULT);
Preconditions.checkArgument(maxTxnsPerRpc > 0,
"Must specify %s greater than 0!", QJM_RPC_MAX_TXNS_KEY);
+this.inProgressTailingEnabled = conf.getBoolean(
+DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY,
+DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT);
// Configure timeouts.
this.startSegmentTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_QJOURNAL_START_SEGMENT_TIMEOUT_KEY,
@@ -420,11 +425,8 @@ public class QuorumJournalManager implements
JournalManager {
layoutVersion);
loggers.waitForWriteQuorum(q, startSegmentTimeoutMs,
"startLogSegment(" + txId + ")");
-boolean updateCommittedTxId = conf.getBoolean(
-DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY,
-DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT);
return new QuorumOutputStream(loggers, txId, outputBufferCapacity,
-writeTxnsTimeoutMs, updateCommittedTxId);
+writeTxnsTimeoutMs);
}
@Override
@@ -493,7 +495,10 @@ public class QuorumJournalManager implements
JournalManager {
public void selectInputStreams(Collection streams,
long fromTxnId, boolean inProgressOk,
boolean onlyDurableTxns) throws IOException {
-if (inProgressOk) {
+// Some calls will use inProgressOK to get in-progress edits even if
+// the cache used for RPC calls is not enabled; fall back to using the
+// streaming mechanism to serve such requests
+if (inProgressOk && inProgressTailingEnabled) {
LOG.info("Tailing edits starting from txn ID " + fromTxnId +
" via RPC mechanism");
try {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
index 3ffcd3e..e094b21 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
@@ -33,17 +33,15 @@ class QuorumOutputStream extends EditLogOutputStream {
private EditsDoubleBuffer buf;
private final long segmentTxId;
private final int writeTimeoutMs;
- private final boolean updateCommittedTxId;
public QuorumOutputStream(AsyncLoggerSet loggers,
long txId, int outputBufferCapacity,
- int writeTimeoutMs, boolean updateCommittedTxId) throws IOException {
+ int writeTimeoutMs) throws IOException {
super();
this.buf = new EditsDoubleBuffer(outputBufferCapacity);
this.loggers = loggers;
this.segmentTxId = txId;
this.writeTimeoutMs =
[hadoop] 13/50: HDFS-13610. [SBN read] Edit Tail Fast Path Part 4: Cleanup. Integration test, documentation, remove unnecessary dummy sync, minors fixups. Contributed by Erik Krogen.
This is an automated email from the ASF dual-hosted git repository.
cliang pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit ea402854aa63e10a086460f812afd2c3c9fbf8e8
Author: Erik Krogen
AuthorDate: Fri May 11 13:23:38 2018 -0700
HDFS-13610. [SBN read] Edit Tail Fast Path Part 4: Cleanup. Integration
test, documentation, remove unnecessary dummy sync, minors fixups. Contributed
by Erik Krogen.
---
.../hdfs/qjournal/client/QuorumJournalManager.java | 15 +-
.../hdfs/qjournal/client/QuorumOutputStream.java | 13 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 6 +
.../hdfs/qjournal/server/JournaledEditsCache.java | 63 +---
.../site/markdown/HDFSHighAvailabilityWithQJM.md | 28
.../qjournal/client/TestQuorumJournalManager.java | 4 +
.../client/TestQuorumJournalManagerUnit.java | 2 +
.../hdfs/qjournal/server/JournalTestUtil.java | 48 ++
.../namenode/ha/TestStandbyInProgressTail.java | 164 +
9 files changed, 275 insertions(+), 68 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 91d4995..5525e92 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@ -77,6 +77,8 @@ public class QuorumJournalManager implements JournalManager {
// Maximum number of transactions to fetch at a time when using the
// RPC edit fetch mechanism
private final int maxTxnsPerRpc;
+ // Whether or not in-progress tailing is enabled in the configuration
+ private final boolean inProgressTailingEnabled;
// Timeouts for which the QJM will wait for each of the following actions.
private final int startSegmentTimeoutMs;
private final int prepareRecoveryTimeoutMs;
@@ -139,6 +141,9 @@ public class QuorumJournalManager implements JournalManager
{
conf.getInt(QJM_RPC_MAX_TXNS_KEY, QJM_RPC_MAX_TXNS_DEFAULT);
Preconditions.checkArgument(maxTxnsPerRpc > 0,
"Must specify %s greater than 0!", QJM_RPC_MAX_TXNS_KEY);
+this.inProgressTailingEnabled = conf.getBoolean(
+DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY,
+DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT);
// Configure timeouts.
this.startSegmentTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_QJOURNAL_START_SEGMENT_TIMEOUT_KEY,
@@ -420,11 +425,8 @@ public class QuorumJournalManager implements
JournalManager {
layoutVersion);
loggers.waitForWriteQuorum(q, startSegmentTimeoutMs,
"startLogSegment(" + txId + ")");
-boolean updateCommittedTxId = conf.getBoolean(
-DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY,
-DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT);
return new QuorumOutputStream(loggers, txId, outputBufferCapacity,
-writeTxnsTimeoutMs, updateCommittedTxId);
+writeTxnsTimeoutMs);
}
@Override
@@ -493,7 +495,10 @@ public class QuorumJournalManager implements
JournalManager {
public void selectInputStreams(Collection streams,
long fromTxnId, boolean inProgressOk,
boolean onlyDurableTxns) throws IOException {
-if (inProgressOk) {
+// Some calls will use inProgressOK to get in-progress edits even if
+// the cache used for RPC calls is not enabled; fall back to using the
+// streaming mechanism to serve such requests
+if (inProgressOk && inProgressTailingEnabled) {
LOG.info("Tailing edits starting from txn ID " + fromTxnId +
" via RPC mechanism");
try {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
index 3ffcd3e..e094b21 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumOutputStream.java
@@ -33,17 +33,15 @@ class QuorumOutputStream extends EditLogOutputStream {
private EditsDoubleBuffer buf;
private final long segmentTxId;
private final int writeTimeoutMs;
- private final boolean updateCommittedTxId;
public QuorumOutputStream(AsyncLoggerSet loggers,
long txId, int outputBufferCapacity,
- int writeTimeoutMs, boolean updateCommittedTxId) throws IOException {
+ int writeTimeoutMs) throws IOException {
super();
this.buf = new EditsDoubleBuffer(outputBufferCapacity);
this.loggers = loggers;
this.segmentTxId = txId;
this.writeTimeoutMs =
