[1/2] kudu git commit: Factor out consensus queue methods
Repository: kudu
Updated Branches:
refs/heads/master a74f9a0dc -> f2479e21d
Factor out consensus queue methods
This patch factors a couple of code blocks out into methods to reduce
the line count of PeerMessageQueue::ResponseFromPeer().
Change-Id: I278de150e3dc42181ccfbbd3a4c0e5cc4de90c1a
Reviewed-on: http://gerrit.cloudera.org:8080/9642
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4c1788ea
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4c1788ea
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4c1788ea
Branch: refs/heads/master
Commit: 4c1788eae5bfd4cf4a714f1ca0ab775b005303b3
Parents: a74f9a0
Author: Mike Percy
Authored: Tue Mar 13 20:06:32 2018 -0700
Committer: Alexey Serbin
Committed: Thu Mar 15 06:02:00 2018 +
--
src/kudu/consensus/consensus_queue.cc | 160 -
src/kudu/consensus/consensus_queue.h | 12 ++-
2 files changed, 100 insertions(+), 72 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/4c1788ea/src/kudu/consensus/consensus_queue.cc
--
diff --git a/src/kudu/consensus/consensus_queue.cc
b/src/kudu/consensus/consensus_queue.cc
index 189f499..f85589b 100644
--- a/src/kudu/consensus/consensus_queue.cc
+++ b/src/kudu/consensus/consensus_queue.cc
@@ -921,6 +921,78 @@ void PeerMessageQueue::UpdatePeerStatus(const string&
peer_uuid,
}
}
+void PeerMessageQueue::UpdateExchangeStatus(TrackedPeer* peer,
+const TrackedPeer& prev_peer_state,
+const ConsensusResponsePB&
response,
+bool* lmp_mismatch) {
+ DCHECK(queue_lock_.is_locked());
+ const ConsensusStatusPB& status = response.status();
+
+ peer->last_communication_time = MonoTime::Now();
+ peer->last_known_committed_index = status.last_committed_idx();
+
+ if (PREDICT_TRUE(!status.has_error())) {
+peer->last_exchange_status = PeerStatus::OK;
+*lmp_mismatch = false;
+return;
+ }
+
+ switch (status.error().code()) {
+case ConsensusErrorPB::PRECEDING_ENTRY_DIDNT_MATCH:
+ peer->last_exchange_status = PeerStatus::LMP_MISMATCH;
+ DCHECK(status.has_last_received());
+ if (prev_peer_state.last_exchange_status == PeerStatus::NEW) {
+LOG_WITH_PREFIX_UNLOCKED(INFO) << "Connected to new peer: " <<
peer->ToString();
+ } else {
+LOG_WITH_PREFIX_UNLOCKED(INFO) << "Got LMP mismatch error from peer: "
+ << peer->ToString();
+ }
+ *lmp_mismatch = true;
+ return;
+
+case ConsensusErrorPB::INVALID_TERM:
+ peer->last_exchange_status = PeerStatus::INVALID_TERM;
+ CHECK(response.has_responder_term());
+ LOG_WITH_PREFIX_UNLOCKED(INFO) << "Peer responded invalid term: " <<
peer->ToString();
+ NotifyObserversOfTermChange(response.responder_term());
+ *lmp_mismatch = false;
+ return;
+
+default:
+ // Other ConsensusStatusPB error codes (such as remote errors) are
+ // supposed to be handled higher up in the stack.
+ LOG_WITH_PREFIX_UNLOCKED(FATAL) << "Unexpected consensus error. Code: "
+ << ConsensusErrorPB::Code_Name(status.error().code()) << ".
Response: "
+ << SecureShortDebugString(response);
+ }
+}
+
+void PeerMessageQueue::PromoteIfNeeded(TrackedPeer* peer, const TrackedPeer&
prev_peer_state,
+ const ConsensusStatusPB& status) {
+ DCHECK(queue_lock_.is_locked());
+ int64_t entries_behind = queue_state_.committed_index -
peer->last_received.index();
+ if (queue_state_.mode != PeerMessageQueue::LEADER ||
+ entries_behind > FLAGS_consensus_promotion_max_wal_entries_behind) {
+return;
+ }
+
+ // TODO(mpercy): It would be more efficient to cache the member type in the
+ // TrackedPeer data structure.
+ RaftPeerPB* peer_pb;
+ Status s =
GetRaftConfigMember(DCHECK_NOTNULL(queue_state_.active_config.get()),
+ peer->uuid(), &peer_pb);
+ if (s.ok() &&
+ peer_pb &&
+ peer_pb->member_type() == RaftPeerPB::NON_VOTER &&
+ peer_pb->attrs().promote()) {
+// This peer is ready to promote.
+//
+// TODO(mpercy): Should we introduce a function SafeToPromote() that
+// does the same calculation as SafeToEvict() but for adding a VOTER?
+NotifyObserversOfPeerToPromote(peer->uuid());
+ }
+}
+
void PeerMessageQueue::ResponseFromPeer(const std::string& peer_uuid,
const ConsensusResponsePB& response,
bool* more_pending) {
@@ -946,28 +1018,23 @@ void PeerMessageQueue::ResponseFr
[2/2] kudu git commit: KUDU-2342. consensus: use tighter bound for non-voter promotion
KUDU-2342. consensus: use tighter bound for non-voter promotion
This patch replaces the NON_VOTER promotion heuristic based on the
--consensus_promotion_max_wal_entries_behind gflag with a heuristic that
roughly matches the promotion criteria outlined in Diego Ongaro's
dissertation.
The approach is that we track the operation count of the last batch
successfully sent from leader to peer. If the peer is within that number
of ops of the commit index, the peer is eligible for promotion.
The aforementioned gflag has been removed.
This patch passes existing tests and did well on manual testing. I will
follow up on this patch with an automated test.
Change-Id: Iff517f01d6dc25eb15d01593dd57b7dc0dd25956
Reviewed-on: http://gerrit.cloudera.org:8080/9627
Reviewed-by: Alexey Serbin
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f2479e21
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f2479e21
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f2479e21
Branch: refs/heads/master
Commit: f2479e21d5a3002ebf5b1012fde83a6cffc2db82
Parents: 4c1788e
Author: Mike Percy
Authored: Wed Mar 14 15:39:16 2018 -0700
Committer: Alexey Serbin
Committed: Thu Mar 15 06:51:25 2018 +
--
src/kudu/consensus/consensus_queue.cc | 35 ++
1 file changed, 21 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/f2479e21/src/kudu/consensus/consensus_queue.cc
--
diff --git a/src/kudu/consensus/consensus_queue.cc
b/src/kudu/consensus/consensus_queue.cc
index f85589b..804739c 100644
--- a/src/kudu/consensus/consensus_queue.cc
+++ b/src/kudu/consensus/consensus_queue.cc
@@ -71,13 +71,6 @@ DEFINE_int32(consensus_inject_latency_ms_in_notifications, 0,
TAG_FLAG(consensus_inject_latency_ms_in_notifications, hidden);
TAG_FLAG(consensus_inject_latency_ms_in_notifications, unsafe);
-DEFINE_int32(consensus_promotion_max_wal_entries_behind, 100,
- "The number of WAL entries that a NON_VOTER with PROMOTE=true can
"
- "be behind the leader's committed index and be considered ready "
- "for promotion to VOTER.");
-TAG_FLAG(consensus_promotion_max_wal_entries_behind, advanced);
-TAG_FLAG(consensus_promotion_max_wal_entries_behind, experimental);
-
DECLARE_int32(consensus_rpc_timeout_ms);
DECLARE_bool(safe_time_advancement_without_writes);
DECLARE_bool(raft_prepare_replacement_before_eviction);
@@ -970,9 +963,8 @@ void PeerMessageQueue::UpdateExchangeStatus(TrackedPeer*
peer,
void PeerMessageQueue::PromoteIfNeeded(TrackedPeer* peer, const TrackedPeer&
prev_peer_state,
const ConsensusStatusPB& status) {
DCHECK(queue_lock_.is_locked());
- int64_t entries_behind = queue_state_.committed_index -
peer->last_received.index();
if (queue_state_.mode != PeerMessageQueue::LEADER ||
- entries_behind > FLAGS_consensus_promotion_max_wal_entries_behind) {
+ peer->last_exchange_status != PeerStatus::OK) {
return;
}
@@ -982,13 +974,28 @@ void PeerMessageQueue::PromoteIfNeeded(TrackedPeer* peer,
const TrackedPeer& pre
Status s =
GetRaftConfigMember(DCHECK_NOTNULL(queue_state_.active_config.get()),
peer->uuid(), &peer_pb);
if (s.ok() &&
- peer_pb &&
peer_pb->member_type() == RaftPeerPB::NON_VOTER &&
peer_pb->attrs().promote()) {
-// This peer is ready to promote.
-//
-// TODO(mpercy): Should we introduce a function SafeToPromote() that
-// does the same calculation as SafeToEvict() but for adding a VOTER?
+
+// Only promote the peer if it is within one round-trip of being fully
+// caught-up with the current commit index, as measured by recent
+// UpdateConsensus() operation batch sizes.
+
+// If we had never previously contacted this peer, wait until the second
+// time we contact them to try to promote them.
+if (prev_peer_state.last_received.index() == 0) return;
+
+int64_t last_batch_size =
+std::max(0, peer->last_received.index() -
prev_peer_state.last_received.index());
+bool peer_caught_up =
+!OpIdEquals(status.last_received_current_leader(), MinimumOpId()) &&
+status.last_received_current_leader().index() + last_batch_size
+>= queue_state_.committed_index;
+if (!peer_caught_up) return;
+
+// TODO(mpercy): Implement a SafeToPromote() check to ensure that we only
+// try to promote a NON_VOTER to VOTER if we will be able to commit the
+// resulting config change operation.
NotifyObserversOfPeerToPromote(peer->uuid());
}
}
kudu git commit: [consensus] FAILED_UNRECOVERABLE replica health status
Repository: kudu
Updated Branches:
refs/heads/master e684de337 -> a74f9a0dc
[consensus] FAILED_UNRECOVERABLE replica health status
Added HealthStatus::FAILED_UNRECOVERABLE for a tablet replica. This is
to mark replicas which are not able to catch up with the leader due to
GC-collected segments of WAL and other unrecoverable cases.
With the introduction of the FAILED_UNRECOVERABLE health status, the
replica management scheme becomes hybrid: the system evicts replicas
with FAILED_UNRECOVERABLE health status before adding a replacement
if it anticipates that it can commit the transaction.
This patch is a part of the fix to address KUDU-2342. It also addresses
KUDU-2322 as well: evicting voter replicas more aggressively if they
fall behind log segment GC threshold.
Change-Id: I35637c5bda6681b732dbc2bbf94b9d4258b12095
Reviewed-on: http://gerrit.cloudera.org:8080/9625
Tested-by: Alexey Serbin
Reviewed-by: Mike Percy
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a74f9a0d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a74f9a0d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a74f9a0d
Branch: refs/heads/master
Commit: a74f9a0dcaf88315c8563b95cdeb5701d9ce5438
Parents: e684de3
Author: Alexey Serbin
Authored: Tue Mar 13 22:31:32 2018 -0700
Committer: Alexey Serbin
Committed: Thu Mar 15 05:19:07 2018 +
--
src/kudu/consensus/consensus_queue.cc | 16 +-
src/kudu/consensus/metadata.proto | 17 +-
src/kudu/consensus/quorum_util-test.cc | 302 ---
src/kudu/consensus/quorum_util.cc | 237 +--
.../raft_consensus_nonvoter-itest.cc| 41 ++-
.../ts_tablet_manager-itest.cc | 6 +-
6 files changed, 455 insertions(+), 164 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/a74f9a0d/src/kudu/consensus/consensus_queue.cc
--
diff --git a/src/kudu/consensus/consensus_queue.cc
b/src/kudu/consensus/consensus_queue.cc
index 28110aa..189f499 100644
--- a/src/kudu/consensus/consensus_queue.cc
+++ b/src/kudu/consensus/consensus_queue.cc
@@ -520,7 +520,8 @@ void
PeerMessageQueue::UpdatePeerHealthUnlocked(TrackedPeer* peer) {
// Prepare error messages for different conditions.
string error_msg;
- if (overall_health_status == HealthReportPB::FAILED) {
+ if (overall_health_status == HealthReportPB::FAILED ||
+ overall_health_status == HealthReportPB::FAILED_UNRECOVERABLE) {
if (peer->last_exchange_status == PeerStatus::TABLET_FAILED) {
error_msg = Substitute("The tablet replica hosted on peer $0 has
failed", peer->uuid());
} else if (!peer->wal_catchup_possible) {
@@ -541,7 +542,8 @@ void
PeerMessageQueue::UpdatePeerHealthUnlocked(TrackedPeer* peer) {
if (FLAGS_raft_prepare_replacement_before_eviction) {
if (changed) {
- if (overall_health_status == HealthReportPB::FAILED) {
+ if (overall_health_status == HealthReportPB::FAILED ||
+ overall_health_status == HealthReportPB::FAILED_UNRECOVERABLE) {
// Only log when the status changes to FAILED.
LOG_WITH_PREFIX_UNLOCKED(INFO) << error_msg;
}
@@ -549,7 +551,8 @@ void
PeerMessageQueue::UpdatePeerHealthUnlocked(TrackedPeer* peer) {
NotifyObserversOfPeerHealthChange();
}
} else {
-if (overall_health_status == HealthReportPB::FAILED &&
+if ((overall_health_status == HealthReportPB::FAILED ||
+ overall_health_status == HealthReportPB::FAILED_UNRECOVERABLE) &&
SafeToEvictUnlocked(peer->uuid())) {
NotifyObserversOfFailedFollower(peer->uuid(), queue_state_.current_term,
error_msg);
}
@@ -563,14 +566,15 @@ HealthReportPB::HealthStatus
PeerMessageQueue::PeerHealthStatus(const TrackedPee
return HealthReportPB::FAILED;
}
- // Replicas that have fallen behind the leader's retained WAL are considered
failed.
+ // Replicas that have fallen behind the leader's retained WAL are failed
+ // and have no chance to come back.
if (!peer.wal_catchup_possible) {
-return HealthReportPB::FAILED;
+return HealthReportPB::FAILED_UNRECOVERABLE;
}
// Replicas returning TABLET_FAILED status are considered failed.
if (peer.last_exchange_status == PeerStatus::TABLET_FAILED) {
-return HealthReportPB::FAILED;
+return HealthReportPB::FAILED_UNRECOVERABLE;
}
// The happy case: replicas returned OK during the recent exchange are
considered healthy.
http://git-wip-us.apache.org/repos/asf/kudu/blob/a74f9a0d/src/kudu/consensus/metadata.proto
--
diff --git a/src/kudu/consensus/metadata.proto
b/src/kudu/consensus/metadata.proto
index a310127..6d21a41 10
kudu git commit: KUDU-2259: add real user to AuthenticationCredentialsPB
Repository: kudu
Updated Branches:
refs/heads/branch-1.7.x 132da394d -> c243d68be
KUDU-2259: add real user to AuthenticationCredentialsPB
This commit adds the 'real user' to the authn credentials token, which
is used when negotiating connections with SASL PLAIN authentication.
This is useful when scan tokens are being sent to remote tasks, it's not
possible to authenticate with a signed authn token to the remote
server[1], coarse-grained ACLs have been set, and the 'planner' and
'executor' processes are being run with different users.
This problematic scenario might also have been solved by allowing tokens
to be used in all scenarios, even when encryption is disabled, but the
approach taken by this commit allows that invariant to remain.
[1]: this most often occurs because the remote server has encryption disabled.
Change-Id: I5d2d901d42501ecfc0f6372f68cf7335eb188b45
Reviewed-on: http://gerrit.cloudera.org:8080/9374
Reviewed-by: Todd Lipcon
Tested-by: Kudu Jenkins
(cherry picked from commit e684de3371941cc5ae8fc4a546ecda7dbe9f4f2f)
Reviewed-on: http://gerrit.cloudera.org:8080/9647
Reviewed-by: Grant Henke
Tested-by: Grant Henke
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c243d68b
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c243d68b
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c243d68b
Branch: refs/heads/branch-1.7.x
Commit: c243d68be01d6172df75069d343fed136df34c2b
Parents: 132da39
Author: Dan Burkert
Authored: Tue Feb 20 17:37:17 2018 -0800
Committer: Grant Henke
Committed: Thu Mar 15 03:48:20 2018 +
--
.../java/org/apache/kudu/client/Negotiator.java | 8 +-
.../org/apache/kudu/client/SecurityContext.java | 47 ++
.../client/TestSecurityContextRealUser.java | 97
src/kudu/client/client-internal.cc | 2 +
src/kudu/client/client-internal.h | 5 +
src/kudu/client/client-test.cc | 36
src/kudu/client/client.cc | 23 -
src/kudu/client/client.proto| 7 +-
src/kudu/client/master_rpc.cc | 10 ++
src/kudu/client/master_rpc.h| 5 +
src/kudu/client/meta_cache.cc | 1 +
src/kudu/mini-cluster/external_mini_cluster.cc | 6 +-
src/kudu/rpc/connection_id.cc | 1 +
src/kudu/rpc/server_negotiation.cc | 6 +-
src/kudu/rpc/user_credentials.cc| 7 +-
src/kudu/rpc/user_credentials.h | 5 +
src/kudu/util/user.cc | 2 +-
17 files changed, 235 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/c243d68b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
--
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
index 95ad907..47e4854 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
@@ -92,7 +92,7 @@ import org.apache.kudu.util.SecurityUtil;
public class Negotiator extends SimpleChannelUpstreamHandler {
private static final Logger LOG = LoggerFactory.getLogger(Negotiator.class);
- private static final SaslClientCallbackHandler SASL_CALLBACK = new
SaslClientCallbackHandler();
+ private final SaslClientCallbackHandler SASL_CALLBACK = new
SaslClientCallbackHandler();
private static final Set SUPPORTED_RPC_FEATURES =
ImmutableSet.of(
RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS,
@@ -772,7 +772,7 @@ public class Negotiator extends
SimpleChannelUpstreamHandler {
// The UserInformationPB is deprecated, but used by servers prior to Kudu
1.1.
RpcHeader.UserInformationPB.Builder userBuilder =
RpcHeader.UserInformationPB.newBuilder();
-String user = System.getProperty("user.name");
+String user = securityContext.getRealUser();
userBuilder.setEffectiveUser(user);
userBuilder.setRealUser(user);
builder.setDEPRECATEDUserInfo(userBuilder.build());
@@ -826,11 +826,11 @@ public class Negotiator extends
SimpleChannelUpstreamHandler {
}
}
- private static class SaslClientCallbackHandler implements CallbackHandler {
+ private class SaslClientCallbackHandler implements CallbackHandler {
public void handle(Callback[] callbacks) throws
UnsupportedCallbackException {
for (Callback callback : callbacks) {
if (callback instanceof NameCallback) {
- ((NameCallback) callback).setName(System.getProperty("user.name"));
+ ((NameCallback) callback).setName(securityContext.getRealUse
kudu git commit: KUDU-2259: add real user to AuthenticationCredentialsPB
Repository: kudu
Updated Branches:
refs/heads/master 30cf90028 -> e684de337
KUDU-2259: add real user to AuthenticationCredentialsPB
This commit adds the 'real user' to the authn credentials token, which
is used when negotiating connections with SASL PLAIN authentication.
This is useful when scan tokens are being sent to remote tasks, it's not
possible to authenticate with a signed authn token to the remote
server[1], coarse-grained ACLs have been set, and the 'planner' and
'executor' processes are being run with different users.
This problematic scenario might also have been solved by allowing tokens
to be used in all scenarios, even when encryption is disabled, but the
approach taken by this commit allows that invariant to remain.
[1]: this most often occurs because the remote server has encryption disabled.
Change-Id: I5d2d901d42501ecfc0f6372f68cf7335eb188b45
Reviewed-on: http://gerrit.cloudera.org:8080/9374
Reviewed-by: Todd Lipcon
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e684de33
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e684de33
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e684de33
Branch: refs/heads/master
Commit: e684de3371941cc5ae8fc4a546ecda7dbe9f4f2f
Parents: 30cf900
Author: Dan Burkert
Authored: Tue Feb 20 17:37:17 2018 -0800
Committer: Dan Burkert
Committed: Thu Mar 15 02:47:42 2018 +
--
.../java/org/apache/kudu/client/Negotiator.java | 8 +-
.../org/apache/kudu/client/SecurityContext.java | 47 ++
.../client/TestSecurityContextRealUser.java | 97
src/kudu/client/client-internal.cc | 2 +
src/kudu/client/client-internal.h | 5 +
src/kudu/client/client-test.cc | 36
src/kudu/client/client.cc | 23 -
src/kudu/client/client.proto| 7 +-
src/kudu/client/master_rpc.cc | 10 ++
src/kudu/client/master_rpc.h| 5 +
src/kudu/client/meta_cache.cc | 1 +
src/kudu/mini-cluster/external_mini_cluster.cc | 6 +-
src/kudu/rpc/connection_id.cc | 1 +
src/kudu/rpc/server_negotiation.cc | 6 +-
src/kudu/rpc/user_credentials.cc| 7 +-
src/kudu/rpc/user_credentials.h | 5 +
src/kudu/util/user.cc | 2 +-
17 files changed, 235 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/e684de33/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
--
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
index 95ad907..47e4854 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java
@@ -92,7 +92,7 @@ import org.apache.kudu.util.SecurityUtil;
public class Negotiator extends SimpleChannelUpstreamHandler {
private static final Logger LOG = LoggerFactory.getLogger(Negotiator.class);
- private static final SaslClientCallbackHandler SASL_CALLBACK = new
SaslClientCallbackHandler();
+ private final SaslClientCallbackHandler SASL_CALLBACK = new
SaslClientCallbackHandler();
private static final Set SUPPORTED_RPC_FEATURES =
ImmutableSet.of(
RpcHeader.RpcFeatureFlag.APPLICATION_FEATURE_FLAGS,
@@ -772,7 +772,7 @@ public class Negotiator extends
SimpleChannelUpstreamHandler {
// The UserInformationPB is deprecated, but used by servers prior to Kudu
1.1.
RpcHeader.UserInformationPB.Builder userBuilder =
RpcHeader.UserInformationPB.newBuilder();
-String user = System.getProperty("user.name");
+String user = securityContext.getRealUser();
userBuilder.setEffectiveUser(user);
userBuilder.setRealUser(user);
builder.setDEPRECATEDUserInfo(userBuilder.build());
@@ -826,11 +826,11 @@ public class Negotiator extends
SimpleChannelUpstreamHandler {
}
}
- private static class SaslClientCallbackHandler implements CallbackHandler {
+ private class SaslClientCallbackHandler implements CallbackHandler {
public void handle(Callback[] callbacks) throws
UnsupportedCallbackException {
for (Callback callback : callbacks) {
if (callback instanceof NameCallback) {
- ((NameCallback) callback).setName(System.getProperty("user.name"));
+ ((NameCallback) callback).setName(securityContext.getRealUser());
} else if (callback instanceof PasswordCallback) {
((PasswordCallback) callback).setPassword(new char[0]);
} else {
http://git-wip-us.apache.org/
kudu git commit: KUDU-2343. java: properly reconnect to new leader master after failover
Repository: kudu
Updated Branches:
refs/heads/branch-1.5.x 8c9dcc0ee -> 011a70b91
KUDU-2343. java: properly reconnect to new leader master after failover
This fixes the "fake" location information returned in response to a
ConnectToMaster RPC to include a distinct "fake UUID" for each master.
Previously, we were using an empty string for the UUID of the masters.
This caused collisions in the ConnectionCache, which is keyed by server
UUIDs.
The fake UUID added by this patch matches the fake UUID already in use
by AsyncKuduClient.newMasterRpcProxy. This should allow us to share the
RPC connection between the ConnectToMaster RPCs and the subsequent
GetTableLocation RPCs, which is also a benefit for latency after a
failover or on a fresh client.
Additionally, this will help with various log messages that previously
would print an empty UUID string.
A prior version of this patch solved the problem by changing the key for
the ConnectionCache to be based on IP address, which has other benefits
in terms of future support for servers changing their DNS resolution at
runtime. However, since this patch is intended for backport into prior
releases, this simpler approach is taken for now. A TODO is added for
the longer-term idea.
An existing test which tested killing a master now runs in a second mode
which restarts the master. This reproduced the bug prior to the fix.
This patch also cleans up that test somewhat - it was doing some buggy
logic to attempt to kill more than one tablet server, but in fact just
called "killTabletServer" three times on the same one. Killing three
tablet servers never made sense, either, since the table in the test
only had three replicas. Neither did it make sense to start six tablet
servers for the test.
Change-Id: I36f96c6712800e398ed46887d97d4b09fd993b04
Reviewed-on: http://gerrit.cloudera.org:8080/9612
Reviewed-by: Alexey Serbin
Tested-by: Todd Lipcon
Reviewed-on: http://gerrit.cloudera.org:8080/9638
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/011a70b9
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/011a70b9
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/011a70b9
Branch: refs/heads/branch-1.5.x
Commit: 011a70b911e75068aa0cafc2e96cc2a49bc1caf2
Parents: 8c9dcc0
Author: Todd Lipcon
Authored: Tue Mar 13 11:43:47 2018 -0700
Committer: Todd Lipcon
Committed: Wed Mar 14 23:46:13 2018 +
--
.../org/apache/kudu/client/AsyncKuduClient.java | 6 +-
.../kudu/client/ConnectToClusterResponse.java | 3 +-
.../org/apache/kudu/client/ConnectionCache.java | 3 +
.../kudu/client/TestClientFailoverSupport.java | 64 +++-
.../apache/kudu/client/TestConnectionCache.java | 6 +-
5 files changed, 63 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/011a70b9/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
--
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index aec75e7..1292d20 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -279,7 +279,11 @@ public class AsyncKuduClient implements AutoCloseable {
return null;
}
return newRpcProxy(
-new ServerInfo("master-" + hostPort.toString(), hostPort,
inetAddress), credentialsPolicy);
+new ServerInfo(getFakeMasterUuid(hostPort), hostPort, inetAddress),
credentialsPolicy);
+ }
+
+ static String getFakeMasterUuid(HostAndPort hostPort) {
+return "master-" + hostPort.toString();
}
void reconnectToCluster(Callback cb,
http://git-wip-us.apache.org/repos/asf/kudu/blob/011a70b9/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
--
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
index 6cb687f..bfa1f2b 100644
---
a/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
+++
b/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
@@ -61,6 +61,7 @@ class ConnectToClusterResponse {
* if it were a tablet.
*/
public GetTableLocationsResponsePB getAsTableLocations() {
+String fakeUuid = AsyncKuduClient.getFakeMasterUuid(leaderHostAndPort);
return GetTableLocationsResponsePB.newBuilder()
.addTabletLocations(TabletLocationsPB.newBuilder()
.setPartition(PartitionPB.newBuilder()
[1/2] kudu git commit: KUDU-1704: add python client support for READ_YOUR_WRITES mode
Repository: kudu
Updated Branches:
refs/heads/master 84244461d -> 30cf90028
KUDU-1704: add python client support for READ_YOUR_WRITES mode
This patch allows users to specify READ_YOUR_WRITES as a read mode in
python client. It adds correponding tests to ensure the mode is actually
working.
Change-Id: I281a73ead2d606e698ff7f44ddb2cd1c78ffdd2a
Reviewed-on: http://gerrit.cloudera.org:8080/9617
Reviewed-by: David Ribeiro Alves
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9d233f45
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9d233f45
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9d233f45
Branch: refs/heads/master
Commit: 9d233f457af6bea7f30779fe0b9c812859d90a02
Parents: 8424446
Author: hahao
Authored: Tue Mar 13 17:01:04 2018 -0700
Committer: Hao Hao
Committed: Wed Mar 14 20:48:36 2018 +
--
python/kudu/__init__.py | 1 +
python/kudu/client.pyx | 9 ++---
python/kudu/libkudu_client.pxd | 1 +
python/kudu/tests/test_scanner.py | 12 +---
python/kudu/tests/test_scantoken.py | 17 ++---
5 files changed, 31 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/__init__.py
--
diff --git a/python/kudu/__init__.py b/python/kudu/__init__.py
index 75260d6..61c729d 100644
--- a/python/kudu/__init__.py
+++ b/python/kudu/__init__.py
@@ -27,6 +27,7 @@ from kudu.client import (Client, Table, Scanner, Session, #
noqa
FLUSH_MANUAL,
READ_LATEST,
READ_AT_SNAPSHOT,
+ READ_YOUR_WRITES,
EXCLUSIVE_BOUND,
INCLUSIVE_BOUND)
http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/client.pyx
--
diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx
index 34cb53a..ab43851 100644
--- a/python/kudu/client.pyx
+++ b/python/kudu/client.pyx
@@ -49,10 +49,12 @@ cdef dict _replica_selection_policies = {
# Read mode enums
READ_LATEST = ReadMode_Latest
READ_AT_SNAPSHOT = ReadMode_Snapshot
+READ_YOUR_WRITES = ReadMode_ReadYourWrites
cdef dict _read_modes = {
'latest': ReadMode_Latest,
-'snapshot': ReadMode_Snapshot
+'snapshot': ReadMode_Snapshot,
+'read_your_writes': ReadMode_ReadYourWrites
}
cdef dict _type_names = {
@@ -1557,8 +1559,9 @@ cdef class Scanner:
Parameters
--
-read_mode : {'latest', 'snapshot'}
- You can also use the constants READ_LATEST, READ_AT_SNAPSHOT
+read_mode : {'latest', 'snapshot', 'read_your_writes'}
+ You can also use the constants READ_LATEST, READ_AT_SNAPSHOT,
+ READ_YOUR_WRITES
Returns
---
http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/libkudu_client.pxd
--
diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd
index b834bf0..e2bc90d 100644
--- a/python/kudu/libkudu_client.pxd
+++ b/python/kudu/libkudu_client.pxd
@@ -496,6 +496,7 @@ cdef extern from "kudu/client/client.h" namespace
"kudu::client" nogil:
enum ReadMode" kudu::client::KuduScanner::ReadMode":
ReadMode_Latest " kudu::client::KuduScanner::READ_LATEST"
ReadMode_Snapshot " kudu::client::KuduScanner::READ_AT_SNAPSHOT"
+ReadMode_ReadYourWrites " kudu::client::KuduScanner::READ_YOUR_WRITES"
enum RangePartitionBound"
kudu::client::KuduTableCreator::RangePartitionBound":
PartitionType_Exclusive "
kudu::client::KuduTableCreator::EXCLUSIVE_BOUND"
http://git-wip-us.apache.org/repos/asf/kudu/blob/9d233f45/python/kudu/tests/test_scanner.py
--
diff --git a/python/kudu/tests/test_scanner.py
b/python/kudu/tests/test_scanner.py
index fa94ea3..23d5462 100644
--- a/python/kudu/tests/test_scanner.py
+++ b/python/kudu/tests/test_scanner.py
@@ -182,8 +182,7 @@ class TestScanner(TestScanBase):
def test_read_mode(self):
"""
-Test setting the read mode and scanning against a
-snapshot and latest
+Test scanning in latest, snapshot and read_your_writes read modes.
"""
# Delete row
self.delete_insert_row_for_read_test()
@@ -196,7 +195,7 @@ class TestScanner(TestScanBase):
self.assertEqual(sorted(self.tuples[1:]),
sorted(scanner.read_all_tuples()))
-#Check scanner results after delete
+# Check scanner results after delete with latest mode
timeout = time.
[3/3] kudu git commit: [release notes]: update notes for RYW read mode
[release notes]: update notes for RYW read mode Change-Id: I0bc5ae8452634a015891064bab5249eed6badf3e Reviewed-on: http://gerrit.cloudera.org:8080/9640 Tested-by: Hao Hao Reviewed-by: Grant Henke Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/132da394 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/132da394 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/132da394 Branch: refs/heads/branch-1.7.x Commit: 132da394d630bf3748b6b35aada3ae5efd780404 Parents: 759e9ab Author: hahao Authored: Wed Mar 14 11:06:15 2018 -0700 Committer: Grant Henke Committed: Wed Mar 14 23:14:15 2018 + -- docs/release_notes.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/132da394/docs/release_notes.adoc -- diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 1c7f7c5..522f868 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -80,7 +80,7 @@ automatically move existing metadata based on this configuration. * Kudu 1.7 introduces a new scan read mode READ_YOUR_WRITES. Users can specify - READ_YOUR_WRITES when creating a new scanner in both C++ and Java clients. + READ_YOUR_WRITES when creating a new scanner in C++, Java and Python clients. If this mode is used, the client will perform a read such that it follows all previously known writes and reads from this client. Reads in this mode ensure read-your-writes and read-your-reads session guarantees, while minimizing
[2/3] kudu git commit: KUDU-1704: add python client support for READ_YOUR_WRITES mode
KUDU-1704: add python client support for READ_YOUR_WRITES mode
This patch allows users to specify READ_YOUR_WRITES as a read mode in
python client. It adds correponding tests to ensure the mode is actually
working.
Change-Id: I281a73ead2d606e698ff7f44ddb2cd1c78ffdd2a
Reviewed-on: http://gerrit.cloudera.org:8080/9617
Reviewed-by: David Ribeiro Alves
Tested-by: Kudu Jenkins
(cherry picked from commit 9d233f457af6bea7f30779fe0b9c812859d90a02)
Reviewed-on: http://gerrit.cloudera.org:8080/9639
Tested-by: Hao Hao
Reviewed-by: Grant Henke
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/759e9ab6
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/759e9ab6
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/759e9ab6
Branch: refs/heads/branch-1.7.x
Commit: 759e9ab6c418f99c181046a1ebef8cd608b264c9
Parents: e6415bc
Author: hahao
Authored: Tue Mar 13 17:01:04 2018 -0700
Committer: Grant Henke
Committed: Wed Mar 14 23:14:08 2018 +
--
python/kudu/__init__.py | 1 +
python/kudu/client.pyx | 9 ++---
python/kudu/libkudu_client.pxd | 1 +
python/kudu/tests/test_scanner.py | 12 +---
python/kudu/tests/test_scantoken.py | 17 ++---
5 files changed, 31 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/759e9ab6/python/kudu/__init__.py
--
diff --git a/python/kudu/__init__.py b/python/kudu/__init__.py
index 75260d6..61c729d 100644
--- a/python/kudu/__init__.py
+++ b/python/kudu/__init__.py
@@ -27,6 +27,7 @@ from kudu.client import (Client, Table, Scanner, Session, #
noqa
FLUSH_MANUAL,
READ_LATEST,
READ_AT_SNAPSHOT,
+ READ_YOUR_WRITES,
EXCLUSIVE_BOUND,
INCLUSIVE_BOUND)
http://git-wip-us.apache.org/repos/asf/kudu/blob/759e9ab6/python/kudu/client.pyx
--
diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx
index 34cb53a..ab43851 100644
--- a/python/kudu/client.pyx
+++ b/python/kudu/client.pyx
@@ -49,10 +49,12 @@ cdef dict _replica_selection_policies = {
# Read mode enums
READ_LATEST = ReadMode_Latest
READ_AT_SNAPSHOT = ReadMode_Snapshot
+READ_YOUR_WRITES = ReadMode_ReadYourWrites
cdef dict _read_modes = {
'latest': ReadMode_Latest,
-'snapshot': ReadMode_Snapshot
+'snapshot': ReadMode_Snapshot,
+'read_your_writes': ReadMode_ReadYourWrites
}
cdef dict _type_names = {
@@ -1557,8 +1559,9 @@ cdef class Scanner:
Parameters
--
-read_mode : {'latest', 'snapshot'}
- You can also use the constants READ_LATEST, READ_AT_SNAPSHOT
+read_mode : {'latest', 'snapshot', 'read_your_writes'}
+ You can also use the constants READ_LATEST, READ_AT_SNAPSHOT,
+ READ_YOUR_WRITES
Returns
---
http://git-wip-us.apache.org/repos/asf/kudu/blob/759e9ab6/python/kudu/libkudu_client.pxd
--
diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd
index b834bf0..e2bc90d 100644
--- a/python/kudu/libkudu_client.pxd
+++ b/python/kudu/libkudu_client.pxd
@@ -496,6 +496,7 @@ cdef extern from "kudu/client/client.h" namespace
"kudu::client" nogil:
enum ReadMode" kudu::client::KuduScanner::ReadMode":
ReadMode_Latest " kudu::client::KuduScanner::READ_LATEST"
ReadMode_Snapshot " kudu::client::KuduScanner::READ_AT_SNAPSHOT"
+ReadMode_ReadYourWrites " kudu::client::KuduScanner::READ_YOUR_WRITES"
enum RangePartitionBound"
kudu::client::KuduTableCreator::RangePartitionBound":
PartitionType_Exclusive "
kudu::client::KuduTableCreator::EXCLUSIVE_BOUND"
http://git-wip-us.apache.org/repos/asf/kudu/blob/759e9ab6/python/kudu/tests/test_scanner.py
--
diff --git a/python/kudu/tests/test_scanner.py
b/python/kudu/tests/test_scanner.py
index fa94ea3..23d5462 100644
--- a/python/kudu/tests/test_scanner.py
+++ b/python/kudu/tests/test_scanner.py
@@ -182,8 +182,7 @@ class TestScanner(TestScanBase):
def test_read_mode(self):
"""
-Test setting the read mode and scanning against a
-snapshot and latest
+Test scanning in latest, snapshot and read_your_writes read modes.
"""
# Delete row
self.delete_insert_row_for_read_test()
@@ -196,7 +195,7 @@ class TestScanner(TestScanBase):
self.assertEqual(sorted(self.tuples[1:]),
sorted(scanner.read_all_tuples()))
-#Check scanner results after
[1/3] kudu git commit: KUDU-2153. Servers should not delete tmp files until after locking directories
Repository: kudu Updated Branches: refs/heads/branch-1.7.x 9f4b70f89 -> 132da394d KUDU-2153. Servers should not delete tmp files until after locking directories This changes the order of FsManager startup to not try to clean tmp files until after successfully locking the data directories. This prevents potential issues such as: - a tserver is already running on some host, and in the middle of consensus voting. Thus it has created a tmp file. - someone accidentally attempts to start a tserver with the same set of data dirs. Prior to this patch, it would delete the tmp file before realizing that it could not successfully lock its data dirs and aborting. - the original tserver would crash or otherwise get very confused because the tmp file it just wrote would be gone. This patch relies on the locking on the block manager instance files to provide exclusive access to some non-block-manager-related storage such as the consensus meta, etc. That means that it's still possible for someone to hit the above issue if they were to start servers with disjoint sets of data dirs but with the same meta dir. However, the patch is still a net improvement because the most likely scenario is that the two servers are started with identical configurations. This patch also removes the block_manager_lock_dirs flag which was apparently unused. It was always marked as 'unsafe' so it's not a compatibility issue to remove it without a deprecation period. Change-Id: I3a3471c8ce00e77fa1712ea518f6ab281864a08d Reviewed-on: http://gerrit.cloudera.org:8080/9596 Reviewed-by: Adar Dembo Tested-by: Todd Lipcon (cherry picked from commit d88e5772ee323c8a305250c7d0aa0b49f67475dc) Reviewed-on: http://gerrit.cloudera.org:8080/9622 Reviewed-by: Grant Henke Tested-by: Grant Henke Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e6415bc5 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e6415bc5 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e6415bc5 Branch: refs/heads/branch-1.7.x Commit: e6415bc5863c895639d0c472b003f46334c8053d Parents: 9f4b70f Author: Todd Lipcon Authored: Mon Mar 12 18:25:22 2018 -0700 Committer: Grant Henke Committed: Wed Mar 14 23:13:54 2018 + -- src/kudu/fs/block_manager.cc | 5 - src/kudu/fs/file_block_manager.cc | 1 - src/kudu/fs/fs_manager-test.cc| 38 +++--- src/kudu/fs/fs_manager.cc | 19 + src/kudu/fs/log_block_manager.cc | 1 - src/kudu/util/env_posix.cc| 9 6 files changed, 49 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/e6415bc5/src/kudu/fs/block_manager.cc -- diff --git a/src/kudu/fs/block_manager.cc b/src/kudu/fs/block_manager.cc index d3ce1c0..71a3075 100644 --- a/src/kudu/fs/block_manager.cc +++ b/src/kudu/fs/block_manager.cc @@ -49,11 +49,6 @@ DEFINE_string(block_manager_preflush_control, "finalize", "never be pre-flushed but still be flushed when closed."); TAG_FLAG(block_manager_preflush_control, experimental); -DEFINE_bool(block_manager_lock_dirs, true, -"Lock the data block directories to prevent concurrent usage. " -"Note that read-only concurrent usage is still allowed."); -TAG_FLAG(block_manager_lock_dirs, unsafe); - DEFINE_int64(block_manager_max_open_files, -1, "Maximum number of open file descriptors to be used for data " "blocks. If -1, Kudu will use 40% of its resource limit as per " http://git-wip-us.apache.org/repos/asf/kudu/blob/e6415bc5/src/kudu/fs/file_block_manager.cc -- diff --git a/src/kudu/fs/file_block_manager.cc b/src/kudu/fs/file_block_manager.cc index d5452f2..54ae7fa 100644 --- a/src/kudu/fs/file_block_manager.cc +++ b/src/kudu/fs/file_block_manager.cc @@ -65,7 +65,6 @@ using std::unique_ptr; using std::vector; using strings::Substitute; -DECLARE_bool(block_manager_lock_dirs); DECLARE_bool(enable_data_block_fsync); DECLARE_string(block_manager_preflush_control); http://git-wip-us.apache.org/repos/asf/kudu/blob/e6415bc5/src/kudu/fs/fs_manager-test.cc -- diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc index e21fe2c..5c0b3a5 100644 --- a/src/kudu/fs/fs_manager-test.cc +++ b/src/kudu/fs/fs_manager-test.cc @@ -18,7 +18,6 @@ #include #include -#include #include #include #include @@ -66,6 +65,7 @@ DECLARE_bool(crash_on_eio); DECLARE_double(env_inject_eio); DECLARE_string(block_manager); DECLARE_string(env_inject_eio_globs); +DECLARE_string(env_inject_lock_failure_globs); DECLARE_string(umask);
[2/2] kudu git commit: [release notes]: update notes for RYW read mode
[release notes]: update notes for RYW read mode Change-Id: I0bc5ae8452634a015891064bab5249eed6badf3e Reviewed-on: http://gerrit.cloudera.org:8080/9634 Tested-by: Hao Hao Reviewed-by: David Ribeiro Alves Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/30cf9002 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/30cf9002 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/30cf9002 Branch: refs/heads/master Commit: 30cf9002879c48e190260d115d0e5dacd0fad8bb Parents: 9d233f4 Author: hahao Authored: Wed Mar 14 11:06:15 2018 -0700 Committer: Hao Hao Committed: Wed Mar 14 22:15:33 2018 + -- docs/release_notes.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/30cf9002/docs/release_notes.adoc -- diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 1c7f7c5..522f868 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -80,7 +80,7 @@ automatically move existing metadata based on this configuration. * Kudu 1.7 introduces a new scan read mode READ_YOUR_WRITES. Users can specify - READ_YOUR_WRITES when creating a new scanner in both C++ and Java clients. + READ_YOUR_WRITES when creating a new scanner in C++, Java and Python clients. If this mode is used, the client will perform a read such that it follows all previously known writes and reads from this client. Reads in this mode ensure read-your-writes and read-your-reads session guarantees, while minimizing
kudu git commit: [release_notes] replica management scheme notes
Repository: kudu Updated Branches: refs/heads/branch-1.7.x 29d86fc83 -> 9f4b70f89 [release_notes] replica management scheme notes Added relevant notes on the new replica management scheme used in Kudu 1.7 by default: * the new replica management scheme is incompatible with old one * rolling upgrade 1.6 -> 1.7 is not possible Change-Id: I49f1f1e17cdaee272592d598431a33dbfe55123f Reviewed-on: http://gerrit.cloudera.org:8080/9571 Tested-by: Kudu Jenkins Reviewed-by: Grant Henke (cherry picked from commit 84244461d0aa5df1c041869ecbd2d18a0cca7659) Reviewed-on: http://gerrit.cloudera.org:8080/9630 Tested-by: Grant Henke Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/9f4b70f8 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/9f4b70f8 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/9f4b70f8 Branch: refs/heads/branch-1.7.x Commit: 9f4b70f899ae65f262b9589b57c21bcecdceba83 Parents: 29d86fc Author: Alexey Serbin Authored: Fri Mar 9 16:40:13 2018 -0800 Committer: Grant Henke Committed: Wed Mar 14 16:10:42 2018 + -- docs/release_notes.adoc | 40 ++-- 1 file changed, 38 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/9f4b70f8/docs/release_notes.adoc -- diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 740c64d..1c7f7c5 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -32,8 +32,13 @@ == Upgrade Notes * Upgrading directly from Kudu 1.6.0 is supported and no special upgrade steps - are required. A rolling upgrade may work, however it has not been tested. - When upgrading Kudu, it is recommended to first shut down all Kudu processes + are required. A rolling upgrade of the server side will _not_ work because + the default replica management scheme changed, and running masters and tablet + servers with different replica management schemes is not supported, see + <> for details. However, mixing client and + server sides of different versions is not a problem, i.e. you can still + update your clients before your servers or vice versa. + When upgrading to Kudu 1.7, it is required to first shut down all Kudu processes across the cluster, then upgrade the software on all servers, then restart the Kudu processes on all servers in the cluster. @@ -89,6 +94,16 @@ reporting changes have been made to make various common scenarios, particularly tablet copies, less alarming. +* KUDU-1097: a new replica management scheme is implemented and enabled by + default. With the new replica management scheme, the system first adds a + replacement tablet replica before evicting the failed one. With the previous + replica management scheme, the system first evicts the failed replica and + then adds a replacement. The new replica management scheme allows for much + faster recovery of tablets in scenarios where one tablet server goes down and + then returns back shortly after 5 minutes or so. To switch back to the old + scheme, set the `--raft_prepare_replacement_before_eviction` run-time flag to + `false` for *all* tablet servers and masters in Kudu 1.7 cluster. + [[rn_1.7.0_fixed_issues]] == Fixed Issues @@ -123,6 +138,27 @@ on wire compatibility between Kudu 1.7 and versions earlier than 1.3: [[rn_1.7.0_incompatible_changes]] == Incompatible Changes in Kudu 1.7.0 +* The newly introduced replica management scheme is not compatible with the + old scheme, so it's not possible to run pre-1.7 Kudu masters with + 1.7 Kudu tablet servers and vice versa, unless setting the run-time flag + `--raft_prepare_replacement_before_eviction` to `false` for 1.7 masters + and tablet servers. In essence, tablet servers cannot register with masters + running with different replica management scheme. This is the server-side + incompatibility only and it does not affect the client side. In other words, + Kudu clients of prior versions are compatible with the Kudu server side + running with either scheme, assuming the same replica management scheme + is used by all masters and tablet servers in the Kudu cluster. +** Kudu masters of 1.7 version will not register Kudu tablet servers of 1.6 +and prior revisions. To run 1.7 masters with the old scheme, set the +`--raft_prepare_replacement_before_eviction` to `false`. +** Kudu tablet servers of 1.7 version will not work with Kudu masters of 1.6 +and prior versions. To make the case of such misconfiguration easily +detectable, Kudu tablet servers of 1.7 version crash when they detect their +masters running with different replica management scheme. The crashing of +tablet servers in such scenarios can be disabled by setting their +`-
[1/2] kudu git commit: Update version to 1.8.0-SNAPSHOT
Repository: kudu Updated Branches: refs/heads/master 4c2bb92f1 -> 84244461d Update version to 1.8.0-SNAPSHOT Change-Id: I6552ec90cab4de52a88b1d3df7170ec6f3623140 Reviewed-on: http://gerrit.cloudera.org:8080/9628 Reviewed-by: Todd Lipcon Tested-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/706b3c3b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/706b3c3b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/706b3c3b Branch: refs/heads/master Commit: 706b3c3b186b5d196d396fbcb66b8ad005084b25 Parents: 4c2bb92 Author: Grant Henke Authored: Wed Mar 14 09:06:13 2018 -0500 Committer: Grant Henke Committed: Wed Mar 14 16:07:50 2018 + -- java/gradle.properties | 2 +- java/kudu-client-tools/pom.xml | 2 +- java/kudu-client/pom.xml | 2 +- java/kudu-flume-sink/pom.xml | 2 +- java/kudu-hive/pom.xml | 2 +- java/kudu-jepsen/pom.xml | 2 +- java/kudu-mapreduce/pom.xml| 2 +- java/kudu-spark-tools/pom.xml | 2 +- java/kudu-spark/pom.xml| 2 +- java/pom.xml | 2 +- python/setup.py| 2 +- version.txt| 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/gradle.properties -- diff --git a/java/gradle.properties b/java/gradle.properties index d00a394..2d2ce50 100755 --- a/java/gradle.properties +++ b/java/gradle.properties @@ -20,7 +20,7 @@ # https://docs.gradle.org/current/userguide/build_environment.html#sec:gradle_configuration_properties group = org.apache.kudu -version = 1.7.0-SNAPSHOT +version = 1.8.0-SNAPSHOT url = https://kudu.apache.org/ scmUrl = git://git.apache.org/kudu.git issueTrackerUrl = https://issues.apache.org/jira/projects/KUDU http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-client-tools/pom.xml -- diff --git a/java/kudu-client-tools/pom.xml b/java/kudu-client-tools/pom.xml index 4294617..d72b9cb 100644 --- a/java/kudu-client-tools/pom.xml +++ b/java/kudu-client-tools/pom.xml @@ -23,7 +23,7 @@ org.apache.kudu kudu-parent -1.7.0-SNAPSHOT +1.8.0-SNAPSHOT kudu-client-tools http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-client/pom.xml -- diff --git a/java/kudu-client/pom.xml b/java/kudu-client/pom.xml index 6ab6ae3..7481a8e 100644 --- a/java/kudu-client/pom.xml +++ b/java/kudu-client/pom.xml @@ -23,7 +23,7 @@ org.apache.kudu kudu-parent -1.7.0-SNAPSHOT +1.8.0-SNAPSHOT kudu-client http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-flume-sink/pom.xml -- diff --git a/java/kudu-flume-sink/pom.xml b/java/kudu-flume-sink/pom.xml index 376250c..ff23150 100644 --- a/java/kudu-flume-sink/pom.xml +++ b/java/kudu-flume-sink/pom.xml @@ -15,7 +15,7 @@ kudu-parent org.apache.kudu -1.7.0-SNAPSHOT +1.8.0-SNAPSHOT kudu-flume-sink Kudu Flume NG Sink http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-hive/pom.xml -- diff --git a/java/kudu-hive/pom.xml b/java/kudu-hive/pom.xml index 2ebdbea..e247db7 100644 --- a/java/kudu-hive/pom.xml +++ b/java/kudu-hive/pom.xml @@ -22,7 +22,7 @@ org.apache.kudu kudu-parent -1.7.0-SNAPSHOT +1.8.0-SNAPSHOT kudu-hive http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-jepsen/pom.xml -- diff --git a/java/kudu-jepsen/pom.xml b/java/kudu-jepsen/pom.xml index f611d8d..4de2418 100644 --- a/java/kudu-jepsen/pom.xml +++ b/java/kudu-jepsen/pom.xml @@ -20,7 +20,7 @@ org.apache.kudu kudu-parent -1.7.0-SNAPSHOT +1.8.0-SNAPSHOT kudu-jepsen http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-mapreduce/pom.xml -- diff --git a/java/kudu-mapreduce/pom.xml b/java/kudu-mapreduce/pom.xml index 2265bc7..b3f6a27 100644 --- a/java/kudu-mapreduce/pom.xml +++ b/java/kudu-mapreduce/pom.xml @@ -23,7 +23,7 @@ org.apache.kudu kudu-parent -1.7.0-SNAPSHOT +1.8.0-SNAPSHOT kudu-mapreduce http://git-wip-us.apache.org/repos/asf/kudu/blob/706b3c3b/java/kudu-spark-tools/pom.xml -
[2/2] kudu git commit: [release_notes] replica management scheme notes
[release_notes] replica management scheme notes Added relevant notes on the new replica management scheme used in Kudu 1.7 by default: * the new replica management scheme is incompatible with old one * rolling upgrade 1.6 -> 1.7 is not possible Change-Id: I49f1f1e17cdaee272592d598431a33dbfe55123f Reviewed-on: http://gerrit.cloudera.org:8080/9571 Tested-by: Kudu Jenkins Reviewed-by: Grant Henke Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/84244461 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/84244461 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/84244461 Branch: refs/heads/master Commit: 84244461d0aa5df1c041869ecbd2d18a0cca7659 Parents: 706b3c3 Author: Alexey Serbin Authored: Fri Mar 9 16:40:13 2018 -0800 Committer: Grant Henke Committed: Wed Mar 14 16:09:40 2018 + -- docs/release_notes.adoc | 40 ++-- 1 file changed, 38 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/84244461/docs/release_notes.adoc -- diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 740c64d..1c7f7c5 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -32,8 +32,13 @@ == Upgrade Notes * Upgrading directly from Kudu 1.6.0 is supported and no special upgrade steps - are required. A rolling upgrade may work, however it has not been tested. - When upgrading Kudu, it is recommended to first shut down all Kudu processes + are required. A rolling upgrade of the server side will _not_ work because + the default replica management scheme changed, and running masters and tablet + servers with different replica management schemes is not supported, see + <> for details. However, mixing client and + server sides of different versions is not a problem, i.e. you can still + update your clients before your servers or vice versa. + When upgrading to Kudu 1.7, it is required to first shut down all Kudu processes across the cluster, then upgrade the software on all servers, then restart the Kudu processes on all servers in the cluster. @@ -89,6 +94,16 @@ reporting changes have been made to make various common scenarios, particularly tablet copies, less alarming. +* KUDU-1097: a new replica management scheme is implemented and enabled by + default. With the new replica management scheme, the system first adds a + replacement tablet replica before evicting the failed one. With the previous + replica management scheme, the system first evicts the failed replica and + then adds a replacement. The new replica management scheme allows for much + faster recovery of tablets in scenarios where one tablet server goes down and + then returns back shortly after 5 minutes or so. To switch back to the old + scheme, set the `--raft_prepare_replacement_before_eviction` run-time flag to + `false` for *all* tablet servers and masters in Kudu 1.7 cluster. + [[rn_1.7.0_fixed_issues]] == Fixed Issues @@ -123,6 +138,27 @@ on wire compatibility between Kudu 1.7 and versions earlier than 1.3: [[rn_1.7.0_incompatible_changes]] == Incompatible Changes in Kudu 1.7.0 +* The newly introduced replica management scheme is not compatible with the + old scheme, so it's not possible to run pre-1.7 Kudu masters with + 1.7 Kudu tablet servers and vice versa, unless setting the run-time flag + `--raft_prepare_replacement_before_eviction` to `false` for 1.7 masters + and tablet servers. In essence, tablet servers cannot register with masters + running with different replica management scheme. This is the server-side + incompatibility only and it does not affect the client side. In other words, + Kudu clients of prior versions are compatible with the Kudu server side + running with either scheme, assuming the same replica management scheme + is used by all masters and tablet servers in the Kudu cluster. +** Kudu masters of 1.7 version will not register Kudu tablet servers of 1.6 +and prior revisions. To run 1.7 masters with the old scheme, set the +`--raft_prepare_replacement_before_eviction` to `false`. +** Kudu tablet servers of 1.7 version will not work with Kudu masters of 1.6 +and prior versions. To make the case of such misconfiguration easily +detectable, Kudu tablet servers of 1.7 version crash when they detect their +masters running with different replica management scheme. The crashing of +tablet servers in such scenarios can be disabled by setting their +`--heartbeat_incompatible_replica_management_is_fatal` run-time flag to +`false`. + [[rn_1.7.0_client_compatibility]] === Client Library Compatibility
kudu git commit: KUDU-2343. java: properly reconnect to new leader master after failover
Repository: kudu
Updated Branches:
refs/heads/branch-1.6.x 3aa933191 -> f8ae5117b
KUDU-2343. java: properly reconnect to new leader master after failover
This fixes the "fake" location information returned in response to a
ConnectToMaster RPC to include a distinct "fake UUID" for each master.
Previously, we were using an empty string for the UUID of the masters.
This caused collisions in the ConnectionCache, which is keyed by server
UUIDs.
The fake UUID added by this patch matches the fake UUID already in use
by AsyncKuduClient.newMasterRpcProxy. This should allow us to share the
RPC connection between the ConnectToMaster RPCs and the subsequent
GetTableLocation RPCs, which is also a benefit for latency after a
failover or on a fresh client.
Additionally, this will help with various log messages that previously
would print an empty UUID string.
A prior version of this patch solved the problem by changing the key for
the ConnectionCache to be based on IP address, which has other benefits
in terms of future support for servers changing their DNS resolution at
runtime. However, since this patch is intended for backport into prior
releases, this simpler approach is taken for now. A TODO is added for
the longer-term idea.
An existing test which tested killing a master now runs in a second mode
which restarts the master. This reproduced the bug prior to the fix.
This patch also cleans up that test somewhat - it was doing some buggy
logic to attempt to kill more than one tablet server, but in fact just
called "killTabletServer" three times on the same one. Killing three
tablet servers never made sense, either, since the table in the test
only had three replicas. Neither did it make sense to start six tablet
servers for the test.
Change-Id: I36f96c6712800e398ed46887d97d4b09fd993b04
Reviewed-on: http://gerrit.cloudera.org:8080/9612
Reviewed-by: Alexey Serbin
Tested-by: Todd Lipcon
(cherry picked from commit 4c2bb92f14e1346928ef16dacd63812602683ed2)
Reviewed-on: http://gerrit.cloudera.org:8080/9616
Reviewed-by: Grant Henke
Tested-by: Grant Henke
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f8ae5117
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f8ae5117
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f8ae5117
Branch: refs/heads/branch-1.6.x
Commit: f8ae5117b257887e6ef222f751ad7cca5640d4d9
Parents: 3aa9331
Author: Todd Lipcon
Authored: Tue Mar 13 11:43:47 2018 -0700
Committer: Todd Lipcon
Committed: Wed Mar 14 02:30:32 2018 +
--
.../org/apache/kudu/client/AsyncKuduClient.java | 6 +-
.../kudu/client/ConnectToClusterResponse.java | 3 +-
.../org/apache/kudu/client/ConnectionCache.java | 3 +
.../org/apache/kudu/client/MiniKuduCluster.java | 1 +
.../kudu/client/TestClientFailoverSupport.java | 65 +++-
.../apache/kudu/client/TestConnectionCache.java | 6 +-
6 files changed, 65 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/f8ae5117/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
--
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index f23b342..50f695b 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -270,7 +270,11 @@ public class AsyncKuduClient implements AutoCloseable {
return null;
}
return newRpcProxy(
-new ServerInfo("master-" + hostPort.toString(), hostPort,
inetAddress), credentialsPolicy);
+new ServerInfo(getFakeMasterUuid(hostPort), hostPort, inetAddress),
credentialsPolicy);
+ }
+
+ static String getFakeMasterUuid(HostAndPort hostPort) {
+return "master-" + hostPort.toString();
}
void reconnectToCluster(Callback cb,
http://git-wip-us.apache.org/repos/asf/kudu/blob/f8ae5117/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
--
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
index 6cb687f..bfa1f2b 100644
---
a/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
+++
b/java/kudu-client/src/main/java/org/apache/kudu/client/ConnectToClusterResponse.java
@@ -61,6 +61,7 @@ class ConnectToClusterResponse {
* if it were a tablet.
*/
public GetTableLocationsResponsePB getAsTableLocations() {
+String fakeUuid = AsyncKuduClient.getFakeMasterUuid(leaderHos
