kudu git commit: Add thrift module for common thrift utilities
Repository: kudu
Updated Branches:
refs/heads/master 6129467d9 -> ddaa766e2
Add thrift module for common thrift utilities
The HMS patch series inlined all necessary Thrift utilities into the hms
module, since it was the only use of Thrift in the codebase. Now that
we're also planning on having a Sentry client it makes sense to properly
abstract the common Thrift code into its own shared module.
Change-Id: I6f6f843f42b37cb1170df03da01fc0790fe94acb
Reviewed-on: http://gerrit.cloudera.org:8080/11493
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ddaa766e
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ddaa766e
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ddaa766e
Branch: refs/heads/master
Commit: ddaa766e2fdf177a86ed3eb57e6b917730b75843
Parents: 6129467
Author: Dan Burkert
Authored: Fri Sep 21 12:40:18 2018 -0700
Committer: Dan Burkert
Committed: Sat Sep 22 00:39:23 2018 +
--
CMakeLists.txt | 1 +
src/kudu/hms/CMakeLists.txt | 4 +-
src/kudu/hms/hms_catalog-test.cc| 3 +-
src/kudu/hms/hms_catalog.cc | 14 +-
src/kudu/hms/hms_catalog.h | 2 -
src/kudu/hms/hms_client-test.cc | 11 +-
src/kudu/hms/hms_client.cc | 66 +--
src/kudu/hms/hms_client.h | 27 +-
src/kudu/hms/sasl_client_transport.cc | 402 ---
src/kudu/hms/sasl_client_transport.h| 176
src/kudu/integration-tests/master_hms-itest.cc | 4 +-
.../mini-cluster/external_mini_cluster-test.cc | 3 +-
src/kudu/thrift/CMakeLists.txt | 33 ++
src/kudu/thrift/client.cc | 85
src/kudu/thrift/client.h| 65 +++
src/kudu/thrift/sasl_client_transport.cc| 402 +++
src/kudu/thrift/sasl_client_transport.h | 176
src/kudu/tools/kudu-tool-test.cc| 10 +-
18 files changed, 806 insertions(+), 678 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/CMakeLists.txt
--
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6c9d35e..df02893 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1477,6 +1477,7 @@ add_subdirectory(src/kudu/security)
add_subdirectory(src/kudu/sentry)
add_subdirectory(src/kudu/server)
add_subdirectory(src/kudu/tablet)
+add_subdirectory(src/kudu/thrift)
add_subdirectory(src/kudu/tools)
add_subdirectory(src/kudu/tserver)
add_subdirectory(src/kudu/util)
http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/src/kudu/hms/CMakeLists.txt
--
diff --git a/src/kudu/hms/CMakeLists.txt b/src/kudu/hms/CMakeLists.txt
index 90794a6..287307d 100644
--- a/src/kudu/hms/CMakeLists.txt
+++ b/src/kudu/hms/CMakeLists.txt
@@ -34,14 +34,14 @@ add_dependencies(hms_thrift ${HMS_THRIFT_TGTS})
set(HMS_SRCS
hms_catalog.cc
- hms_client.cc
- sasl_client_transport.cc)
+ hms_client.cc)
set(HMS_DEPS
gflags
glog
hms_thrift
krpc
kudu_common
+ kudu_thrift
kudu_util)
add_library(kudu_hms ${HMS_SRCS})
http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index cf9977b..a2bfabf 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -37,6 +37,7 @@
#include "kudu/hms/mini_hms.h"
#include "kudu/rpc/sasl_common.h"
#include "kudu/security/test/mini_kdc.h"
+#include "kudu/thrift/client.h"
#include "kudu/util/net/net_util.h"
#include "kudu/util/slice.h"
#include "kudu/util/status.h"
@@ -161,7 +162,7 @@ class HmsCatalogTest : public KuduTest {
void SetUp() override {
bool enable_kerberos = EnableKerberos();
-HmsClientOptions hms_client_opts;
+thrift::ClientOptions hms_client_opts;
hms_.reset(new hms::MiniHms());
if (enable_kerberos) {
http://git-wip-us.apache.org/repos/asf/kudu/blob/ddaa766e/src/kudu/hms/hms_catalog.cc
--
diff --git a/src/kudu/hms/hms_catalog.cc b/src/kudu/hms/hms_catalog.cc
index f62f9ba..aeaf226 100644
--- a/src/kudu/hms/hms_catalog.cc
+++ b/src/kudu/hms/hms_catalog.cc
@@ -42,6 +42,7 @@
#include "kudu/gutil/strings/substitute.h"
#include "kudu/hms/hive_metastore_types.h"
#include "kudu/hms/hms_client.h"
+#include "kudu/thrift/client.h"
#include "kudu/util/async_util.h"
#include "kudu/util/flag_tags.h"
#include
kudu git commit: KUDU-2463 pt 1: adjust MVCC when replaying no-ops
Repository: kudu
Updated Branches:
refs/heads/master 34e88d3da -> 6129467d9
KUDU-2463 pt 1: adjust MVCC when replaying no-ops
Previously, during tablet bootstrap, a tablet replica would only update
its MVCC safetime based on write and alter schema messages, as the
timestamps in these messages are guaranteed to be serialized with
respect to one another, by virtue of being assigned in a single thread
(the prepare thread) on the leader replica.
>From this, we conclude that timestamps for write and alter schema
operations are monotonically increasing in unison with opid. Because
no-op and change config operations are not serialized through the
prepare thread, they don't necessarily provide the same timestamp
monotonicity guarantees.
However, in the case of no-ops replicated to assert leadership, we get
the same timestamp monotonicity guarantee due to a different mechanism.
This patch takes advantage of the fact that our Raft implementation
ensures the following sequence of events:
1. replica A becomes leader of Term N
2. leader A assigns a timestamp t1 to its no-op
3. leader A replicates the no-op to replicas B and C, asserting its
leadership for Term N
4. leader A prepares a write and assigns it a timestamp t2 > t1
5. leader A replicates the write to replicas B and C, checking that it
is leader for the Term N
Given the above series of operations, for a given term, the no-op used
to assert leadership is always assigned a timestamp that must be lower
than any other ops in that term. As such, the timestamps assigned to
no-ops can and should be used to bump MVCC safe time. This patch does so
at bootstrap time with committed no-ops found in the WAL.
This alone isn't enough to always prevent KUDU-2463, but it dramatically
reduces the likelihood of it occuring.
Change-Id: I26deff32da8c990cb8a2ba220bb81858ddd6d73f
Reviewed-on: http://gerrit.cloudera.org:8080/11142
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6129467d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6129467d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6129467d
Branch: refs/heads/master
Commit: 6129467d964a073f3e68474200452a8c3f639996
Parents: 34e88d3
Author: Andrew Wong
Authored: Tue Sep 11 12:09:58 2018 -0700
Committer: Mike Percy
Committed: Fri Sep 21 19:55:23 2018 +
--
src/kudu/consensus/consensus.proto | 6 +
src/kudu/integration-tests/CMakeLists.txt | 1 +
.../integration-tests/raft_consensus-itest.cc | 6 +-
.../timestamp_advancement-itest.cc | 236 +++
src/kudu/tablet/tablet_bootstrap-test.cc| 1 +
src/kudu/tablet/tablet_bootstrap.cc | 35 ++-
6 files changed, 276 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/6129467d/src/kudu/consensus/consensus.proto
--
diff --git a/src/kudu/consensus/consensus.proto
b/src/kudu/consensus/consensus.proto
index 5eeecdd..68877ca 100644
--- a/src/kudu/consensus/consensus.proto
+++ b/src/kudu/consensus/consensus.proto
@@ -229,6 +229,12 @@ message CommitMsg {
message NoOpRequestPB {
// Allows to set a dummy payload, for tests.
optional bytes payload_for_tests = 1;
+
+ // Set to true if the op id for this request is expected to be monotonically
+ // increasing with the assigned timestamp. For no-ops that are sent by a
+ // leader marking a successful Raft election, this is true. If not set, it is
+ // assumed to be true.
+ optional bool timestamp_in_opid_order = 2;
}
// Status message received in the peer responses.
http://git-wip-us.apache.org/repos/asf/kudu/blob/6129467d/src/kudu/integration-tests/CMakeLists.txt
--
diff --git a/src/kudu/integration-tests/CMakeLists.txt
b/src/kudu/integration-tests/CMakeLists.txt
index ad814e3..512a454 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -109,6 +109,7 @@ ADD_KUDU_TEST(tablet_copy-itest NUM_SHARDS 6 PROCESSORS 4)
ADD_KUDU_TEST(tablet_copy_client_session-itest)
ADD_KUDU_TEST(tablet_history_gc-itest)
ADD_KUDU_TEST(tablet_replacement-itest)
+ADD_KUDU_TEST(timestamp_advancement-itest)
ADD_KUDU_TEST(tombstoned_voting-imc-itest)
ADD_KUDU_TEST(tombstoned_voting-itest)
ADD_KUDU_TEST(tombstoned_voting-stress-test RUN_SERIAL true)
http://git-wip-us.apache.org/repos/asf/kudu/blob/6129467d/src/kudu/integration-tests/raft_consensus-itest.cc
--
diff --git a/src/kudu/integration-tests/raft_consensus-itest.cc
b/src/kudu/integration-tests/raft_consensus-itest.cc
index f7be807..20c3c7b 100644
---
kudu git commit: [test] Adjust Kudu binary locator logic.
Repository: kudu
Updated Branches:
refs/heads/master fd1ffd0fb -> 34e88d3da
[test] Adjust Kudu binary locator logic.
This patch adjusts the binary locating logic to be less
fragile and more externally useful. It also breaks the
logic out of the TestUtils class into itâs own class.
I broke out the remaining TestUtils methods as
well and remove any that were unused.
This patch also adjusts the Maven and Gradle build
to define the `kuduBinDir` system property. Doing
this means we donât need to detect git repos and
search the repository for binaries. This also fixes
the issue where Maven tests would fail to locate the
binary because they were using jars installed into
the local repository.
Change-Id: I365dfb9fdeeb4fd985f370a6b8305345e0a2ac7d
Reviewed-on: http://gerrit.cloudera.org:8080/11475
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo
Reviewed-by: Dan Burkert
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/34e88d3d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/34e88d3d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/34e88d3d
Branch: refs/heads/master
Commit: 34e88d3dafc421ccaabae5767aad2fd9fa015d39
Parents: fd1ffd0
Author: Grant Henke
Authored: Wed Sep 19 14:04:36 2018 -0500
Committer: Grant Henke
Committed: Fri Sep 21 16:07:20 2018 +
--
java/README.adoc| 2 +-
.../org/apache/kudu/gradle/DistTestTask.java| 2 +-
java/gradle/tests.gradle| 5 +-
.../org/apache/kudu/backup/TestKuduBackup.scala | 4 +-
.../org/apache/kudu/client/BaseKuduTest.java| 3 +-
.../org/apache/kudu/client/MiniKuduCluster.java | 3 +-
.../apache/kudu/client/TestAsyncKuduClient.java | 7 +-
.../apache/kudu/client/TestRemoteTablet.java| 5 +-
.../org/apache/kudu/client/TestTestUtils.java | 115 -
.../java/org/apache/kudu/client/TestUtils.java | 238 ---
.../org/apache/kudu/util/KuduBinaryLocator.java | 94
.../org/apache/kudu/util/ProtobufUtils.java | 64 +
.../java/org/apache/kudu/util/RandomUtils.java | 49
java/pom.xml| 6 +
14 files changed, 232 insertions(+), 365 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/34e88d3d/java/README.adoc
--
diff --git a/java/README.adoc b/java/README.adoc
index 23052c6..42da7e7 100644
--- a/java/README.adoc
+++ b/java/README.adoc
@@ -55,7 +55,7 @@ for Kudu, those should be present already.
If for some reason the binaries aren't in the expected location
as shown above, you can pass
-`-DbinDir=/path/to/directory`.
+`-DkuduBinDir=/path/to/directory`.
Once everything is setup correctly, run:
http://git-wip-us.apache.org/repos/asf/kudu/blob/34e88d3d/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
--
diff --git
a/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
b/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
index 3645017..32c5b24 100644
--- a/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
+++ b/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java
@@ -185,7 +185,7 @@ public class DistTestTask extends DefaultTask {
for (Map.Entry e : test.getSystemProperties().entrySet()) {
cmd.add("-D" + e.getKey() + "=" + e.getValue());
}
-cmd.add("-DbinDir=" + isolateFileDir.relativize(binDir),
+cmd.add("-DkuduBinDir=" + isolateFileDir.relativize(binDir),
"org.junit.runner.JUnitCore",
testClass);
http://git-wip-us.apache.org/repos/asf/kudu/blob/34e88d3d/java/gradle/tests.gradle
--
diff --git a/java/gradle/tests.gradle b/java/gradle/tests.gradle
index 19f85ee..d356ad0 100644
--- a/java/gradle/tests.gradle
+++ b/java/gradle/tests.gradle
@@ -57,9 +57,12 @@ tasks.withType(Test) {
systemProperty "java.net.preferIPv4Stack", true
systemProperty "java.security.egd", "file:/dev/urandom" // Improve RNG
generation speed.
- // Set rerunFailingTestsCount for use in BaseKuduTest.java to rerun failing
tests
+ // Set rerunFailingTestsCount for use in BaseKuduTest.java to rerun failing
tests.
systemProperty "rerunFailingTestsCount",
propertyWithDefault("rerunFailingTestsCount", 0)
+ // Set kuduBinDir to the binaries to use with the MiniKuduCluster.
+ systemProperty "kuduBinDir", "$project.rootDir/../build/latest/bin"
+
// Don't fail the build if a --tests filter doesn't match any tests.
// This is useful for filtering tests from the top of the project where some
// subprojects won't match the filter.
[4/4] kudu git commit: [test] Clean up MiniKuduCluster and BaseKuduTest
[test] Clean up MiniKuduCluster and BaseKuduTest
This patch cleans up the MiniKuduCluster and
BaseKuduTest API in preparation for creating a
public test API and improving per-test
MiniKuduCluster parameters.
This patch also removes the use of Guavaâs
HostAndPort as part of the clean up. It replaces
it with our own HostAndPort which wraps an
unresolved InetSocketAddress. We canât use
Guavaâs HostAndPort because it is shaded and
relocated in all of our libraries. This means it shouldnât
be a part of the âpublicâ API. In this case that means
the test API too. Additionally HostAndPort is marked
as beta.
Change-Id: I2a55c41f48f31c4a12b41508dbb343c4419e65b1
Reviewed-on: http://gerrit.cloudera.org:8080/11474
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fd1ffd0f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fd1ffd0f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fd1ffd0f
Branch: refs/heads/master
Commit: fd1ffd0fb65e138f1f015a55aa96ae870c1d51cd
Parents: e48157a
Author: Grant Henke
Authored: Wed Sep 19 12:47:38 2018 -0500
Committer: Grant Henke
Committed: Fri Sep 21 13:42:35 2018 +
--
.../org/apache/kudu/backup/TestKuduBackup.scala | 4 +-
.../kudu/mapreduce/tools/ITExportCsv.java | 2 +-
.../kudu/mapreduce/tools/ITImportCsv.java | 2 +-
.../kudu/mapreduce/tools/ITImportParquet.java | 2 +-
.../tools/ITImportParquetPreCheck.java | 2 +-
.../tools/ITIntegrationTestBigLinkedList.java | 2 +-
.../kudu/mapreduce/tools/ITRowCounter.java | 2 +-
.../org/apache/kudu/client/AsyncKuduClient.java | 5 +-
.../apache/kudu/client/ConnectToCluster.java| 9 +-
.../kudu/client/ConnectToClusterResponse.java | 3 +-
.../org/apache/kudu/client/HostAndPort.java | 74 +++
.../org/apache/kudu/client/ProtobufHelper.java | 11 +-
.../java/org/apache/kudu/client/ServerInfo.java | 1 -
.../main/java/org/apache/kudu/util/NetUtil.java | 13 +-
.../org/apache/kudu/client/BaseKuduTest.java| 140
.../java/org/apache/kudu/client/ITClient.java | 2 +-
.../org/apache/kudu/client/ITClientStress.java | 2 +-
.../org/apache/kudu/client/MiniKuduCluster.java | 213 ++-
.../apache/kudu/client/TestAsyncKuduClient.java | 2 +-
.../kudu/client/TestAsyncKuduSession.java | 10 +-
.../kudu/client/TestClientFailoverSupport.java | 7 +-
.../kudu/client/TestConnectToCluster.java | 13 +-
.../apache/kudu/client/TestConnectionCache.java | 5 +-
.../apache/kudu/client/TestHandleTooBusy.java | 2 +-
.../org/apache/kudu/client/TestKuduClient.java | 14 +-
.../apache/kudu/client/TestMasterFailover.java | 14 +-
.../apache/kudu/client/TestMiniKuduCluster.java | 24 +--
.../kudu/client/TestMultipleLeaderFailover.java | 7 +-
.../org/apache/kudu/client/TestNegotiation.java | 2 +-
.../apache/kudu/client/TestRemoteTablet.java| 3 +-
.../org/apache/kudu/client/TestSecurity.java| 22 +-
.../client/TestSecurityContextRealUser.java | 6 +-
.../org/apache/kudu/client/TestServerInfo.java | 5 +-
.../org/apache/kudu/client/TestTimeouts.java| 2 +-
.../java/org/apache/kudu/util/TestNetUtil.java | 13 +-
.../sink/AvroKuduOperationsProducerTest.java| 2 +-
.../sink/KeyedKuduOperationsProducerTest.java | 2 +-
.../apache/kudu/flume/sink/KuduSinkTest.java| 2 +-
.../sink/RegexpKuduOperationsProducerTest.java | 2 +-
.../apache/kudu/mapreduce/ITInputFormatJob.java | 2 +-
.../kudu/mapreduce/ITKuduTableInputFormat.java | 2 +-
.../kudu/mapreduce/ITKuduTableOutputFormat.java | 2 +-
.../kudu/mapreduce/ITOutputFormatJob.java | 2 +-
.../kudu/spark/tools/ITBigLinkedListTest.scala | 4 +-
.../spark/tools/TestImportExportFiles.scala | 2 +-
.../kudu/spark/kudu/DefaultSourceTest.scala | 39 ++--
.../kudu/spark/kudu/KuduContextTest.scala | 2 +-
.../apache/kudu/spark/kudu/KuduTestSuite.scala | 4 +-
48 files changed, 415 insertions(+), 292 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/fd1ffd0f/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
--
diff --git
a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index 4c4e5f0..aa4d603 100644
---
a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++
b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -351,11 +351,11 @@ class TestKuduBackup extends KuduTestSuite {
val path = dir.toUri.toString
val backupOptions =
- new
[1/4] kudu git commit: [docs] Add tip on dealing with planned TS downtime
Repository: kudu Updated Branches: refs/heads/master 816bc6fd8 -> fd1ffd0fb [docs] Add tip on dealing with planned TS downtime Rendering available at https://github.com/wdberkeley/kudu/blob/docfollowerunavailablesec/docs/administration.adoc. Change-Id: I55a992a00f35945187e02c55594edc6e261a72c4 Reviewed-on: http://gerrit.cloudera.org:8080/11486 Reviewed-by: Andrew Wong Reviewed-by: Grant Henke Tested-by: Will Berkeley Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3a033d82 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3a033d82 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3a033d82 Branch: refs/heads/master Commit: 3a033d829cd6aab17995b68371e7e136c47cc9b8 Parents: 816bc6f Author: Will Berkeley Authored: Thu Sep 20 12:23:41 2018 -0700 Committer: Will Berkeley Committed: Thu Sep 20 21:32:51 2018 + -- docs/administration.adoc | 37 + 1 file changed, 37 insertions(+) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/3a033d82/docs/administration.adoc -- diff --git a/docs/administration.adoc b/docs/administration.adoc index 74de5a0..b176f58 100644 --- a/docs/administration.adoc +++ b/docs/administration.adoc @@ -1120,6 +1120,43 @@ a node onto another machine. . Start all Kudu processes in the cluster. +[[minimizing_cluster_disruption_during_temporary_single_ts_downtime]] +=== Minimizing cluster disruption during temporary planned downtime of a single tablet server + +If a single tablet server is brought down temporarily in a healthy cluster, all +tablets will remain available and clients will function as normal, after +potential short delays due to leader elections. However, if the downtime lasts +for more than `--follower_unavailable_considered_failed_sec` (default 300) +seconds, the tablet replicas on the down tablet server will be replaced by new +replicas on available tablet servers. This will cause stress on the cluster +as tablets re-replicate and, if the downtime lasts long enough, significant +reduction in the number of replicas on the down tablet server. This may require +the rebalancer to fix. + +To work around this, increase `--follower_unavailable_considered_failed_sec` on +all tablet servers so the amount of time before re-replication will start is +longer than the expected downtime of the tablet server, including the time it +takes the tablet server to restart and bootstrap its tablet replicas. To do +this, run the following command for each tablet server: + +[source,bash] + +$ sudo -u kudu kudu tserver set_flag follower_unavailable_considered_failed_sec + + +where `` is the number of seconds that will encompass the downtime. +Once the downtime is finished, reset the flag to its original value. + + +$ sudo -u kudu kudu tserver set_flag follower_unavailable_considered_failed_sec + + +WARNING: Be sure to reset the value of `--follower_unavailable_considered_failed_sec` +to its original value. + +NOTE: On Kudu versions prior to 1.8, the `--force` flag must be provided in the above +commands. + [[rebalancer_tool]] === Running the tablet rebalancing tool
[2/4] kudu git commit: [tools] Add table name filter to kudu fs list
[tools] Add table name filter to kudu fs list
Due to the use of '--table_name' in 'kudu perf loadgen', I had to move
the definition of the flag to tool_action_common.cc and override the
description for the loadgen flag.
There's no unit tests for the filtering functionality, and I didn't add
any. I did test manually on a cluster with two tables that
'kudu fs list' works as expected with no table name filter, a table name
filter that matches no tables, and a table name filter that matches one
table. I also checked that 'kudu perf loadgen' still works as expected
when `--table_name' is supplied and when it is not supplied.
Change-Id: Iaed1aa0a566a7bfed3fac2574c0da1651059e9d0
Reviewed-on: http://gerrit.cloudera.org:8080/11487
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/e48157a0
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/e48157a0
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/e48157a0
Branch: refs/heads/master
Commit: e48157a0e6d1d4d3ee5fb700e020f0c648ff0560
Parents: 3a033d8
Author: Will Berkeley
Authored: Thu Sep 20 11:29:41 2018 -0700
Committer: Will Berkeley
Committed: Thu Sep 20 22:29:08 2018 +
--
src/kudu/tools/tool_action_common.cc | 2 ++
src/kudu/tools/tool_action_fs.cc | 9 -
src/kudu/tools/tool_action_perf.cc | 28 +++-
3 files changed, 25 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/e48157a0/src/kudu/tools/tool_action_common.cc
--
diff --git a/src/kudu/tools/tool_action_common.cc
b/src/kudu/tools/tool_action_common.cc
index ecc06a5..2005059 100644
--- a/src/kudu/tools/tool_action_common.cc
+++ b/src/kudu/tools/tool_action_common.cc
@@ -88,6 +88,8 @@ DEFINE_string(print_entries, "decoded",
" true|1|yes|decoded = print them decoded\n"
" pb = print the raw protobuf\n"
" id = print only their ids");
+DEFINE_string(table_name, "",
+ "Restrict output to a specific table by name");
DEFINE_int64(timeout_ms, 1000 * 60, "RPC timeout in milliseconds");
DEFINE_int32(truncate_data, 100,
"Truncate the data fields to the given number of bytes "
http://git-wip-us.apache.org/repos/asf/kudu/blob/e48157a0/src/kudu/tools/tool_action_fs.cc
--
diff --git a/src/kudu/tools/tool_action_fs.cc b/src/kudu/tools/tool_action_fs.cc
index f23ef81..5da0319 100644
--- a/src/kudu/tools/tool_action_fs.cc
+++ b/src/kudu/tools/tool_action_fs.cc
@@ -88,7 +88,8 @@ DEFINE_bool(repair, false,
"Repair any inconsistencies in the filesystem.");
DEFINE_string(table_id, "",
- "Restrict output to a specific table");
+ "Restrict output to a specific table by id");
+DECLARE_string(table_name);
DEFINE_string(tablet_id, "",
"Restrict output to a specific tablet");
DEFINE_int64(rowset_id, -1,
@@ -728,6 +729,7 @@ Status List(const RunnerContext& /*context*/) {
RETURN_NOT_OK(fs_manager.ListTabletIds(_ids));
}
+ string table_name = FLAGS_table_name;
string table_id = FLAGS_table_id;
ToLowerCase(_id);
@@ -739,6 +741,10 @@ Status List(const RunnerContext& /*context*/) {
RETURN_NOT_OK(TabletMetadata::Load(_manager, tablet_id,
_metadata));
const TabletMetadata& tablet = *tablet_metadata.get();
+if (!table_name.empty() && table_name != tablet.table_name()) {
+ continue;
+}
+
if (!table_id.empty() && table_id != tablet.table_id()) {
continue;
}
@@ -892,6 +898,7 @@ unique_ptr BuildFsMode() {
.AddOptionalParameter("fs_metadata_dir")
.AddOptionalParameter("fs_wal_dir")
.AddOptionalParameter("table_id")
+ .AddOptionalParameter("table_name")
.AddOptionalParameter("tablet_id")
.AddOptionalParameter("rowset_id")
.AddOptionalParameter("column_id")
http://git-wip-us.apache.org/repos/asf/kudu/blob/e48157a0/src/kudu/tools/tool_action_perf.cc
--
diff --git a/src/kudu/tools/tool_action_perf.cc
b/src/kudu/tools/tool_action_perf.cc
index 63de462..d162bbf 100644
--- a/src/kudu/tools/tool_action_perf.cc
+++ b/src/kudu/tools/tool_action_perf.cc
@@ -174,7 +174,9 @@
#include
#include
+#include
#include
+#include
#include
#include "kudu/client/client.h"
@@ -295,18 +297,7 @@ DEFINE_string(auto_database, "default",
"not created. This flag is useful primarily when the Hive
Metastore "
"integration is enabled in the cluster. If empty, no database is
"
"used.");
-DEFINE_string(table_name, "",
- "Name
[3/4] kudu git commit: [test] Clean up MiniKuduCluster and BaseKuduTest
http://git-wip-us.apache.org/repos/asf/kudu/blob/fd1ffd0f/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java
--
diff --git
a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java
b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java
index 675760d..8b4c3df 100644
---
a/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java
+++
b/java/kudu-flume-sink/src/test/java/org/apache/kudu/flume/sink/RegexpKuduOperationsProducerTest.java
@@ -206,7 +206,7 @@ public class RegexpKuduOperationsProducerTest extends
BaseKuduTest {
KuduSink sink = new KuduSink(syncClient);
HashMap parameters = new HashMap<>();
parameters.put(TABLE_NAME, tableName);
-parameters.put(MASTER_ADDRESSES, getMasterAddresses());
+parameters.put(MASTER_ADDRESSES, getMasterAddressesAsString());
parameters.put(PRODUCER, RegexpKuduOperationsProducer.class.getName());
parameters.put(PRODUCER_PREFIX + PATTERN_PROP, TEST_REGEXP);
parameters.put(PRODUCER_PREFIX + OPERATION_PROP, operation);
http://git-wip-us.apache.org/repos/asf/kudu/blob/fd1ffd0f/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
--
diff --git
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
index 69696e4..cd0e95a 100644
---
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
+++
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITInputFormatJob.java
@@ -92,7 +92,7 @@ public class ITInputFormatJob extends BaseKuduTest {
job,
TABLE_NAME,
"*",
-getMasterAddresses())
+getMasterAddressesAsString())
.operationTimeoutMs(DEFAULT_SLEEP)
.addDependencies(false)
.cacheBlocks(false)
http://git-wip-us.apache.org/repos/asf/kudu/blob/fd1ffd0f/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
--
diff --git
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
index 2cfb5cb..3266535 100644
---
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
+++
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableInputFormat.java
@@ -122,7 +122,7 @@ public class ITKuduTableInputFormat extends BaseKuduTest {
List predicates) throws IOException,
InterruptedException {
KuduTableInputFormat input = new KuduTableInputFormat();
Configuration conf = new Configuration();
-conf.set(KuduTableInputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
+conf.set(KuduTableInputFormat.MASTER_ADDRESSES_KEY,
getMasterAddressesAsString());
conf.set(KuduTableInputFormat.INPUT_TABLE_KEY, TABLE_NAME);
if (columnProjection != null) {
conf.set(KuduTableInputFormat.COLUMN_PROJECTION_KEY, columnProjection);
http://git-wip-us.apache.org/repos/asf/kudu/blob/fd1ffd0f/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
--
diff --git
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
index f97ee0a..597461f 100644
---
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
+++
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITKuduTableOutputFormat.java
@@ -45,7 +45,7 @@ public class ITKuduTableOutputFormat extends BaseKuduTest {
KuduTableOutputFormat output = new KuduTableOutputFormat();
Configuration conf = new Configuration();
-conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
+conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY,
getMasterAddressesAsString());
conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, TABLE_NAME);
output.setConf(conf);
http://git-wip-us.apache.org/repos/asf/kudu/blob/fd1ffd0f/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
--
diff --git
a/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
b/java/kudu-mapreduce/src/test/java/org/apache/kudu/mapreduce/ITOutputFormatJob.java
index 7b2745b..9c9918d 100644
---
