kudu git commit: [tools] fix glog output colorizing for kudu CLI tests
Repository: kudu
Updated Branches:
refs/heads/master a81d80a79 -> 0dcf16673
[tools] fix glog output colorizing for kudu CLI tests
This patch adds the '--nocolorlogtostderr' flag to every invocation
of the kudu CLI tool run by the kudu::tools::RunKuduTool() function.
Prior to this fix, the following tests would fail if running in the
environment where the GLOG_colorlogtostderr variable is set to 1:
ToolTest.TestTopLevelHelp
ToolTest.TestModeHelp
ToolTest.TestActionHelp
ToolTest.TestActionMissingRequiredArg
Change-Id: I6635955182e8fdbb4406c9909a7a09a7d3a55955
Reviewed-on: http://gerrit.cloudera.org:8080/10865
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0dcf1667
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0dcf1667
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0dcf1667
Branch: refs/heads/master
Commit: 0dcf166730c81fa07039788215279ef0bcfc03fe
Parents: a81d80a
Author: Alexey Serbin
Authored: Tue Jul 3 18:18:06 2018 -0700
Committer: Alexey Serbin
Committed: Wed Jul 4 01:49:32 2018 +
--
src/kudu/tools/tool_test_util.cc | 6 ++
1 file changed, 6 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/0dcf1667/src/kudu/tools/tool_test_util.cc
--
diff --git a/src/kudu/tools/tool_test_util.cc b/src/kudu/tools/tool_test_util.cc
index 60da0bb..8f7318f 100644
--- a/src/kudu/tools/tool_test_util.cc
+++ b/src/kudu/tools/tool_test_util.cc
@@ -54,6 +54,12 @@ Status RunKuduTool(const vector& args, string* out,
string* err,
total_args.emplace_back("--unlock_unsafe_flags");
total_args.emplace_back("--never_fsync");
+ // Do not colorize glog's output (i.e. messages logged via LOG()) even
+ // if the GLOG_colorlogtostderr environment variable is set. This is to avoid
+ // failing of tests that depend on the exact output from the tool
+ // (e.g., the exact location of some substring/character in the output line).
+ total_args.emplace_back("--nocolorlogtostderr");
+
total_args.insert(total_args.end(), args.begin(), args.end());
return Subprocess::Call(total_args, in, out, err);
}
kudu git commit: [tools] extra integration tests for the rebalancer
Repository: kudu
Updated Branches:
refs/heads/master 7b048b8db -> a81d80a79
[tools] extra integration tests for the rebalancer
This patch adds more integration tests for the rebalancer,
providing coverage for the following scenarios:
* a new tablet server added post-rebalancing,
and the rebalancer is run again
* DDL operations are run concurrently with the rebalancing
* two rebalancers are running concurrently
* a tablet server goes down during rebalancing
* a tablet server is added during rebalancing
Change-Id: I78b3dcea71ed303f6ecd199604b2385796d05da8
Reviewed-on: http://gerrit.cloudera.org:8080/10540
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a81d80a7
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a81d80a7
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a81d80a7
Branch: refs/heads/master
Commit: a81d80a7987cd66c4d0e697058ecd817d2707a9e
Parents: 7b048b8
Author: Alexey Serbin
Authored: Tue May 29 12:42:43 2018 -0700
Committer: Alexey Serbin
Committed: Wed Jul 4 01:24:17 2018 +
--
src/kudu/tools/CMakeLists.txt | 3 +-
src/kudu/tools/kudu-admin-test.cc | 654 ++---
2 files changed, 610 insertions(+), 47 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/a81d80a7/src/kudu/tools/CMakeLists.txt
--
diff --git a/src/kudu/tools/CMakeLists.txt b/src/kudu/tools/CMakeLists.txt
index 97d8bdb..e367d64 100644
--- a/src/kudu/tools/CMakeLists.txt
+++ b/src/kudu/tools/CMakeLists.txt
@@ -164,7 +164,8 @@ set(KUDU_TEST_LINK_LIBS
ADD_KUDU_TEST(diagnostics_log_parser-test)
ADD_KUDU_TEST(ksck-test)
ADD_KUDU_TEST(ksck_remote-test PROCESSORS 3)
-ADD_KUDU_TEST(kudu-admin-test PROCESSORS 3)
+ADD_KUDU_TEST(kudu-admin-test
+ NUM_SHARDS 4 PROCESSORS 3)
ADD_KUDU_TEST_DEPENDENCIES(kudu-admin-test
kudu)
ADD_KUDU_TEST(kudu-tool-test
http://git-wip-us.apache.org/repos/asf/kudu/blob/a81d80a7/src/kudu/tools/kudu-admin-test.cc
--
diff --git a/src/kudu/tools/kudu-admin-test.cc
b/src/kudu/tools/kudu-admin-test.cc
index 1868c90..856f1f4 100644
--- a/src/kudu/tools/kudu-admin-test.cc
+++ b/src/kudu/tools/kudu-admin-test.cc
@@ -16,6 +16,7 @@
// under the License.
#include
+#include
#include
#include
#include
@@ -23,6 +24,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -46,6 +48,7 @@
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/strings/split.h"
#include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/strings/util.h"
#include "kudu/integration-tests/cluster_itest_util.h"
#include "kudu/integration-tests/cluster_verifier.h"
#include "kudu/integration-tests/test_workload.h"
@@ -55,10 +58,13 @@
#include "kudu/tablet/metadata.pb.h"
#include "kudu/tools/tool_test_util.h"
#include "kudu/tserver/tablet_server-test-base.h"
+#include "kudu/util/countdown_latch.h"
#include "kudu/util/monotime.h"
#include "kudu/util/net/net_util.h"
#include "kudu/util/net/sockaddr.h"
#include "kudu/util/pb_util.h"
+#include "kudu/util/random.h"
+#include "kudu/util/scoped_cleanup.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
@@ -68,7 +74,10 @@ DECLARE_int32(num_tablet_servers);
using kudu::client::KuduClient;
using kudu::client::KuduClientBuilder;
+using kudu::client::KuduColumnSchema;
using kudu::client::KuduSchema;
+using kudu::client::KuduSchemaBuilder;
+using kudu::client::KuduTableAlterer;
using kudu::client::KuduTableCreator;
using kudu::client::sp::shared_ptr;
using kudu::cluster::ExternalTabletServer;
@@ -92,16 +101,22 @@ using kudu::itest::WaitUntilTabletInState;
using kudu::itest::WaitUntilTabletRunning;
using kudu::master::VOTER_REPLICA;
using kudu::pb_util::SecureDebugString;
+using std::atomic;
using std::back_inserter;
using std::copy;
using std::deque;
+using std::ostringstream;
using std::string;
+using std::thread;
using std::unique_ptr;
using std::vector;
using strings::Split;
using strings::Substitute;
namespace kudu {
+
+class Schema;
+
namespace tools {
class AdminCliTest : public tserver::TabletServerIntegrationTestBase {
@@ -1357,6 +1372,55 @@ TEST_P(RebalanceStartCriteriaTest, TabletServerIsDown) {
ASSERT_STR_MATCHES(err, err_msg_pattern);
}
+// Create tables with unbalanced replica distribution: useful in
+// rebalancer-related tests.
+static Status CreateUnbalancedTables(
+cluster::ExternalMiniCluster* cluster,
+client::KuduClient* client,
+const Schema& table_schema,
+const string& table_name_pattern,
+int num_tables,
+int rep_factor,
+int tserver_idx_from,
+
kudu git commit: KUDU-2191: support table-name identifiers with upper case chars
Repository: kudu
Updated Branches:
refs/heads/master 1ae050e4d -> 7b048b8db
KUDU-2191: support table-name identifiers with upper case chars
Summary: When the HMS integration is enabled, Kudu now preserves table
name casing, but uses case-insensitive lookups to retrieve tables.
Background: The HMS lowercases all database (table) identifiers during database
(table) creation, only storing the lowercased version. On database and
table lookup the HMS automatically does a case-insensitive compare.
During table creation Kudu checks that table names are valid UTF-8, and
does no transformations on identiers. During table lookups Kudu requires
that the table name match exactly, including case.
As a result of these behavior differences and the design of the
notification log listener, tables with upper-case characters can not be
altered or deleted when the HMS integration is enabled. This commit
fixes this by changing how the Catalog Manager handles identifiers when
the HMS integration is enabled:
* During table creation, the Catalog Manager preserves the case of table
names.
* On table lookup, the Catalog Manager does a case-insensitive
comparison to find the table.
This is implemented by storing the preserved case in the table's
sys-catalog metadata entry, and storing a 'normalized' (down-cased)
identifier in the ephemeral by-name table map. The various parts of the
catalog manager which deal with the by-name map are converted to use the
normalized version of the name. When the HMS integration is not
configured, normalized table names are equal to the original table name,
so the behavior changes that this patch introduces are entirely opt-in.
There is one edge case that complicates turning on the HMS integration
in rare circumstances: if there are existing (legacy) tables with names
which map to the same normalized form (e.g. differ only in case), the
catalog manager will fail to startup and instruct the operator to rename
the offending tables before trying again. Additionally, this check only
applies to tables that otherwise follow the Hive table naming rules
(matching regex '[\w_/]+\.[\w_/]+').
Change-Id: I18977d6fe7b2999a36681a728ac0d1e54b7f38cd
Reviewed-on: http://gerrit.cloudera.org:8080/10817
Reviewed-by: Adar Dembo
Tested-by: Kudu Jenkins
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/7b048b8d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/7b048b8d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/7b048b8d
Branch: refs/heads/master
Commit: 7b048b8dbe2562e68dd2b2b7f1a4db83b1ae10bf
Parents: 1ae050e
Author: Dan Burkert
Authored: Fri Jun 22 14:19:38 2018 -0700
Committer: Dan Burkert
Committed: Tue Jul 3 23:23:32 2018 +
--
src/kudu/hms/hms_catalog-test.cc| 62 +---
src/kudu/hms/hms_catalog.cc | 97 +
src/kudu/hms/hms_catalog.h | 26 +++-
src/kudu/hms/hms_client-test.cc | 28
.../integration-tests/master-stress-test.cc | 2 +-
src/kudu/integration-tests/master_hms-itest.cc | 126 +++--
src/kudu/master/catalog_manager.cc | 141 +--
src/kudu/master/catalog_manager.h | 34 +++--
src/kudu/mini-cluster/external_mini_cluster.cc | 15 ++
src/kudu/mini-cluster/external_mini_cluster.h | 8 +-
10 files changed, 422 insertions(+), 117 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/kudu/blob/7b048b8d/src/kudu/hms/hms_catalog-test.cc
--
diff --git a/src/kudu/hms/hms_catalog-test.cc b/src/kudu/hms/hms_catalog-test.cc
index 12066f6..472f3a8 100644
--- a/src/kudu/hms/hms_catalog-test.cc
+++ b/src/kudu/hms/hms_catalog-test.cc
@@ -37,6 +37,7 @@
#include "kudu/rpc/sasl_common.h"
#include "kudu/security/test/mini_kdc.h"
#include "kudu/util/net/net_util.h"
+#include "kudu/util/slice.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
#include "kudu/util/test_util.h"
@@ -55,35 +56,64 @@ namespace kudu {
namespace hms {
TEST(HmsCatalogStaticTest, TestParseTableName) {
- string db;
- string tbl;
+ Slice db;
+ Slice tbl;
+ string table;
- EXPECT_OK(HmsCatalog::ParseTableName("foo.bar", , ));
+ table = "foo.bar";
+ ASSERT_OK(HmsCatalog::ParseTableName(table, , ));
EXPECT_EQ("foo", db);
EXPECT_EQ("bar", tbl);
- EXPECT_OK(HmsCatalog::ParseTableName("99bottles.my_awesome/table/22", ,
));
+ table = "99bottles.my_awesome/table/22";
+ ASSERT_OK(HmsCatalog::ParseTableName(table, , ));
EXPECT_EQ("99bottles", db);
EXPECT_EQ("my_awesome/table/22", tbl);
-
EXPECT_OK(HmsCatalog::ParseTableName("_leading_underscore.trailing_underscore_",
, ));
+ table = "_leading_underscore.trailing_underscore_";
+
