kudu git commit: Add a column renaming tool
Repository: kudu Updated Branches: refs/heads/master 3855328b7 -> c18e7e1a6 Add a column renaming tool This commit introduces a tool to rename a table's column. Similar to table renaming tool, when HMS integration feature is enabled, users can use this tool to rename table's column that have hive incompatible names. In order to allow these tables to be upgraded. Change-Id: Ibe98ce52d6865fa61c0903b38cfac3e86fc05a66 Reviewed-on: http://gerrit.cloudera.org:8080/10368 Tested-by: Hao HaoReviewed-by: Dan Burkert Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/c18e7e1a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/c18e7e1a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/c18e7e1a Branch: refs/heads/master Commit: c18e7e1a6c811fc432ecc5f0bce66afc91bf4cce Parents: 3855328 Author: hahao Authored: Wed May 9 18:17:08 2018 -0700 Committer: Hao Hao Committed: Thu May 10 23:24:02 2018 + -- src/kudu/tools/kudu-tool-test.cc| 40 +++- src/kudu/tools/tool_action_table.cc | 32 + 2 files changed, 67 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/c18e7e1a/src/kudu/tools/kudu-tool-test.cc -- diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index cc4d08c..6e8aeae 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -1847,7 +1847,7 @@ TEST_F(ToolTest, TestRenameTable) { string master_addr = cluster_->master()->bound_rpc_addr().ToString(); string out; - NO_FATALS(RunActionStdoutNone(Substitute("table rename $0 $1 $2", + NO_FATALS(RunActionStdoutNone(Substitute("table rename_table $0 $1 $2", master_addr, kTableName, kNewTableName))); shared_ptr client; @@ -1858,6 +1858,44 @@ TEST_F(ToolTest, TestRenameTable) { ASSERT_OK(client->OpenTable(kNewTableName, )); } +TEST_F(ToolTest, TestRenameColumn) { + NO_FATALS(StartExternalMiniCluster()); + const string& kTableName = "table"; + const string& kColumnName = "col.0"; + const string& kNewColumnName = "col_0"; + + KuduSchemaBuilder schema_builder; + schema_builder.AddColumn("key") + ->Type(client::KuduColumnSchema::INT32) + ->NotNull() + ->PrimaryKey(); + schema_builder.AddColumn(kColumnName) + ->Type(client::KuduColumnSchema::INT32) + ->NotNull(); + KuduSchema schema; + ASSERT_OK(schema_builder.Build()); + + // Create the table. + TestWorkload workload(cluster_.get()); + workload.set_table_name(kTableName); + workload.set_schema(schema); + workload.set_num_replicas(1); + workload.Setup(); + + string master_addr = cluster_->master()->bound_rpc_addr().ToString(); + string out; + NO_FATALS(RunActionStdoutNone(Substitute("table rename_column $0 $1 $2 $3", + master_addr, kTableName, + kColumnName, kNewColumnName))); + shared_ptr client; + ASSERT_OK(KuduClientBuilder() +.add_master_server_addr(master_addr) +.Build()); + shared_ptr table; + ASSERT_OK(client->OpenTable(kTableName, )); + ASSERT_STR_CONTAINS(table->schema().ToString(), kNewColumnName); +} + // This test is parameterized on the serialization mode and Kerberos. class ControlShellToolTest : public ToolTest, http://git-wip-us.apache.org/repos/asf/kudu/blob/c18e7e1a/src/kudu/tools/tool_action_table.cc -- diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc index b9f5308..6fd0037 100644 --- a/src/kudu/tools/tool_action_table.cc +++ b/src/kudu/tools/tool_action_table.cc @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include "kudu/tools/tool_action.h" - #include #include #include @@ -28,11 +26,13 @@ #include "kudu/client/client.h" #include "kudu/client/replica_controller-internal.h" +#include "kudu/client/schema.h" #include "kudu/client/shared_ptr.h" #include "kudu/gutil/map-util.h" #include "kudu/gutil/stl_util.h" #include "kudu/gutil/strings/split.h" #include "kudu/gutil/strings/substitute.h" +#include "kudu/tools/tool_action.h" #include "kudu/tools/tool_action_common.h" #include "kudu/util/status.h" @@ -103,6 +103,8 @@ namespace { const char* const kTableNameArg = "table_name"; const char* const kNewTableNameArg = "new_table_name"; +const char* const kColumnNameArg = "column_name"; +const char* const kNewColumnNameArg =
[2/3] kudu git commit: Fix int overflow GetClockTimeMicros() on macOS
Fix int overflow GetClockTimeMicros() on macOS On macOS mach_timespec_t.tv_sec is only 4 bytes and we were converting to micros before moving to a bigger var. This would cause all the wall times obtained on a mac (through GetClockTimeMicros() to be wrong. This was likely the cause of KUDU-2435 and KUDU-2408 too, since the time would easily wrap, causing us to a update the clock with a value that was seemingly from the future. Posix just requires it to be an integer (see: https://en.wikipedia.org/w/index.php?title=Time_t=450752800) so also fixed it on the non-macOS path. Testing this is likely not worth it since the only real change was on macOS where the overlow doesn't happen anymore. Change-Id: Ie0eaa548f61352be529755a732566613cfa72098 Reviewed-on: http://gerrit.cloudera.org:8080/10371 Reviewed-by: Todd LipconReviewed-by: Dan Burkert Reviewed-by: Grant Henke Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/07d6b5f2 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/07d6b5f2 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/07d6b5f2 Branch: refs/heads/master Commit: 07d6b5f2ba384e10288d5926af93933f0a74185d Parents: 525943f Author: David Alves Authored: Thu May 10 13:52:14 2018 -0700 Committer: David Ribeiro Alves Committed: Thu May 10 22:38:23 2018 + -- src/kudu/gutil/walltime.h | 15 +-- 1 file changed, 13 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/07d6b5f2/src/kudu/gutil/walltime.h -- diff --git a/src/kudu/gutil/walltime.h b/src/kudu/gutil/walltime.h index e9cab67..4b15f42 100644 --- a/src/kudu/gutil/walltime.h +++ b/src/kudu/gutil/walltime.h @@ -81,7 +81,12 @@ inline void GetCurrentTime(mach_timespec_t* ts) { inline MicrosecondsInt64 GetCurrentTimeMicros() { mach_timespec_t ts; GetCurrentTime(); - return ts.tv_sec * 100 + ts.tv_nsec / 1000; + // 'tv_sec' is just 4 bytes on macOS, need to be careful not + // to convert to nanos until we've moved to a larger int. + MicrosecondsInt64 micros_from_secs = ts.tv_sec; + micros_from_secs *= 1000 * 1000; + micros_from_secs += ts.tv_nsec / 1000; + return micros_from_secs; } inline int64_t GetMonoTimeNanos() { @@ -130,7 +135,13 @@ inline MicrosecondsInt64 GetThreadCpuTimeMicros() { inline MicrosecondsInt64 GetClockTimeMicros(clockid_t clock) { timespec ts; clock_gettime(clock, ); - return ts.tv_sec * 100 + ts.tv_nsec / 1000; + // 'tv_sec' is usually 8 bytes, but the spec says it only + // needs to be 'a signed int'. Moved to a 64 bit var before + // converting to micros to be safe. + MicrosecondsInt64 micros_from_secs = ts.tv_sec; + micros_from_secs *= 1000 * 1000; + micros_from_secs += ts.tv_nsec / 1000; + return micros_from_secs; } #endif // defined(__APPLE__)
[1/3] kudu git commit: [tools] Add ids to tables and tablets; table name to tablets
Repository: kudu Updated Branches: refs/heads/master 3f43c03a1 -> 3855328b7 [tools] Add ids to tables and tablets; table name to tablets This adds tablet id, table id, and table name to KsckTabletSummary, and table id to KsckTableSummary. This will be tested in the follow-up JSON formatting patch. Change-Id: I19cd76a4c4c59c28930a44c0593021039903bec1 Reviewed-on: http://gerrit.cloudera.org:8080/10369 Reviewed-by: Alexey SerbinTested-by: Will Berkeley Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/525943fd Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/525943fd Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/525943fd Branch: refs/heads/master Commit: 525943fd189acc20a3199f84431d62a776dd8f30 Parents: 3f43c03 Author: Will Berkeley Authored: Sun May 6 12:56:41 2018 -0700 Committer: Will Berkeley Committed: Thu May 10 20:07:41 2018 + -- src/kudu/tools/ksck-test.cc | 5 +++-- src/kudu/tools/ksck.cc| 6 +- src/kudu/tools/ksck.h | 12 ++-- src/kudu/tools/ksck_remote.cc | 3 ++- src/kudu/tools/ksck_results.h | 4 5 files changed, 24 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck-test.cc -- diff --git a/src/kudu/tools/ksck-test.cc b/src/kudu/tools/ksck-test.cc index f4011cc..18d28ef 100644 --- a/src/kudu/tools/ksck-test.cc +++ b/src/kudu/tools/ksck-test.cc @@ -278,8 +278,9 @@ class KsckTest : public KuduTest { table->set_tablets({ tablet }); } - shared_ptr CreateAndAddTable(const string& name, int num_replicas) { -shared_ptr table(new KsckTable(name, Schema(), num_replicas)); + shared_ptr CreateAndAddTable(const string& id_and_name, int num_replicas) { +shared_ptr table(new KsckTable(id_and_name, id_and_name, + Schema(), num_replicas)); cluster_->tables_.push_back(table); return table; } http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck.cc -- diff --git a/src/kudu/tools/ksck.cc b/src/kudu/tools/ksck.cc index 191838b..70a1b64 100644 --- a/src/kudu/tools/ksck.cc +++ b/src/kudu/tools/ksck.cc @@ -716,10 +716,11 @@ bool Ksck::VerifyTable(const shared_ptr& table) { } KsckTableSummary ts; + ts.id = table->id(); + ts.name = table->name(); ts.replication_factor = table->num_replicas(); VLOG(1) << Substitute("Verifying $0 tablet(s) for table $1 configured with num_replicas = $2", tablets.size(), table->name(), table->num_replicas()); - ts.name = table->name(); for (const auto& tablet : tablets) { auto tablet_result = VerifyTablet(tablet, table->num_replicas()); switch (tablet_result) { @@ -876,6 +877,9 @@ KsckCheckResult Ksck::VerifyTablet(const shared_ptr& tablet, } KsckTabletSummary tablet_summary; + tablet_summary.id = tablet->id(); + tablet_summary.table_id = tablet->table()->id(); + tablet_summary.table_name = tablet->table()->name(); tablet_summary.result = result; tablet_summary.status = status; tablet_summary.master_cstate = std::move(master_config); http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck.h -- diff --git a/src/kudu/tools/ksck.h b/src/kudu/tools/ksck.h index edae81b..1c596e4 100644 --- a/src/kudu/tools/ksck.h +++ b/src/kudu/tools/ksck.h @@ -138,8 +138,15 @@ class KsckTablet { // Representation of a table. Composed of tablets. class KsckTable { public: - KsckTable(std::string name, const Schema& schema, int num_replicas) - : name_(std::move(name)), schema_(schema), num_replicas_(num_replicas) {} + KsckTable(std::string id, std::string name, const Schema& schema, int num_replicas) + : id_(std::move(id)), +name_(std::move(name)), +schema_(schema), +num_replicas_(num_replicas) {} + + const std::string& id() const { +return id_; + } const std::string& name() const { return name_; @@ -162,6 +169,7 @@ class KsckTable { } private: + const std::string id_; const std::string name_; const Schema schema_; const int num_replicas_; http://git-wip-us.apache.org/repos/asf/kudu/blob/525943fd/src/kudu/tools/ksck_remote.cc -- diff --git a/src/kudu/tools/ksck_remote.cc b/src/kudu/tools/ksck_remote.cc index d24f8c7..017321e 100644 --- a/src/kudu/tools/ksck_remote.cc +++ b/src/kudu/tools/ksck_remote.cc @@ -426,7 +426,8 @@ Status
[3/3] kudu git commit: thirdparty: tweak clang compiler flags
thirdparty: tweak clang compiler flags Prior to this small tweak the thirdparty clang build output thousands of warnings similar to this one when compiling with clang: Building CXX object lib/Analysis/CMakeFiles/LLVMAnalysis.dir/LazyValueInfo.cpp.o clang: warning: -Wl,-rpath,/Users/dan/src/cpp/kudu/thirdparty/installed/uninstrumented/lib: 'linker' input unused [-Wunused-command-line-argument] clang: warning: argument unused during compilation: '-L/Users/dan/src/cpp/kudu/thirdparty/installed/uninstrumented/lib' [-Wunused-command-line-argument] Change-Id: Ide4ddbff14d3745c6f2c2f9b14b00da790a6cec6 Reviewed-on: http://gerrit.cloudera.org:8080/10352 Reviewed-by: Adar DemboTested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3855328b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3855328b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3855328b Branch: refs/heads/master Commit: 3855328b73c28d9ce4dbe5e1773bd6abde2e935a Parents: 07d6b5f Author: Dan Burkert Authored: Tue May 8 17:00:22 2018 -0700 Committer: Dan Burkert Committed: Thu May 10 22:53:24 2018 + -- thirdparty/build-definitions.sh | 29 - 1 file changed, 16 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/3855328b/thirdparty/build-definitions.sh -- diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh index fe3f477..ebb4f72 100644 --- a/thirdparty/build-definitions.sh +++ b/thirdparty/build-definitions.sh @@ -252,6 +252,11 @@ build_llvm() { TOOLS_ARGS="$TOOLS_ARGS -D${arg}=OFF" done + # Remove '-nostdinc++' from the cflags for clang, since it already + # handles this when passing --stdlib=libc++ and passing this confuses + # the check for -fPIC. + CLANG_CXXFLAGS=$(echo "$EXTRA_CXXFLAGS" | sed -e 's,-nostdinc++,,g;') + case $BUILD_TYPE in "normal") # Default build: core LLVM libraries, clang, compiler-rt, and all tools. @@ -269,6 +274,15 @@ build_llvm() { if [ -n "$GCC_INSTALL_PREFIX" ]; then TOOLS_ARGS="$TOOLS_ARGS -DGCC_INSTALL_PREFIX=$GCC_INSTALL_PREFIX" fi + + # Depend on zlib from the thirdparty tree. It's an optional dependency for + # LLVM, but a required [1] one for IWYU. When TSAN is enabled these flags + # are already set by build-thirdparty.sh in order to support the + # thirdparty libc++, so it's not necessary to set them again. + # + # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539 + CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include" + CLANG_LDFLAGS="$CLANG_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" ;; "tsan") # Build just the core LLVM libraries, dependent on libc++. @@ -303,18 +317,6 @@ build_llvm() { $PREFIX/lib/clang/ \ $PREFIX/lib/cmake/{llvm,clang} - # Remove '-nostdinc++' from the cflags for clang, since it already - # handles this when passing --stdlib=libc++ and passing this confuses - # the check for -fPIC. - CLANG_CXXFLAGS=$(echo "$EXTRA_CXXFLAGS" | sed -e 's,-nostdinc++,,g;') - - # Depend on zlib from the thirdparty tree. It's an optional dependency for - # LLVM, but a required [1] one for IWYU. - # - # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539 - CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include" - CLANG_LDFLAGS="$EXTRA_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" - cmake \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ @@ -324,7 +326,8 @@ build_llvm() { -DLLVM_INCLUDE_UTILS=OFF \ -DLLVM_TARGETS_TO_BUILD=X86 \ -DLLVM_ENABLE_RTTI=ON \ --DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS $CLANG_LDFLAGS" \ +-DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS $EXTRA_LDFLAGS" \ +-DCMAKE_EXE_LINKER_FLAGS="$CLANG_LDFLAGS" \ -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \ $TOOLS_ARGS \ $EXTRA_CMAKE_FLAGS \
kudu git commit: [flags] run validators after processing help flags
Repository: kudu Updated Branches: refs/heads/master 26af97c8f -> 3f43c03a1 [flags] run validators after processing help flags Run flag validators (both individual and group ones) after processing the help flags. Also, this is a follow-up for 297b72bd26cdd546f0a73cda7487c80566388492: the default value for the --force_block_cache_capacity flag is set to 'true' everywhere but in master and tablet server. Otherwise, corresponding group validator could strike while running binaries that link the cfile library (e.g. the kudu CLI tool) at machines with less than 256 MB of available memory. Change-Id: I51f263890c91251f0d13c826c40dfd20fe284b59 Reviewed-on: http://gerrit.cloudera.org:8080/10342 Reviewed-by: Adar DemboTested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3f43c03a Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3f43c03a Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3f43c03a Branch: refs/heads/master Commit: 3f43c03a1e4130cde6556a298ad8bb3f32897993 Parents: 26af97c Author: Alexey Serbin Authored: Mon May 7 22:37:34 2018 -0700 Committer: Alexey Serbin Committed: Thu May 10 19:05:50 2018 + -- src/kudu/cfile/block_cache.cc | 6 +- src/kudu/consensus/CMakeLists.txt | 2 +- src/kudu/integration-tests/CMakeLists.txt | 4 ++-- src/kudu/master/master_main.cc| 11 +-- src/kudu/tools/CMakeLists.txt | 27 -- src/kudu/tserver/CMakeLists.txt | 2 +- src/kudu/tserver/tablet_server_main.cc| 10 -- src/kudu/util/flags.cc| 22 + src/kudu/util/metrics.cc | 4 +--- src/kudu/util/metrics.h | 4 ++-- 10 files changed, 50 insertions(+), 42 deletions(-) -- http://git-wip-us.apache.org/repos/asf/kudu/blob/3f43c03a/src/kudu/cfile/block_cache.cc -- diff --git a/src/kudu/cfile/block_cache.cc b/src/kudu/cfile/block_cache.cc index 42c4201..ff7219d 100644 --- a/src/kudu/cfile/block_cache.cc +++ b/src/kudu/cfile/block_cache.cc @@ -38,7 +38,11 @@ DEFINE_int64(block_cache_capacity_mb, 512, "block cache capacity in MB"); TAG_FLAG(block_cache_capacity_mb, stable); -DEFINE_bool(force_block_cache_capacity, false, +// Yes, it's strange: the default value is 'true' but that's intentional. +// The idea is to avoid the corresponding group flag validator striking +// while running anything but master and tserver. As for the master and tserver, +// those have the default value for this flag set to 'false'. +DEFINE_bool(force_block_cache_capacity, true, "Force Kudu to accept the block cache size, even if it is unsafe."); TAG_FLAG(force_block_cache_capacity, unsafe); TAG_FLAG(force_block_cache_capacity, hidden); http://git-wip-us.apache.org/repos/asf/kudu/blob/3f43c03a/src/kudu/consensus/CMakeLists.txt -- diff --git a/src/kudu/consensus/CMakeLists.txt b/src/kudu/consensus/CMakeLists.txt index 11463a6..ca21394 100644 --- a/src/kudu/consensus/CMakeLists.txt +++ b/src/kudu/consensus/CMakeLists.txt @@ -120,7 +120,7 @@ target_link_libraries(consensus set(KUDU_TEST_LINK_LIBS log consensus - tserver + tserver_proto cfile tablet kudu_util http://git-wip-us.apache.org/repos/asf/kudu/blob/3f43c03a/src/kudu/integration-tests/CMakeLists.txt -- diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index 60ff72a..f5ed494 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -34,9 +34,9 @@ set(INTEGRATION_TESTS_SRCS add_library(itest_util ${INTEGRATION_TESTS_SRCS}) target_link_libraries(itest_util - tserver + tserver_proto tserver_test_util - master + master_proto mini_cluster ksck kudu_client http://git-wip-us.apache.org/repos/asf/kudu/blob/3f43c03a/src/kudu/master/master_main.cc -- diff --git a/src/kudu/master/master_main.cc b/src/kudu/master/master_main.cc index 2dbcac2..1abb837 100644 --- a/src/kudu/master/master_main.cc +++ b/src/kudu/master/master_main.cc @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -33,9 +34,9 @@ using kudu::master::Master; -DECLARE_string(rpc_bind_addresses); -DECLARE_int32(webserver_port); DECLARE_bool(evict_failed_followers); +DECLARE_int32(webserver_port); +DECLARE_string(rpc_bind_addresses); namespace kudu { namespace master { @@ -53,6 +54,12 @@ static int