This is an automated email from the ASF dual-hosted git repository. wdberkeley pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 3047db9283432746a3282b180d41f08a97d4aa3f Author: acelyc111 <405403...@qq.com> AuthorDate: Tue Jan 22 00:51:23 2019 +0800 [tools] Describe more table attributes This adds more attributes to `kudu table describe`, including column encoding type, compression type, and default read/write value. My use case for the extra information is to compare two tables in the same or different clusters: export the full descriptions of the two tables into text, and then use any text compare tool to check the difference. Change-Id: I4f8cf682f3a2a30f98edbc4aff6951225d8d81e8 Reviewed-on: http://gerrit.cloudera.org:8080/12281 Tested-by: Kudu Jenkins Reviewed-by: Will Berkeley <wdberke...@gmail.com> --- src/kudu/cfile/encoding-test.cc | 1 - src/kudu/client/schema.cc | 11 +++++- src/kudu/common/schema-test.cc | 2 +- src/kudu/common/schema.cc | 24 ++++++++--- src/kudu/common/schema.h | 23 ++++++++--- src/kudu/common/types.h | 1 + src/kudu/tools/kudu-admin-test.cc | 79 +++++++++++++++++++++++++++++++------ src/kudu/tools/kudu-tool-test.cc | 1 - src/kudu/tools/tool_action_table.cc | 1 + src/kudu/util/int128_util.h | 3 +- 10 files changed, 118 insertions(+), 28 deletions(-) diff --git a/src/kudu/cfile/encoding-test.cc b/src/kudu/cfile/encoding-test.cc index 9bffc8f..5d6e3b7 100644 --- a/src/kudu/cfile/encoding-test.cc +++ b/src/kudu/cfile/encoding-test.cc @@ -51,7 +51,6 @@ #include "kudu/util/group_varint-inl.h" #include "kudu/util/hexdump.h" #include "kudu/util/int128.h" -#include "kudu/util/int128_util.h" // IWYU pragma: keep #include "kudu/util/memory/arena.h" #include "kudu/util/random.h" #include "kudu/util/random_util.h" diff --git a/src/kudu/client/schema.cc b/src/kudu/client/schema.cc index 6e7824e..cdc510c 100644 --- a/src/kudu/client/schema.cc +++ b/src/kudu/client/schema.cc @@ -24,6 +24,7 @@ #include <utility> #include <boost/optional/optional.hpp> +#include <gflags/gflags.h> #include <glog/logging.h> #include "kudu/client/schema-internal.h" @@ -38,10 +39,14 @@ #include "kudu/gutil/macros.h" #include "kudu/gutil/map-util.h" #include "kudu/gutil/strings/substitute.h" -#include "kudu/util/decimal_util.h" #include "kudu/util/compression/compression.pb.h" +#include "kudu/util/decimal_util.h" #include "kudu/util/slice.h" +DEFINE_bool(show_attributes, false, + "Whether to show column attributes, including column encoding type, " + "compression type, and default read/write value."); + MAKE_ENUM_LIMITS(kudu::client::KuduColumnStorageAttributes::EncodingType, kudu::client::KuduColumnStorageAttributes::AUTO_ENCODING, kudu::client::KuduColumnStorageAttributes::RLE); @@ -747,7 +752,9 @@ void KuduSchema::GetPrimaryKeyColumnIndexes(vector<int>* indexes) const { } string KuduSchema::ToString() const { - return schema_ ? schema_->ToString(Schema::ToStringMode::WITHOUT_COLUMN_IDS) + return schema_ ? schema_->ToString(FLAGS_show_attributes ? + Schema::ToStringMode::WITH_COLUMN_ATTRIBUTES + : Schema::ToStringMode::BASE_INFO) : "()"; } diff --git a/src/kudu/common/schema-test.cc b/src/kudu/common/schema-test.cc index d32e666..198bf00 100644 --- a/src/kudu/common/schema-test.cc +++ b/src/kudu/common/schema-test.cc @@ -118,7 +118,7 @@ TEST_F(TestSchema, TestSchemaToStringMode) { " key INT32 NOT NULL,\n" " PRIMARY KEY (key)\n" ")", - schema.ToString(Schema::ToStringMode::WITHOUT_COLUMN_IDS)); + schema.ToString(Schema::ToStringMode::BASE_INFO)); } TEST_F(TestSchema, TestCopyAndMove) { diff --git a/src/kudu/common/schema.cc b/src/kudu/common/schema.cc index df04e67..46f74a1 100644 --- a/src/kudu/common/schema.cc +++ b/src/kudu/common/schema.cc @@ -118,10 +118,11 @@ Status ColumnSchema::ApplyDelta(const ColumnSchemaDelta& col_delta) { return Status::OK(); } -string ColumnSchema::ToString() const { - return Substitute("$0 $1", +string ColumnSchema::ToString(ToStringMode mode) const { + return Substitute("$0 $1$2", name_, - TypeToString()); + TypeToString(), + mode == ToStringMode::WITH_ATTRIBUTES ? " " + AttrToString() : ""); } string ColumnSchema::TypeToString() const { @@ -133,6 +134,13 @@ string ColumnSchema::TypeToString() const { is_nullable_ ? "NULLABLE" : "NOT NULL"); } +string ColumnSchema::AttrToString() const { + return Substitute("$0 $1 $2", + attributes_.ToString(), + has_read_default() ? Stringify(read_default_value()) : "-", + has_write_default() ? Stringify(write_default_value()) : "-"); +} + size_t ColumnSchema::memory_footprint_excluding_this() const { // Rough approximation. return name_.capacity(); @@ -399,14 +407,18 @@ string Schema::ToString(ToStringMode mode) const { pk_strs.push_back(cols_[i].name()); } + auto col_mode = ColumnSchema::ToStringMode::WITHOUT_ATTRIBUTES; + if (mode & ToStringMode::WITH_COLUMN_ATTRIBUTES) { + col_mode = ColumnSchema::ToStringMode::WITH_ATTRIBUTES; + } vector<string> col_strs; - if (has_column_ids() && mode != ToStringMode::WITHOUT_COLUMN_IDS) { + if (has_column_ids() && (mode & ToStringMode::WITH_COLUMN_IDS)) { for (int i = 0; i < cols_.size(); ++i) { - col_strs.push_back(Substitute("$0:$1", col_ids_[i], cols_[i].ToString())); + col_strs.push_back(Substitute("$0:$1", col_ids_[i], cols_[i].ToString(col_mode))); } } else { for (const ColumnSchema &col : cols_) { - col_strs.push_back(col.ToString()); + col_strs.push_back(col.ToString(col_mode)); } } diff --git a/src/kudu/common/schema.h b/src/kudu/common/schema.h index 4c19969..cd6dbc9 100644 --- a/src/kudu/common/schema.h +++ b/src/kudu/common/schema.h @@ -228,14 +228,26 @@ class ColumnSchema { return name_; } + // Enum to configure how a ColumnSchema is stringified. + enum class ToStringMode { + // Include encoding type, compression type, and default read/write value. + WITH_ATTRIBUTES, + // Do not include above attributes. + WITHOUT_ATTRIBUTES, + }; + // Return a string identifying this column, including its // name. - std::string ToString() const; + std::string ToString(ToStringMode mode = ToStringMode::WITHOUT_ATTRIBUTES) const; // Same as above, but only including the type information. // For example, "STRING NOT NULL". std::string TypeToString() const; + // Same as above, but only including the attributes information. + // For example, "AUTO_ENCODING ZLIB 123 123". + std::string AttrToString() const; + // Returns true if the column has a read default value bool has_read_default() const { return read_default_ != nullptr; @@ -724,11 +736,12 @@ class Schema { } // Enum to configure how a Schema is stringified. - enum class ToStringMode { + enum ToStringMode { + BASE_INFO = 0, // Include column ids if this instance has them. - WITH_COLUMN_IDS, - // Do not include column ids. - WITHOUT_COLUMN_IDS, + WITH_COLUMN_IDS = 1 << 0, + // Include column attributes. + WITH_COLUMN_ATTRIBUTES = 1 << 1, }; // Stringify this Schema. This is not particularly efficient, // so should only be used when necessary for output. diff --git a/src/kudu/common/types.h b/src/kudu/common/types.h index b3603da..36f9b6f 100644 --- a/src/kudu/common/types.h +++ b/src/kudu/common/types.h @@ -37,6 +37,7 @@ #include "kudu/gutil/strings/escaping.h" #include "kudu/gutil/strings/numbers.h" #include "kudu/util/int128.h" +#include "kudu/util/int128_util.h" #include "kudu/util/slice.h" // IWYU pragma: no_include "kudu/util/status.h" diff --git a/src/kudu/tools/kudu-admin-test.cc b/src/kudu/tools/kudu-admin-test.cc index a870026..a5c8b0c 100644 --- a/src/kudu/tools/kudu-admin-test.cc +++ b/src/kudu/tools/kudu-admin-test.cc @@ -38,6 +38,7 @@ #include "kudu/client/client.h" #include "kudu/client/schema.h" #include "kudu/client/shared_ptr.h" +#include "kudu/client/value.h" #include "kudu/client/write_op.h" #include "kudu/common/common.pb.h" #include "kudu/common/partial_row.h" @@ -66,6 +67,7 @@ #include "kudu/util/net/net_util.h" #include "kudu/util/net/sockaddr.h" #include "kudu/util/pb_util.h" +#include "kudu/util/slice.h" #include "kudu/util/status.h" #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" @@ -82,12 +84,14 @@ DECLARE_int32(num_tablet_servers); using kudu::client::KuduClient; using kudu::client::KuduClientBuilder; using kudu::client::KuduColumnSchema; +using kudu::client::KuduColumnStorageAttributes; using kudu::client::KuduInsert; using kudu::client::KuduSchema; using kudu::client::KuduSchemaBuilder; using kudu::client::KuduTable; using kudu::client::KuduTableAlterer; using kudu::client::KuduTableCreator; +using kudu::client::KuduValue; using kudu::client::sp::shared_ptr; using kudu::cluster::ExternalTabletServer; using kudu::consensus::COMMITTED_OPID; @@ -1730,18 +1734,31 @@ TEST_F(AdminCliTest, TestDescribeTable) { builder.AddColumn("key_hash1")->Type(KuduColumnSchema::INT32)->NotNull(); builder.AddColumn("key_hash2")->Type(KuduColumnSchema::INT32)->NotNull(); builder.AddColumn("key_range")->Type(KuduColumnSchema::INT32)->NotNull(); - builder.AddColumn("int8_val")->Type(KuduColumnSchema::INT8); - builder.AddColumn("int16_val")->Type(KuduColumnSchema::INT16); - builder.AddColumn("int32_val")->Type(KuduColumnSchema::INT32); - builder.AddColumn("int64_val")->Type(KuduColumnSchema::INT64); + builder.AddColumn("int8_val")->Type(KuduColumnSchema::INT8) + ->Compression(KuduColumnStorageAttributes::CompressionType::NO_COMPRESSION) + ->Encoding(KuduColumnStorageAttributes::EncodingType::PLAIN_ENCODING); + builder.AddColumn("int16_val")->Type(KuduColumnSchema::INT16) + ->Compression(KuduColumnStorageAttributes::CompressionType::SNAPPY) + ->Encoding(KuduColumnStorageAttributes::EncodingType::RLE); + builder.AddColumn("int32_val")->Type(KuduColumnSchema::INT32) + ->Compression(KuduColumnStorageAttributes::CompressionType::LZ4) + ->Encoding(KuduColumnStorageAttributes::EncodingType::BIT_SHUFFLE); + builder.AddColumn("int64_val")->Type(KuduColumnSchema::INT64) + ->Compression(KuduColumnStorageAttributes::CompressionType::ZLIB) + ->Default(KuduValue::FromInt(123)); builder.AddColumn("timestamp_val")->Type(KuduColumnSchema::UNIXTIME_MICROS); - builder.AddColumn("string_val")->Type(KuduColumnSchema::STRING); - builder.AddColumn("bool_val")->Type(KuduColumnSchema::BOOL); + builder.AddColumn("string_val")->Type(KuduColumnSchema::STRING) + ->Encoding(KuduColumnStorageAttributes::EncodingType::PREFIX_ENCODING) + ->Default(KuduValue::CopyString(Slice("hello")));; + builder.AddColumn("bool_val")->Type(KuduColumnSchema::BOOL) + ->Default(KuduValue::FromBool(false)); builder.AddColumn("float_val")->Type(KuduColumnSchema::FLOAT); - builder.AddColumn("double_val")->Type(KuduColumnSchema::DOUBLE); - builder.AddColumn("binary_val")->Type(KuduColumnSchema::BINARY); + builder.AddColumn("double_val")->Type(KuduColumnSchema::DOUBLE) + ->Default(KuduValue::FromDouble(123.4)); + builder.AddColumn("binary_val")->Type(KuduColumnSchema::BINARY) + ->Encoding(KuduColumnStorageAttributes::EncodingType::DICT_ENCODING); builder.AddColumn("decimal_val")->Type(KuduColumnSchema::DECIMAL) - ->Precision(10) + ->Precision(30) ->Scale(4); builder.SetPrimaryKey({ "key_hash0", "key_hash1", "key_hash2", "key_range" }); ASSERT_OK(builder.Build(&schema)); @@ -1776,7 +1793,7 @@ TEST_F(AdminCliTest, TestDescribeTable) { "table", "describe", cluster_->master()->bound_rpc_addr().ToString(), - kAnotherTableId + kAnotherTableId, }, &stdout, &stderr); ASSERT_TRUE(s.ok()) << ToolRunInfo(s, stdout, stderr); @@ -1797,7 +1814,47 @@ TEST_F(AdminCliTest, TestDescribeTable) { " float_val FLOAT NULLABLE,\n" " double_val DOUBLE NULLABLE,\n" " binary_val BINARY NULLABLE,\n" - " decimal_val DECIMAL(10, 4) NULLABLE,\n" + " decimal_val DECIMAL(30, 4) NULLABLE,\n" + " PRIMARY KEY (key_hash0, key_hash1, key_hash2, key_range)\n" + ")\n" + "HASH (key_hash0) PARTITIONS 2,\n" + "HASH (key_hash1, key_hash2) PARTITIONS 3,\n" + "RANGE (key_range) (\n" + " PARTITION 0 <= VALUES < 1,\n" + " PARTITION 2 <= VALUES < 3\n" + ")\n" + "REPLICAS 1"); + + // Test the describe output with `-show_attributes=true`. + stdout.clear(); + stderr.clear(); + s = RunKuduTool({ + "table", + "describe", + cluster_->master()->bound_rpc_addr().ToString(), + kAnotherTableId, + "-show_attributes=true" + }, &stdout, &stderr); + ASSERT_TRUE(s.ok()) << ToolRunInfo(s, stdout, stderr); + + ASSERT_STR_CONTAINS( + stdout, + "(\n" + " key_hash0 INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" + " key_hash1 INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" + " key_hash2 INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" + " key_range INT32 NOT NULL AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" + " int8_val INT8 NULLABLE PLAIN_ENCODING NO_COMPRESSION - -,\n" + " int16_val INT16 NULLABLE RLE SNAPPY - -,\n" + " int32_val INT32 NULLABLE BIT_SHUFFLE LZ4 - -,\n" + " int64_val INT64 NULLABLE AUTO_ENCODING ZLIB 123 123,\n" + " timestamp_val UNIXTIME_MICROS NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" + " string_val STRING NULLABLE PREFIX_ENCODING DEFAULT_COMPRESSION \"hello\" \"hello\",\n" + " bool_val BOOL NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION false false,\n" + " float_val FLOAT NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" + " double_val DOUBLE NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION 123.4 123.4,\n" + " binary_val BINARY NULLABLE DICT_ENCODING DEFAULT_COMPRESSION - -,\n" + " decimal_val DECIMAL(30, 4) NULLABLE AUTO_ENCODING DEFAULT_COMPRESSION - -,\n" " PRIMARY KEY (key_hash0, key_hash1, key_hash2, key_range)\n" ")\n" "HASH (key_hash0) PARTITIONS 2,\n" diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 1c0633f..9838620 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -112,7 +112,6 @@ #include "kudu/tserver/tserver_admin.proxy.h" #include "kudu/util/async_util.h" #include "kudu/util/env.h" -#include "kudu/util/int128_util.h" // IWYU pragma: keep #include "kudu/util/metrics.h" #include "kudu/util/monotime.h" #include "kudu/util/net/net_util.h" diff --git a/src/kudu/tools/tool_action_table.cc b/src/kudu/tools/tool_action_table.cc index a0c3f06..4710c12 100644 --- a/src/kudu/tools/tool_action_table.cc +++ b/src/kudu/tools/tool_action_table.cc @@ -418,6 +418,7 @@ unique_ptr<Mode> BuildTableMode() { .Description("Describe a table") .AddRequiredParameter({ kMasterAddressesArg, kMasterAddressesArgDesc }) .AddRequiredParameter({ kTableNameArg, "Name of the table to describe" }) + .AddOptionalParameter("show_attributes") .Build(); unique_ptr<Action> list_tables = diff --git a/src/kudu/util/int128_util.h b/src/kudu/util/int128_util.h index 2d01de7..5965da1 100644 --- a/src/kudu/util/int128_util.h +++ b/src/kudu/util/int128_util.h @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#pragma once + #include "kudu/util/int128.h" #include <iostream> @@ -36,4 +38,3 @@ inline std::ostream& operator<<(std::ostream& os, const unsigned __int128& val) } } // namespace std -