Repository: kudu Updated Branches: refs/heads/master 723ced836 -> 0815a1238
KUDU-721: Support range partitions on decimal columns Add the DataType handling to enable range partitions on decimal columns and a test that fails without the changes. Also adds range partition tests for edge cases where: * The upper bound is less than than the lower bound * The upper bound is equal to the lower bound * The bounds use MIN/MAX values Change-Id: I419d442dd233c73166c6e391907c8443ebecc917 Reviewed-on: http://gerrit.cloudera.org:8080/9363 Reviewed-by: Alexey Serbin <[email protected]> Tested-by: Kudu Jenkins Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/0815a123 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/0815a123 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/0815a123 Branch: refs/heads/master Commit: 0815a1238d18e3a82c33018ea07014d202a16018 Parents: 723ced8 Author: Grant Henke <[email protected]> Authored: Mon Feb 19 20:45:45 2018 -0600 Committer: Grant Henke <[email protected]> Committed: Wed Feb 21 19:40:35 2018 +0000 ---------------------------------------------------------------------- src/kudu/client/client-test.cc | 122 ++++++++++++++++++++++++++++++-- src/kudu/common/partition.cc | 26 ++++++- src/kudu/util/decimal_util-test.cc | 8 +++ src/kudu/util/decimal_util.cc | 4 ++ src/kudu/util/decimal_util.h | 6 +- 5 files changed, 156 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/0815a123/src/kudu/client/client-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc index d8993fb..4a44761 100644 --- a/src/kudu/client/client-test.cc +++ b/src/kudu/client/client-test.cc @@ -1611,29 +1611,137 @@ TEST_F(ClientTest, TestExclusiveInclusiveUnixTimeMicrosRangeBounds) { // Create test table with range partition using non-default bound types. // KUDU-1722 KuduSchemaBuilder builder; - KuduSchema u_schema_; + KuduSchema schema; builder.AddColumn("key")->Type(KuduColumnSchema::UNIXTIME_MICROS)->NotNull()->PrimaryKey(); builder.AddColumn("value")->Type(KuduColumnSchema::INT32)->NotNull(); - CHECK_OK(builder.Build(&u_schema_)); - const string table_name = "TestExclusiveInclusiveUnixTimeMicrosRangeBounds"; - shared_ptr<KuduTable> table; + ASSERT_OK(builder.Build(&schema)); - unique_ptr<KuduPartialRow> lower_bound(u_schema_.NewRow()); + unique_ptr<KuduPartialRow> lower_bound(schema.NewRow()); ASSERT_OK(lower_bound->SetUnixTimeMicros("key", -1)); - unique_ptr<KuduPartialRow> upper_bound(u_schema_.NewRow()); + unique_ptr<KuduPartialRow> upper_bound(schema.NewRow()); ASSERT_OK(upper_bound->SetUnixTimeMicros("key", 99)); unique_ptr<KuduTableCreator> table_creator(client_->NewTableCreator()); table_creator->add_range_partition(lower_bound.release(), upper_bound.release(), KuduTableCreator::EXCLUSIVE_BOUND, KuduTableCreator::INCLUSIVE_BOUND); + + const string table_name = "TestExclusiveInclusiveUnixTimeMicrosRangeBounds"; ASSERT_OK(table_creator->table_name(table_name) - .schema(&u_schema_) + .schema(&schema) .num_replicas(1) .set_range_partition_columns({ "key" }) .Create()); } +TEST_F(ClientTest, TestExclusiveInclusiveDecimalRangeBounds) { + KuduSchemaBuilder builder; + KuduSchema schema; + builder.AddColumn("key")->Type(KuduColumnSchema::DECIMAL)->NotNull()->PrimaryKey() + ->Precision(9)->Scale(2); + builder.AddColumn("value")->Type(KuduColumnSchema::INT32)->NotNull(); + ASSERT_OK(builder.Build(&schema)); + + unique_ptr<KuduPartialRow> lower_bound(schema.NewRow()); + ASSERT_OK(lower_bound->SetUnscaledDecimal("key", -1)); + unique_ptr<KuduPartialRow> upper_bound(schema.NewRow()); + ASSERT_OK(upper_bound->SetUnscaledDecimal("key", 99)); + + unique_ptr<KuduTableCreator> table_creator(client_->NewTableCreator()); + table_creator->add_range_partition(lower_bound.release(), upper_bound.release(), + KuduTableCreator::EXCLUSIVE_BOUND, + KuduTableCreator::INCLUSIVE_BOUND); + + ASSERT_OK(table_creator->table_name("TestExclusiveInclusiveDecimalRangeBounds") + .schema(&schema) + .num_replicas(1) + .set_range_partition_columns({ "key" }) + .Create()); +} + +TEST_F(ClientTest, TestSwappedRangeBounds) { + KuduSchemaBuilder builder; + KuduSchema schema; + builder.AddColumn("key")->Type(KuduColumnSchema::INT32)->NotNull()->PrimaryKey(); + builder.AddColumn("value")->Type(KuduColumnSchema::INT32)->NotNull(); + ASSERT_OK(builder.Build(&schema)); + + unique_ptr<KuduPartialRow> lower_bound(schema.NewRow()); + ASSERT_OK(lower_bound->SetInt32("key", 90)); + unique_ptr<KuduPartialRow> upper_bound(schema.NewRow()); + ASSERT_OK(upper_bound->SetInt32("key", -1)); + + unique_ptr<KuduTableCreator> table_creator(client_->NewTableCreator()); + table_creator->add_range_partition(lower_bound.release(), upper_bound.release(), + KuduTableCreator::EXCLUSIVE_BOUND, + KuduTableCreator::INCLUSIVE_BOUND); + + Status s = table_creator->table_name("TestSwappedRangeBounds") + .schema(&schema) + .num_replicas(1) + .set_range_partition_columns({ "key" }) + .Create(); + + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_STR_CONTAINS(s.ToString(), + "Error creating table TestSwappedRangeBounds on the master: " + "range partition lower bound must be less than the upper bound"); +} + + +TEST_F(ClientTest, TestEqualRangeBounds) { + KuduSchemaBuilder builder; + KuduSchema schema; + builder.AddColumn("key")->Type(KuduColumnSchema::INT32)->NotNull()->PrimaryKey(); + builder.AddColumn("value")->Type(KuduColumnSchema::INT32)->NotNull(); + ASSERT_OK(builder.Build(&schema)); + + unique_ptr<KuduPartialRow> lower_bound(schema.NewRow()); + ASSERT_OK(lower_bound->SetInt32("key", 10)); + unique_ptr<KuduPartialRow> upper_bound(schema.NewRow()); + ASSERT_OK(upper_bound->SetInt32("key", 10)); + + unique_ptr<KuduTableCreator> table_creator(client_->NewTableCreator()); + table_creator->add_range_partition(lower_bound.release(), upper_bound.release(), + KuduTableCreator::EXCLUSIVE_BOUND, + KuduTableCreator::INCLUSIVE_BOUND); + + Status s = table_creator->table_name("TestEqualRangeBounds") + .schema(&schema) + .num_replicas(1) + .set_range_partition_columns({ "key" }) + .Create(); + + ASSERT_TRUE(s.IsInvalidArgument()); + ASSERT_STR_CONTAINS(s.ToString(), + "Error creating table TestEqualRangeBounds on the master: " + "range partition lower bound must be less than the upper bound"); +} + +TEST_F(ClientTest, TestMinMaxRangeBounds) { + KuduSchemaBuilder builder; + KuduSchema schema; + builder.AddColumn("key")->Type(KuduColumnSchema::INT32)->NotNull()->PrimaryKey(); + builder.AddColumn("value")->Type(KuduColumnSchema::INT32)->NotNull(); + ASSERT_OK(builder.Build(&schema)); + + unique_ptr<KuduPartialRow> lower_bound(schema.NewRow()); + ASSERT_OK(lower_bound->SetInt32("key", INT32_MIN)); + unique_ptr<KuduPartialRow> upper_bound(schema.NewRow()); + ASSERT_OK(upper_bound->SetInt32("key", INT32_MAX)); + + unique_ptr<KuduTableCreator> table_creator(client_->NewTableCreator()); + table_creator->add_range_partition(lower_bound.release(), upper_bound.release(), + KuduTableCreator::EXCLUSIVE_BOUND, + KuduTableCreator::INCLUSIVE_BOUND); + + ASSERT_OK(table_creator->table_name("TestMinMaxRangeBounds") + .schema(&schema) + .num_replicas(1) + .set_range_partition_columns({ "key" }) + .Create()); +} + TEST_F(ClientTest, TestMetaCacheExpiry) { google::FlagSaver saver; FLAGS_table_locations_ttl_ms = 25; http://git-wip-us.apache.org/repos/asf/kudu/blob/0815a123/src/kudu/common/partition.cc ---------------------------------------------------------------------- diff --git a/src/kudu/common/partition.cc b/src/kudu/common/partition.cc index 2132f26..0fd8850 100644 --- a/src/kudu/common/partition.cc +++ b/src/kudu/common/partition.cc @@ -38,7 +38,9 @@ #include "kudu/gutil/strings/join.h" #include "kudu/gutil/strings/substitute.h" #include "kudu/util/bitmap.h" +#include "kudu/util/decimal_util.h" #include "kudu/util/hash_util.h" +#include "kudu/util/int128.h" #include "kudu/util/logging.h" #include "kudu/util/memory/arena.h" #include "kudu/util/pb_util.h" @@ -1000,7 +1002,8 @@ namespace { // Increments an unset column in the row. Status IncrementUnsetColumn(KuduPartialRow* row, int32_t idx) { DCHECK(!row->IsColumnSet(idx)); - switch (row->schema()->column(idx).type_info()->type()) { + const ColumnSchema& col = row->schema()->column(idx); + switch (col.type_info()->type()) { case INT8: RETURN_NOT_OK(row->SetInt8(idx, INT8_MIN + 1)); break; @@ -1020,6 +1023,12 @@ namespace { case BINARY: RETURN_NOT_OK(row->SetBinaryCopy(idx, Slice("\0", 1))); break; + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: + RETURN_NOT_OK(row->SetUnscaledDecimal(idx, + MinUnscaledDecimal(col.type_attributes().precision) + 1)); + break; default: return Status::InvalidArgument("Invalid column type in range partition", row->schema()->column(idx).ToString()); @@ -1031,8 +1040,9 @@ namespace { // succeeds, or false if the column is already the maximum value. Status IncrementColumn(KuduPartialRow* row, int32_t idx, bool* success) { DCHECK(row->IsColumnSet(idx)); + const ColumnSchema& col = row->schema()->column(idx); *success = true; - switch (row->schema()->column(idx).type_info()->type()) { + switch (col.type_info()->type()) { case INT8: { int8_t value; RETURN_NOT_OK(row->GetInt8(idx, &value)); @@ -1083,6 +1093,18 @@ namespace { } break; } + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: { + int128_t value; + RETURN_NOT_OK(row->GetUnscaledDecimal(idx, &value)); + if (value < MaxUnscaledDecimal(col.type_attributes().precision)) { + RETURN_NOT_OK(row->SetUnscaledDecimal(idx, value + 1)); + } else { + *success = false; + } + break; + } case BINARY: { Slice value; RETURN_NOT_OK(row->GetBinary(idx, &value)); http://git-wip-us.apache.org/repos/asf/kudu/blob/0815a123/src/kudu/util/decimal_util-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/decimal_util-test.cc b/src/kudu/util/decimal_util-test.cc index e8c9fc1..d7bfc35 100644 --- a/src/kudu/util/decimal_util-test.cc +++ b/src/kudu/util/decimal_util-test.cc @@ -33,6 +33,14 @@ TEST(TestDecimalUtil, TestMaxUnscaledDecimal) { ASSERT_EQ(kMaxUnscaledDecimal128, MaxUnscaledDecimal(kMaxDecimal128Precision)); } +TEST(TestDecimalUtil, TestMinUnscaledDecimal) { + ASSERT_EQ(-9, MinUnscaledDecimal(1)); + ASSERT_EQ(-99999, MinUnscaledDecimal(5)); + ASSERT_EQ(kMinUnscaledDecimal32, MinUnscaledDecimal(kMaxDecimal32Precision)); + ASSERT_EQ(kMinUnscaledDecimal64, MinUnscaledDecimal(kMaxDecimal64Precision)); + ASSERT_EQ(kMinUnscaledDecimal128, MinUnscaledDecimal(kMaxDecimal128Precision)); +} + TEST(TestDecimalUtil, TestToString) { ASSERT_EQ("999999999", DecimalToString(kMaxUnscaledDecimal32, kDefaultDecimalScale)); http://git-wip-us.apache.org/repos/asf/kudu/blob/0815a123/src/kudu/util/decimal_util.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/decimal_util.cc b/src/kudu/util/decimal_util.cc index af81ae5..d4f7e70 100644 --- a/src/kudu/util/decimal_util.cc +++ b/src/kudu/util/decimal_util.cc @@ -38,6 +38,10 @@ int128_t MaxUnscaledDecimal(int8_t precision) { return result - 1; } +int128_t MinUnscaledDecimal(int8_t precision) { + return -MaxUnscaledDecimal(precision); +} + // Workaround for an ASAN build issue documented here: // https://bugs.llvm.org/show_bug.cgi?id=16404 __attribute__((no_sanitize("undefined"))) http://git-wip-us.apache.org/repos/asf/kudu/blob/0815a123/src/kudu/util/decimal_util.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/decimal_util.h b/src/kudu/util/decimal_util.h index 271bc0e..a465412 100644 --- a/src/kudu/util/decimal_util.h +++ b/src/kudu/util/decimal_util.h @@ -57,9 +57,13 @@ namespace kudu { // The maximum scale is the Decimal's precision. // Returns the maximum unscaled decimal value that can be stored - // based on the precision + // based on the precision. int128_t MaxUnscaledDecimal(int8_t precision); + // Returns the maximum unscaled decimal value that can be stored + // based on the precision. + int128_t MinUnscaledDecimal(int8_t precision); + std::string DecimalToString(int128_t value, int8_t scale); } // namespace kudu
