This is an automated email from the ASF dual-hosted git repository. abukor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 4343fb746766b166f5532d36afbe903fdf092b95 Author: Attila Bukor <[email protected]> AuthorDate: Tue Sep 24 15:19:21 2019 +0200 KUDU-1938 Add C++ client support for VARCHAR pt 2 Change-Id: Ifaf609565b0a0a87d6e645cd3ac14c0965af5ba8 Reviewed-on: http://gerrit.cloudera.org:8080/13869 Reviewed-by: Grant Henke <[email protected]> Reviewed-by: Adar Dembo <[email protected]> Tested-by: Kudu Jenkins --- src/kudu/client/predicate-test.cc | 66 +++++++------ src/kudu/client/scan_batch.cc | 14 +++ src/kudu/client/scan_batch.h | 10 +- src/kudu/client/schema-internal.h | 7 +- src/kudu/client/schema.cc | 131 ++++++++++++++++++-------- src/kudu/client/schema.h | 36 +++++++ src/kudu/client/value.cc | 1 + src/kudu/integration-tests/CMakeLists.txt | 1 + src/kudu/integration-tests/all_types-itest.cc | 12 +++ src/kudu/integration-tests/varchar-itest.cc | 110 +++++++++++++++++++++ 10 files changed, 318 insertions(+), 70 deletions(-) diff --git a/src/kudu/client/predicate-test.cc b/src/kudu/client/predicate-test.cc index e316d10..dd0928f 100644 --- a/src/kudu/client/predicate-test.cc +++ b/src/kudu/client/predicate-test.cc @@ -19,6 +19,7 @@ #include <cstdint> #include <limits> #include <memory> +#include <ostream> #include <string> #include <vector> @@ -26,10 +27,10 @@ #include <gtest/gtest.h> #include "kudu/client/client.h" -#include "kudu/client/shared_ptr.h" #include "kudu/client/scan_batch.h" #include "kudu/client/scan_predicate.h" #include "kudu/client/schema.h" +#include "kudu/client/shared_ptr.h" #include "kudu/client/value.h" #include "kudu/client/write_op.h" #include "kudu/common/partial_row.h" @@ -74,7 +75,19 @@ class PredicateTest : public KuduTest { { KuduSchemaBuilder builder; builder.AddColumn("key")->NotNull()->Type(KuduColumnSchema::INT64)->PrimaryKey(); - builder.AddColumn("value")->Type(value_type); + switch (value_type) { + case KuduColumnSchema::VARCHAR: + builder.AddColumn("value")->Type(value_type) + ->Length(40); + break; + case KuduColumnSchema::DECIMAL: + builder.AddColumn("value")->Type(value_type) + ->Precision(kMaxDecimal128Precision)->Scale(2); + break; + default: + builder.AddColumn("value")->Type(value_type); + break; + } CHECK_OK(builder.Build(&schema)); } unique_ptr<client::KuduTableCreator> table_creator(client_->NewTableCreator()); @@ -387,9 +400,8 @@ class PredicateTest : public KuduTest { } // Check string predicates against the specified table. - void CheckStringPredicates(shared_ptr<KuduTable> table) { + void CheckStringPredicates(const shared_ptr<KuduTable>& table, vector<string> values) { - vector<string> values = CreateStringValues(); ASSERT_EQ(values.size() + 1, CountRows(table, {})); // Add some additional values to check against. @@ -1118,29 +1130,16 @@ TEST_F(PredicateTest, TestDecimalPredicates) { ASSERT_EQ(1, CountRows(table, { table->NewIsNullPredicate("value") })); } -TEST_F(PredicateTest, TestStringPredicates) { - shared_ptr<KuduTable> table = CreateAndOpenTable(KuduColumnSchema::STRING); - shared_ptr<KuduSession> session = CreateSession(); +class ParameterizedPredicateTest : public PredicateTest, + public ::testing::WithParamInterface<KuduColumnSchema::DataType> {}; - vector<string> values = CreateStringValues(); - int i = 0; - for (const string& value : values) { - unique_ptr<KuduInsert> insert(table->NewInsert()); - ASSERT_OK(insert->mutable_row()->SetInt64("key", i++)); - ASSERT_OK(insert->mutable_row()->SetStringNoCopy("value", value)); - ASSERT_OK(session->Apply(insert.release())); - } - unique_ptr<KuduInsert> null_insert(table->NewInsert()); - ASSERT_OK(null_insert->mutable_row()->SetInt64("key", i++)); - ASSERT_OK(null_insert->mutable_row()->SetNull("value")); - ASSERT_OK(session->Apply(null_insert.release())); - ASSERT_OK(session->Flush()); - - CheckStringPredicates(table); -} +INSTANTIATE_TEST_CASE_P(, ParameterizedPredicateTest, + ::testing::Values(KuduColumnSchema::STRING, + KuduColumnSchema::BINARY, + KuduColumnSchema::VARCHAR)); -TEST_F(PredicateTest, TestBinaryPredicates) { - shared_ptr<KuduTable> table = CreateAndOpenTable(KuduColumnSchema::BINARY); +TEST_P(ParameterizedPredicateTest, TestIndirectDataPredicates) { + shared_ptr<KuduTable> table = CreateAndOpenTable(GetParam()); shared_ptr<KuduSession> session = CreateSession(); vector<string> values = CreateStringValues(); @@ -1148,7 +1147,20 @@ TEST_F(PredicateTest, TestBinaryPredicates) { for (const string& value : values) { unique_ptr<KuduInsert> insert(table->NewInsert()); ASSERT_OK(insert->mutable_row()->SetInt64("key", i++)); - ASSERT_OK(insert->mutable_row()->SetBinaryNoCopy("value", value)); + switch (GetParam()) { + case KuduColumnSchema::STRING: + ASSERT_OK(insert->mutable_row()->SetStringNoCopy("value", value)); + break; + case KuduColumnSchema::BINARY: + ASSERT_OK(insert->mutable_row()->SetBinaryNoCopy("value", value)); + break; + case KuduColumnSchema::VARCHAR: + ASSERT_OK(insert->mutable_row()->SetVarchar("value", value)); + break; + default: + LOG(FATAL) << "Invalid type"; + break; + } ASSERT_OK(session->Apply(insert.release())); } unique_ptr<KuduInsert> null_insert(table->NewInsert()); @@ -1157,7 +1169,7 @@ TEST_F(PredicateTest, TestBinaryPredicates) { ASSERT_OK(session->Apply(null_insert.release())); ASSERT_OK(session->Flush()); - CheckStringPredicates(table); + CheckStringPredicates(table, values); } } // namespace client diff --git a/src/kudu/client/scan_batch.cc b/src/kudu/client/scan_batch.cc index 8e59cee..ec5b155 100644 --- a/src/kudu/client/scan_batch.cc +++ b/src/kudu/client/scan_batch.cc @@ -166,6 +166,10 @@ Status KuduScanBatch::RowPtr::GetBinary(const Slice& col_name, Slice* val) const return Get<TypeTraits<BINARY> >(col_name, val); } +Status KuduScanBatch::RowPtr::GetVarchar(const Slice& col_name, Slice* val) const { + return Get<TypeTraits<VARCHAR> >(col_name, val); +} + Status KuduScanBatch::RowPtr::GetBool(int col_idx, bool* val) const { return Get<TypeTraits<BOOL> >(col_idx, val); } @@ -206,6 +210,10 @@ Status KuduScanBatch::RowPtr::GetBinary(int col_idx, Slice* val) const { return Get<TypeTraits<BINARY> >(col_idx, val); } +Status KuduScanBatch::RowPtr::GetVarchar(int col_idx, Slice* val) const { + return Get<TypeTraits<VARCHAR> >(col_idx, val); +} + template<typename T> Status KuduScanBatch::RowPtr::Get(const Slice& col_name, typename T::cpp_type* val) const { int col_idx; @@ -277,6 +285,9 @@ template Status KuduScanBatch::RowPtr::Get<TypeTraits<BINARY> >(const Slice& col_name, Slice* val) const; template +Status KuduScanBatch::RowPtr::Get<TypeTraits<VARCHAR> >(const Slice& col_name, Slice* val) const; + +template Status KuduScanBatch::RowPtr::Get<TypeTraits<BOOL> >(int col_idx, bool* val) const; template @@ -310,6 +321,9 @@ template Status KuduScanBatch::RowPtr::Get<TypeTraits<BINARY> >(int col_idx, Slice* val) const; template +Status KuduScanBatch::RowPtr::Get<TypeTraits<VARCHAR> >(int col_idx, Slice* val) const; + +template Status KuduScanBatch::RowPtr::Get<TypeTraits<DECIMAL32> >(int col_idx, int32_t* val) const; template diff --git a/src/kudu/client/scan_batch.h b/src/kudu/client/scan_batch.h index fd211f5..41e381c 100644 --- a/src/kudu/client/scan_batch.h +++ b/src/kudu/client/scan_batch.h @@ -251,9 +251,9 @@ class KUDU_EXPORT KuduScanBatch::RowPtr { #endif ///@} - /// @name Getters for string/binary column by column name. + /// @name Getters for string/binary/varchar column by column name. /// - /// Get the string/binary value for a column by its name. + /// Get the string/binary/varchar value for a column by its name. /// /// @param [in] col_name /// Name of the column. @@ -270,11 +270,12 @@ class KUDU_EXPORT KuduScanBatch::RowPtr { ///@{ Status GetString(const Slice& col_name, Slice* val) const WARN_UNUSED_RESULT; Status GetBinary(const Slice& col_name, Slice* val) const WARN_UNUSED_RESULT; + Status GetVarchar(const Slice& col_name, Slice* val) const WARN_UNUSED_RESULT; ///@} - /// @name Getters for string/binary column by column index. + /// @name Getters for string/binary/varchar column by column index. /// - /// Get the string/binary value for a column by its index. + /// Get the string/binary/varchar value for a column by its index. /// /// These methods are faster than their name-based counterparts /// since using indices avoids a hashmap lookup, so index-based getters @@ -295,6 +296,7 @@ class KUDU_EXPORT KuduScanBatch::RowPtr { ///@{ Status GetString(int col_idx, Slice* val) const WARN_UNUSED_RESULT; Status GetBinary(int col_idx, Slice* val) const WARN_UNUSED_RESULT; + Status GetVarchar(int col_idx, Slice* val) const WARN_UNUSED_RESULT; ///@} /// Get the column's row data. diff --git a/src/kudu/client/schema-internal.h b/src/kudu/client/schema-internal.h index 5ff642e..0330f02 100644 --- a/src/kudu/client/schema-internal.h +++ b/src/kudu/client/schema-internal.h @@ -47,13 +47,15 @@ KuduColumnSchema::DataType FromInternalDataType(kudu::DataType type); class KuduColumnTypeAttributes::Data { public: - Data(int8_t precision, int8_t scale) + Data(int8_t precision, int8_t scale, uint16_t length) : precision(precision), - scale(scale) { + scale(scale), + length(length) { } int8_t precision; int8_t scale; + uint16_t length; }; class KuduColumnSpec::Data { @@ -75,6 +77,7 @@ class KuduColumnSpec::Data { boost::optional<KuduColumnSchema::DataType> type; boost::optional<int8_t> precision; boost::optional<int8_t> scale; + boost::optional<uint16_t> length; boost::optional<KuduColumnStorageAttributes::EncodingType> encoding; boost::optional<KuduColumnStorageAttributes::CompressionType> compression; boost::optional<int32_t> block_size; diff --git a/src/kudu/client/schema.cc b/src/kudu/client/schema.cc index eaae666..41ab90c 100644 --- a/src/kudu/client/schema.cc +++ b/src/kudu/client/schema.cc @@ -39,6 +39,7 @@ #include "kudu/gutil/macros.h" #include "kudu/gutil/map-util.h" #include "kudu/gutil/strings/substitute.h" +#include "kudu/util/char_util.h" #include "kudu/util/compression/compression.pb.h" #include "kudu/util/decimal_util.h" #include "kudu/util/slice.h" @@ -126,6 +127,7 @@ kudu::DataType ToInternalDataType(KuduColumnSchema::DataType type, case KuduColumnSchema::UNIXTIME_MICROS: return kudu::UNIXTIME_MICROS; case KuduColumnSchema::FLOAT: return kudu::FLOAT; case KuduColumnSchema::DOUBLE: return kudu::DOUBLE; + case KuduColumnSchema::VARCHAR: return kudu::VARCHAR; case KuduColumnSchema::STRING: return kudu::STRING; case KuduColumnSchema::BINARY: return kudu::BINARY; case KuduColumnSchema::BOOL: return kudu::BOOL; @@ -152,6 +154,7 @@ KuduColumnSchema::DataType FromInternalDataType(kudu::DataType type) { case kudu::UNIXTIME_MICROS: return KuduColumnSchema::UNIXTIME_MICROS; case kudu::FLOAT: return KuduColumnSchema::FLOAT; case kudu::DOUBLE: return KuduColumnSchema::DOUBLE; + case kudu::VARCHAR: return KuduColumnSchema::VARCHAR; case kudu::STRING: return KuduColumnSchema::STRING; case kudu::BINARY: return KuduColumnSchema::BINARY; case kudu::BOOL: return KuduColumnSchema::BOOL; @@ -167,7 +170,7 @@ KuduColumnSchema::DataType FromInternalDataType(kudu::DataType type) { //////////////////////////////////////////////////////////// KuduColumnTypeAttributes::KuduColumnTypeAttributes() - : data_(new Data(0, 0)) { + : data_(new Data(0, 0, 0)) { } KuduColumnTypeAttributes::KuduColumnTypeAttributes(const KuduColumnTypeAttributes& other) @@ -176,7 +179,15 @@ KuduColumnTypeAttributes::KuduColumnTypeAttributes(const KuduColumnTypeAttribute } KuduColumnTypeAttributes::KuduColumnTypeAttributes(int8_t precision, int8_t scale) - : data_(new Data(precision, scale)) { + : data_(new Data(precision, scale, 0)) { +} + +KuduColumnTypeAttributes::KuduColumnTypeAttributes(uint16_t length) + : data_(new Data(0, 0, length)) { +} + +KuduColumnTypeAttributes::KuduColumnTypeAttributes(int8_t precision, int8_t scale, uint16_t length) + : data_(new Data(precision, scale, length)) { } KuduColumnTypeAttributes::~KuduColumnTypeAttributes() { @@ -204,6 +215,10 @@ int8_t KuduColumnTypeAttributes::scale() const { return data_->scale; } +uint16_t KuduColumnTypeAttributes::length() const { + return data_->length; +} + //////////////////////////////////////////////////////////// // KuduColumnSpec //////////////////////////////////////////////////////////// @@ -257,6 +272,11 @@ KuduColumnSpec* KuduColumnSpec::Scale(int8_t scale) { return this; } +KuduColumnSpec* KuduColumnSpec::Length(uint16_t length) { + data_->length = length; + return this; +} + KuduColumnSpec* KuduColumnSpec::PrimaryKey() { data_->primary_key = true; return this; @@ -303,46 +323,78 @@ Status KuduColumnSpec::ToColumnSchema(KuduColumnSchema* col) const { return Status::InvalidArgument("no type provided for column", data_->name); } - if (data_->type.value() == KuduColumnSchema::DECIMAL) { - if (!data_->precision) { - return Status::InvalidArgument("no precision provided for decimal column", data_->name); - } - if (data_->precision.value() < kMinDecimalPrecision || - data_->precision.value() > kMaxDecimalPrecision) { - return Status::InvalidArgument( - strings::Substitute("precision must be between $0 and $1", - kMinDecimalPrecision, - kMaxDecimalPrecision), data_->name); - } - if (data_->scale) { - if (data_->scale.value() < kMinDecimalScale) { + switch (data_->type.value()) { + case KuduColumnSchema::DECIMAL: + if (!data_->precision) { + return Status::InvalidArgument("no precision provided for decimal column", data_->name); + } + if (data_->precision.value() < kMinDecimalPrecision || + data_->precision.value() > kMaxDecimalPrecision) { + return Status::InvalidArgument( + strings::Substitute("precision must be between $0 and $1", + kMinDecimalPrecision, + kMaxDecimalPrecision), data_->name); + } + if (data_->scale) { + if (data_->scale.value() < kMinDecimalScale) { + return Status::InvalidArgument( + strings::Substitute("scale is less than the minimum value of $0", + kMinDecimalScale), data_->name); + } + if (data_->scale.value() > data_->precision.value()) { + return Status::InvalidArgument( + strings::Substitute("scale is greater than the precision value of", + data_->precision.value()), data_->name); + } + } + if (data_->length) { return Status::InvalidArgument( - strings::Substitute("scale is less than the minimum value of $0", - kMinDecimalScale), data_->name); + strings::Substitute("length is not applicable for column $0", + data_->type.value()), data_->name); } - if (data_->scale.value() > data_->precision.value()) { + break; + case KuduColumnSchema::VARCHAR: + if (data_->length.value() < kMinVarcharLength || + data_->length.value() > kMaxVarcharLength) { return Status::InvalidArgument( - strings::Substitute("scale is greater than the precision value of", - data_->precision.value()), data_->name); + strings::Substitute("length must be between $0 and $1", + kMinVarcharLength, + kMaxVarcharLength), data_->name); + } + if (data_->precision) { + return Status::InvalidArgument( + strings::Substitute("precision is not valid on a $0 column", + data_->type.value()), data_->name); + } + if (data_->scale) { + return Status::InvalidArgument( + strings::Substitute("scale is not valid on a $0 column", + data_->type.value()), data_->name); + } + break; + default: + if (data_->precision) { + return Status::InvalidArgument( + strings::Substitute("precision is not valid on a $0 column", + data_->type.value()), data_->name); + } + if (data_->scale) { + return Status::InvalidArgument( + strings::Substitute("scale is not valid on a $0 column", + data_->type.value()), data_->name); + } + if (data_->length) { + return Status::InvalidArgument( + strings::Substitute("length is not valid on a $0 column", + data_->type.value()), data_->name); } - } - } else { - if (data_->precision) { - return Status::InvalidArgument( - strings::Substitute("precision is not valid on a $0 column", - data_->type.value()), data_->name); - } - if (data_->scale) { - return Status::InvalidArgument( - strings::Substitute("scale is not valid on a $0 column", - data_->type.value()), data_->name); - } } - int8_t precision = (data_->precision) ? data_->precision.value() : 0; - int8_t scale = (data_->scale) ? data_->scale.value() : kDefaultDecimalScale; + int8_t precision = data_->precision ? data_->precision.value() : 0; + int8_t scale = data_->scale ? data_->scale.value() : kDefaultDecimalScale; + uint16_t length = data_->length ? data_->length.value() : 0; - KuduColumnTypeAttributes type_attrs(precision, scale); + KuduColumnTypeAttributes type_attrs(precision, scale, length); DataType internal_type = ToInternalDataType(data_->type.value(), type_attrs); bool nullable = data_->nullable ? data_->nullable.value() : true; @@ -575,6 +627,8 @@ string KuduColumnSchema::DataTypeToString(DataType type) { return "UNIXTIME_MICROS"; case DECIMAL: return "DECIMAL"; + case VARCHAR: + return "VARCHAR"; } LOG(FATAL) << "Unhandled type " << type; } @@ -593,6 +647,7 @@ KuduColumnSchema::KuduColumnSchema(const string &name, ColumnTypeAttributes type_attr_private; type_attr_private.precision = type_attributes.precision(); type_attr_private.scale = type_attributes.scale(); + type_attr_private.length = type_attributes.length(); col_ = new ColumnSchema(name, ToInternalDataType(type, type_attributes), is_nullable, default_value, default_value, attr_private, @@ -647,7 +702,8 @@ KuduColumnSchema::DataType KuduColumnSchema::type() const { KuduColumnTypeAttributes KuduColumnSchema::type_attributes() const { ColumnTypeAttributes type_attributes = DCHECK_NOTNULL(col_)->type_attributes(); - return KuduColumnTypeAttributes(type_attributes.precision, type_attributes.scale); + return KuduColumnTypeAttributes(type_attributes.precision, type_attributes.scale, + type_attributes.length); } const string& KuduColumnSchema::comment() const { @@ -712,7 +768,8 @@ KuduColumnSchema KuduSchema::Column(size_t idx) const { KuduColumnStorageAttributes attrs(FromInternalEncodingType(col.attributes().encoding), FromInternalCompressionType(col.attributes().compression)); #pragma GCC diagnostic pop - KuduColumnTypeAttributes type_attrs(col.type_attributes().precision, col.type_attributes().scale); + KuduColumnTypeAttributes type_attrs(col.type_attributes().precision, col.type_attributes().scale, + col.type_attributes().length); return KuduColumnSchema(col.name(), FromInternalDataType(col.type_info()->type()), col.is_nullable(), col.read_default_value(), attrs, type_attrs, col.comment()); diff --git a/src/kudu/client/schema.h b/src/kudu/client/schema.h index c5c3c52..ff6b49b 100644 --- a/src/kudu/client/schema.h +++ b/src/kudu/client/schema.h @@ -60,6 +60,8 @@ class MetaCacheEntry; class WriteRpc; } // namespace internal +class KuduColumnSchema; +class KuduColumnSpec; class KuduSchema; class KuduValue; @@ -82,6 +84,12 @@ class KUDU_EXPORT KuduColumnTypeAttributes { /// The scale of a decimal column. KuduColumnTypeAttributes(int8_t precision, int8_t scale); + /// Create a KuduColumnTypeAttributes object + /// + /// @param [in] length + /// The maximum length of a VARCHAR column in characters. + explicit KuduColumnTypeAttributes(uint16_t length); + ~KuduColumnTypeAttributes(); /// @name Assign/copy KuduColumnTypeAttributes. @@ -100,7 +108,16 @@ class KUDU_EXPORT KuduColumnTypeAttributes { /// @return Scale for the column type. int8_t scale() const; + /// @return Length for the column type. + uint16_t length() const; + private: + friend KuduColumnSchema; + friend KuduColumnSpec; + friend KuduSchema; + + KuduColumnTypeAttributes(int8_t precision, int8_t scale, uint16_t length); + class KUDU_NO_EXPORT Data; // Owned. Data* data_; @@ -189,6 +206,7 @@ class KUDU_EXPORT KuduColumnSchema { BINARY = 8, UNIXTIME_MICROS = 9, DECIMAL = 10, + VARCHAR = 11, TIMESTAMP = UNIXTIME_MICROS //!< deprecated, use UNIXTIME_MICROS }; @@ -382,6 +400,24 @@ class KUDU_EXPORT KuduColumnSpec { KuduColumnSpec* Scale(int8_t scale); ///@} + /// @name Operation only relevant for VARCHAR columns. + /// + ///@{ + /// Set the length for a column. + /// + /// Clients can specify a length for VARCHAR columns. + /// Length represents the maximum length of a VARCHAR column in + /// characters. + /// + /// The length must be greater than 0 and less than 65536. + /// If no length is provided a default length of 65535 is used. + /// + /// @param [in] length + /// Desired length to set. + /// @return Pointer to the modified object. + KuduColumnSpec* Length(uint16_t length); + ///@} + /// @name Operations only relevant for Create Table /// ///@{ diff --git a/src/kudu/client/value.cc b/src/kudu/client/value.cc index 479c625..51a8832 100644 --- a/src/kudu/client/value.cc +++ b/src/kudu/client/value.cc @@ -154,6 +154,7 @@ Status KuduValue::Data::CheckTypeAndGetPointer(const string& col_name, case kudu::BINARY: case kudu::STRING: + case kudu::VARCHAR: RETURN_NOT_OK(CheckAndPointToString(col_name, val_void)); break; diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt index 08eb111..22785bb 100644 --- a/src/kudu/integration-tests/CMakeLists.txt +++ b/src/kudu/integration-tests/CMakeLists.txt @@ -62,6 +62,7 @@ ADD_KUDU_TEST(alter_table-test PROCESSORS 3) ADD_KUDU_TEST(auth_token_expire-itest) ADD_KUDU_TEST(authz_token-itest PROCESSORS 2) ADD_KUDU_TEST(catalog_manager_tsk-itest PROCESSORS 2) +ADD_KUDU_TEST(varchar-itest) ADD_KUDU_TEST(client_failover-itest) ADD_KUDU_TEST(client-negotiation-failover-itest) ADD_KUDU_TEST(client-stress-test diff --git a/src/kudu/integration-tests/all_types-itest.cc b/src/kudu/integration-tests/all_types-itest.cc index 21c09b7..201d813 100644 --- a/src/kudu/integration-tests/all_types-itest.cc +++ b/src/kudu/integration-tests/all_types-itest.cc @@ -45,6 +45,7 @@ #include "kudu/integration-tests/cluster_verifier.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/util/bitmap.h" +#include "kudu/util/char_util.h" #include "kudu/util/decimal_util.h" #include "kudu/util/int128.h" #include "kudu/util/slice.h" @@ -255,6 +256,7 @@ struct ExpectedVals { string slice_content; Slice expected_slice_val; Slice expected_binary_val; + Slice expected_varchar_val; bool expected_bool_val; float expected_float_val; double expected_double_val; @@ -284,6 +286,7 @@ class AllTypesItest : public KuduTest { builder.AddColumn("int64_val")->Type(KuduColumnSchema::INT64); builder.AddColumn("timestamp_val")->Type(KuduColumnSchema::UNIXTIME_MICROS); builder.AddColumn("string_val")->Type(KuduColumnSchema::STRING); + builder.AddColumn("varchar_val")->Type(KuduColumnSchema::VARCHAR)->Length(kMaxVarcharLength); builder.AddColumn("bool_val")->Type(KuduColumnSchema::BOOL); builder.AddColumn("float_val")->Type(KuduColumnSchema::FLOAT); builder.AddColumn("double_val")->Type(KuduColumnSchema::DOUBLE); @@ -360,6 +363,7 @@ class AllTypesItest : public KuduTest { Slice slice_val(content); RETURN_NOT_OK(row->SetStringCopy("string_val", slice_val)); RETURN_NOT_OK(row->SetBinaryCopy("binary_val", slice_val)); + RETURN_NOT_OK(row->SetVarchar("varchar_val", slice_val)); double double_val = int_val; RETURN_NOT_OK(row->SetDouble("double_val", double_val)); RETURN_NOT_OK(row->SetFloat("float_val", double_val)); @@ -400,6 +404,7 @@ class AllTypesItest : public KuduTest { projection->push_back("timestamp_val"); projection->push_back("string_val"); projection->push_back("binary_val"); + projection->push_back("varchar_val"); projection->push_back("double_val"); projection->push_back("float_val"); projection->push_back("bool_val"); @@ -418,6 +423,7 @@ class AllTypesItest : public KuduTest { vals.expected_timestamp_val = expected_int_val; vals.slice_content = strings::Substitute("hello $0", expected_int_val); vals.expected_slice_val = Slice(vals.slice_content); + vals.expected_varchar_val = Slice(vals.slice_content); vals.expected_binary_val = Slice(vals.slice_content); vals.expected_bool_val = expected_int_val % 2; vals.expected_float_val = expected_int_val; @@ -452,6 +458,9 @@ class AllTypesItest : public KuduTest { Slice binary_val; ASSERT_OK(row.GetBinary("binary_val", &binary_val)); ASSERT_EQ(binary_val, vals.expected_binary_val); + Slice varchar_val; + ASSERT_OK(row.GetVarchar("varchar_val", &varchar_val)); + ASSERT_EQ(varchar_val, vals.expected_varchar_val); bool bool_val; ASSERT_OK(row.GetBool("bool_val", &bool_val)); ASSERT_EQ(bool_val, vals.expected_bool_val); @@ -660,6 +669,9 @@ TYPED_TEST(AllTypesItest, TestTimestampPadding) { case KuduColumnSchema::BINARY: ASSERT_EQ(*reinterpret_cast<const Slice*>(row_data), vals.expected_binary_val); break; + case KuduColumnSchema::VARCHAR: + ASSERT_EQ(*reinterpret_cast<const Slice*>(row_data), vals.expected_varchar_val); + break; case KuduColumnSchema::BOOL: ASSERT_EQ(*reinterpret_cast<const bool*>(row_data), vals.expected_bool_val); break; diff --git a/src/kudu/integration-tests/varchar-itest.cc b/src/kudu/integration-tests/varchar-itest.cc new file mode 100644 index 0000000..ad92f8a --- /dev/null +++ b/src/kudu/integration-tests/varchar-itest.cc @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <stdint.h> + +#include <memory> +#include <string> + +#include <gtest/gtest.h> + +#include "kudu/client/client.h" +#include "kudu/client/scan_batch.h" +#include "kudu/client/schema.h" +#include "kudu/client/shared_ptr.h" +#include "kudu/client/write_op.h" +#include "kudu/common/partial_row.h" +#include "kudu/integration-tests/external_mini_cluster-itest-base.h" +#include "kudu/util/slice.h" +#include "kudu/util/status.h" +#include "kudu/util/test_macros.h" + +using std::unique_ptr; + +namespace kudu { +namespace client { + +using sp::shared_ptr; + +class VarcharItest : public ExternalMiniClusterITestBase {}; + +TEST_F(VarcharItest, TestCharVarcharTruncation) { + const int kNumServers = 3; + const int kNumTablets = 3; + const char* const kTableName = "varchar-table"; + NO_FATALS(StartCluster({}, {}, kNumServers)); + + KuduSchemaBuilder builder; + builder.AddColumn("key")->Type(KuduColumnSchema::INT64) + ->NotNull()->PrimaryKey(); + builder.AddColumn("value")->Type(KuduColumnSchema::VARCHAR) + ->Length(10); + KuduSchema schema; + ASSERT_OK(builder.Build(&schema)); + + // Create table + unique_ptr<client::KuduTableCreator> table_creator(client_->NewTableCreator()); + ASSERT_OK(table_creator->table_name(kTableName) + .schema(&schema) + .num_replicas(kNumServers) + .add_hash_partitions({ "key" }, kNumTablets) + .Create()); + shared_ptr<KuduTable> table; + ASSERT_OK(client_->OpenTable(kTableName, &table)); + + // Insert row with long varchar values to test the truncation happens properly + shared_ptr<KuduSession> session = client_->NewSession(); + ASSERT_OK(session->SetFlushMode(KuduSession::AUTO_FLUSH_BACKGROUND)); + KuduInsert* insert = table->NewInsert(); + KuduPartialRow* write = insert->mutable_row(); + + ASSERT_OK(write->SetInt64("key", 1)); + ASSERT_OK(write->SetVarchar("value", "foobar baz")); + ASSERT_OK(session->Apply(insert)); + ASSERT_OK(session->Flush()); + + // Read rows + KuduScanner scanner(table.get()); + ASSERT_OK(scanner.SetFaultTolerant()); + ASSERT_OK(scanner.Open()); + while (scanner.HasMoreRows()) { + KuduScanBatch batch; + ASSERT_OK(scanner.NextBatch(&batch)); + for (const KuduScanBatch::RowPtr& read : batch) { + int64_t key; + ASSERT_OK(read.GetInt64("key", &key)); + ASSERT_EQ(1, key); + Slice value; + ASSERT_OK(read.GetVarchar("value", &value)); + ASSERT_EQ("foobar ", value); + } + } +} + +TEST_F(VarcharItest, TestInvalidLength) { + KuduSchemaBuilder builder; + builder.AddColumn("key")->Type(KuduColumnSchema::INT64) + ->NotNull()->PrimaryKey(); + builder.AddColumn("value")->Type(KuduColumnSchema::VARCHAR) + ->Length(0); + KuduSchema schema; + Status s = builder.Build(&schema); + ASSERT_FALSE(s.ok()); +} + +} // namespace client +} // namespace kudu
