This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1ef22d7f7c [Feature](variant) add variant type (#24170)
1ef22d7f7c is described below
commit 1ef22d7f7c612091bdebaf33f86f48f80f6bf2a8
Author: lihangyu <[email protected]>
AuthorDate: Thu Sep 14 14:21:53 2023 +0800
[Feature](variant) add variant type (#24170)
Add variant type for metadata Add persistent information for variant,
including the path of variant sub-columns, persisting them to the segment
footer and tablet schema of the rowset.
---
be/src/olap/tablet_schema.cpp | 62 ++++++++++++++-
be/src/olap/tablet_schema.h | 67 +++++++++++++---
be/src/olap/types.cpp | 6 +-
be/src/olap/types.h | 13 ++++
be/src/runtime/descriptors.cpp | 1 +
be/src/runtime/descriptors.h | 2 +
be/src/runtime/primitive_type.cpp | 5 ++
be/src/runtime/types.cpp | 7 --
be/src/runtime/types.h | 3 +-
be/src/vec/data_types/data_type_factory.cpp | 61 ++++++++++++---
be/src/vec/data_types/data_type_factory.hpp | 5 ++
be/src/vec/json/path_in_data.cpp | 67 ++++++++++++++++
be/src/vec/json/path_in_data.h | 9 +++
.../org/apache/doris/catalog/PrimitiveType.java | 45 ++++++++++-
.../java/org/apache/doris/catalog/ScalarType.java | 16 +++-
.../main/java/org/apache/doris/catalog/Type.java | 77 ++++++++++++++++---
.../apache/doris/analysis/SchemaChangeExpr.java | 89 ----------------------
.../org/apache/doris/planner/FileLoadScanNode.java | 8 +-
gensrc/proto/olap_file.proto | 2 +
gensrc/proto/segment_v2.proto | 29 +++++++
gensrc/thrift/Descriptors.thrift | 2 +
.../suites/nereids_arith_p0/bitmap.groovy | 2 +-
.../forbid_unexpected_type.groovy | 4 +-
.../join/test_bitmap_filter_nereids.groovy | 2 +-
24 files changed, 436 insertions(+), 148 deletions(-)
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index f643531d68..6f25b20e65 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -100,6 +100,8 @@ FieldType TabletColumn::get_field_type_by_string(const
std::string& type_str) {
type = FieldType::OLAP_FIELD_TYPE_STRING;
} else if (0 == upper_type_str.compare("JSONB")) {
type = FieldType::OLAP_FIELD_TYPE_JSONB;
+ } else if (0 == upper_type_str.compare("VARIANT")) {
+ type = FieldType::OLAP_FIELD_TYPE_VARIANT;
} else if (0 == upper_type_str.compare("BOOLEAN")) {
type = FieldType::OLAP_FIELD_TYPE_BOOL;
} else if (0 == upper_type_str.compare(0, 3, "HLL")) {
@@ -230,6 +232,9 @@ std::string
TabletColumn::get_string_by_field_type(FieldType type) {
case FieldType::OLAP_FIELD_TYPE_JSONB:
return "JSONB";
+ case FieldType::OLAP_FIELD_TYPE_VARIANT:
+ return "VARIANT";
+
case FieldType::OLAP_FIELD_TYPE_STRING:
return "STRING";
@@ -328,6 +333,7 @@ uint32_t
TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3
case TPrimitiveType::AGG_STATE:
return string_length + sizeof(OLAP_VARCHAR_MAX_LENGTH);
case TPrimitiveType::STRING:
+ case TPrimitiveType::VARIANT:
return string_length + sizeof(OLAP_STRING_MAX_LENGTH);
case TPrimitiveType::JSONB:
return string_length + sizeof(OLAP_JSONB_MAX_LENGTH);
@@ -393,6 +399,7 @@ void TabletColumn::init_from_thrift(const TColumn& tcolumn)
{
void TabletColumn::init_from_pb(const ColumnPB& column) {
_unique_id = column.unique_id();
_col_name = column.name();
+ _col_name_lower_case = to_lower(_col_name);
_type = TabletColumn::get_field_type_by_string(column.type());
_is_key = column.is_key();
_is_nullable = column.is_nullable();
@@ -444,6 +451,10 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
child_column.init_from_pb(column.children_columns(i));
add_sub_column(child_column);
}
+ if (column.has_column_path_info()) {
+ _column_path.from_protobuf(column.column_path_info());
+ _parent_col_unique_id =
column.column_path_info().parrent_column_unique_id();
+ }
}
void TabletColumn::to_schema_pb(ColumnPB* column) const {
@@ -484,11 +495,17 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
ColumnPB* child = column->add_children_columns();
_sub_columns[i].to_schema_pb(child);
}
+
+ // set parts info
+ if (!_column_path.empty()) {
+ // CHECK_GT(_parent_col_unique_id, 0);
+ _column_path.to_protobuf(column->mutable_column_path_info(),
_parent_col_unique_id);
+ }
}
void TabletColumn::add_sub_column(TabletColumn& sub_column) {
_sub_columns.push_back(sub_column);
- sub_column._parent = this;
+ sub_column._parent_col_unique_id = this->_unique_id;
_sub_column_count += 1;
}
@@ -523,6 +540,14 @@ vectorized::AggregateFunctionPtr
TabletColumn::get_aggregate_function(std::strin
return nullptr;
}
+void TabletColumn::set_path_info(const vectorized::PathInData& path) {
+ _column_path = path;
+}
+
+vectorized::DataTypePtr TabletColumn::get_vec_type() const {
+ return vectorized::DataTypeFactory::instance().create_data_type(*this);
+}
+
void TabletIndex::init_from_thrift(const TOlapTableIndex& index,
const TabletSchema& tablet_schema) {
_index_id = index.index_id;
@@ -614,13 +639,21 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index)
const {
}
}
-void TabletSchema::append_column(TabletColumn column, bool is_dropped_column) {
+void TabletSchema::append_column(TabletColumn column, ColumnType col_type) {
if (column.is_key()) {
_num_key_columns++;
}
if (column.is_nullable()) {
_num_null_columns++;
}
+ if (column.is_variant_type()) {
+ ++_num_variant_columns;
+ if (column.path_info().empty()) {
+ const std::string& col_name = column.name_lower_case();
+ vectorized::PathInData path(col_name);
+ column.set_path_info(path);
+ }
+ }
if (UNLIKELY(column.name() == DELETE_SIGN)) {
_delete_sign_idx = _num_columns;
} else if (UNLIKELY(column.name() == SEQUENCE_COL)) {
@@ -630,8 +663,10 @@ void TabletSchema::append_column(TabletColumn column, bool
is_dropped_column) {
}
// The dropped column may have same name with exsiting column, so that
// not add to name to index map, only for uid to index map
- if (!is_dropped_column) {
+ if (col_type == ColumnType::NORMAL) {
_field_name_to_index[column.name()] = _num_columns;
+ } else if (col_type == ColumnType::VARIANT) {
+ _field_path_to_index[column.path_info()] = _num_columns;
}
_field_id_to_index[column.unique_id()] = _num_columns;
_cols.push_back(std::move(column));
@@ -654,9 +689,11 @@ void TabletSchema::remove_index(int64_t index_id) {
}
void TabletSchema::clear_columns() {
+ _field_path_to_index.clear();
_field_name_to_index.clear();
_field_id_to_index.clear();
_num_columns = 0;
+ _num_variant_columns = 0;
_num_null_columns = 0;
_num_key_columns = 0;
_cols.clear();
@@ -666,6 +703,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema) {
SCOPED_MEM_COUNT(&_mem_size);
_keys_type = schema.keys_type();
_num_columns = 0;
+ _num_variant_columns = 0;
_num_key_columns = 0;
_num_null_columns = 0;
_cols.clear();
@@ -684,6 +722,9 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema) {
if (column.is_nullable()) {
_num_null_columns++;
}
+ if (column.is_variant_type()) {
+ ++_num_variant_columns;
+ }
_field_name_to_index[column.name()] = _num_columns;
_field_id_to_index[column.unique_id()] = _num_columns;
_cols.emplace_back(std::move(column));
@@ -770,6 +811,7 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
// copy from table_schema_param
_schema_version = version;
_num_columns = 0;
+ _num_variant_columns = 0;
_num_key_columns = 0;
_num_null_columns = 0;
bool has_bf_columns = false;
@@ -788,6 +830,9 @@ void TabletSchema::build_current_tablet_schema(int64_t
index_id, int32_t version
if (column->is_bf_column()) {
has_bf_columns = true;
}
+ if (column->is_variant_type()) {
+ ++_num_variant_columns;
+ }
if (UNLIKELY(column->name() == DELETE_SIGN)) {
_delete_sign_idx = _num_columns;
} else if (UNLIKELY(column->name() == SEQUENCE_COL)) {
@@ -827,7 +872,7 @@ void TabletSchema::merge_dropped_columns(TabletSchemaSPtr
src_schema) {
// that deep copy it.
src_col.to_schema_pb(&src_col_pb);
TabletColumn new_col(src_col_pb);
- append_column(new_col, /* is_dropped_column */ true);
+ append_column(new_col, TabletSchema::ColumnType::DROPPED);
}
}
}
@@ -892,6 +937,11 @@ int32_t TabletSchema::field_index(const std::string&
field_name) const {
return (found == _field_name_to_index.end()) ? -1 : found->second;
}
+int32_t TabletSchema::field_index(const vectorized::PathInData& path) const {
+ const auto& found = _field_path_to_index.find(path);
+ return (found == _field_path_to_index.end()) ? -1 : found->second;
+}
+
int32_t TabletSchema::field_index(int32_t col_unique_id) const {
const auto& found = _field_id_to_index.find(col_unique_id);
return (found == _field_id_to_index.end()) ? -1 : found->second;
@@ -901,6 +951,10 @@ const std::vector<TabletColumn>& TabletSchema::columns()
const {
return _cols;
}
+std::vector<TabletColumn>& TabletSchema::mutable_columns() {
+ return _cols;
+}
+
const TabletColumn& TabletSchema::column(size_t ordinal) const {
DCHECK(ordinal < _num_columns) << "ordinal:" << ordinal << ",
_num_columns:" << _num_columns;
return _cols[ordinal];
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index c98e5542f7..72d0636f6e 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -35,11 +35,15 @@
#include "common/status.h"
#include "gutil/stringprintf.h"
#include "olap/olap_common.h"
+#include "util/string_util.h"
#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/json/path_in_data.h"
namespace doris {
namespace vectorized {
class Block;
+class PathInData;
+class IDataType;
} // namespace vectorized
struct OlapTableIndexSchema;
@@ -61,8 +65,12 @@ public:
int32_t unique_id() const { return _unique_id; }
void set_unique_id(int32_t id) { _unique_id = id; }
- std::string name() const { return _col_name; }
- void set_name(std::string col_name) { _col_name = col_name; }
+ const std::string& name() const { return _col_name; }
+ const std::string& name_lower_case() const { return _col_name_lower_case; }
+ void set_name(std::string col_name) {
+ _col_name = col_name;
+ _col_name_lower_case = to_lower(_col_name);
+ }
FieldType type() const { return _type; }
void set_type(FieldType type) { _type = type; }
bool is_key() const { return _is_key; }
@@ -83,11 +91,17 @@ public:
bool has_default_value() const { return _has_default_value; }
std::string default_value() const { return _default_value; }
size_t length() const { return _length; }
+ void set_length(size_t length) { _length = length; }
+ void set_default_value(const std::string& default_value) {
+ _default_value = default_value;
+ _has_default_value = true;
+ }
size_t index_length() const { return _index_length; }
void set_index_length(size_t index_length) { _index_length = index_length;
}
void set_is_key(bool is_key) { _is_key = is_key; }
void set_is_nullable(bool is_nullable) { _is_nullable = is_nullable; }
void set_has_default_value(bool has) { _has_default_value = has; }
+ void set_path_info(const vectorized::PathInData& path);
FieldAggregationMethod aggregation() const { return _aggregation; }
vectorized::AggregateFunctionPtr get_aggregate_function_union(
vectorized::DataTypePtr type) const;
@@ -105,6 +119,7 @@ public:
uint32_t get_subtype_count() const { return _sub_column_count; }
const TabletColumn& get_sub_column(uint32_t i) const { return
_sub_columns[i]; }
+ const std::vector<TabletColumn>& get_sub_columns() const { return
_sub_columns; }
friend bool operator==(const TabletColumn& a, const TabletColumn& b);
friend bool operator!=(const TabletColumn& a, const TabletColumn& b);
@@ -117,10 +132,17 @@ public:
bool is_row_store_column() const;
std::string get_aggregation_name() const { return _aggregation_name; }
bool get_result_is_nullable() const { return _result_is_nullable; }
+ const vectorized::PathInData& path_info() const { return _column_path; }
+ // If it is an extracted column from variant column
+ bool is_extracted_column() const { return !_column_path.empty() &&
_parent_col_unique_id > 0; };
+ int32_t parent_unique_id() const { return _parent_col_unique_id; }
+ void set_parent_unique_id(int32_t col_unique_id) { _parent_col_unique_id =
col_unique_id; }
+ std::shared_ptr<const vectorized::IDataType> get_vec_type() const;
private:
- int32_t _unique_id;
+ int32_t _unique_id = -1;
std::string _col_name;
+ std::string _col_name_lower_case;
FieldType _type;
bool _is_key = false;
FieldAggregationMethod _aggregation;
@@ -141,12 +163,12 @@ private:
bool _has_bitmap_index = false;
bool _visible = true;
-
- TabletColumn* _parent = nullptr;
+ int32_t _parent_col_unique_id = -1;
std::vector<TabletColumn> _sub_columns;
uint32_t _sub_column_count = 0;
bool _result_is_nullable = false;
+ vectorized::PathInData _column_path;
};
bool operator==(const TabletColumn& a, const TabletColumn& b);
@@ -199,13 +221,14 @@ private:
class TabletSchema {
public:
+ enum ColumnType { NORMAL = 0, DROPPED = 1, VARIANT = 2 };
// TODO(yingchun): better to make constructor as private to avoid
// manually init members incorrectly, and define a new function like
// void create_from_pb(const TabletSchemaPB& schema, TabletSchema*
tablet_schema).
TabletSchema() = default;
void init_from_pb(const TabletSchemaPB& schema);
void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const;
- void append_column(TabletColumn column, bool is_dropped_column = false);
+ void append_column(TabletColumn column, ColumnType col_type =
ColumnType::NORMAL);
void append_index(TabletIndex index);
void remove_index(int64_t index_id);
// Must make sure the row column is always the last column
@@ -213,20 +236,22 @@ public:
void copy_from(const TabletSchema& tablet_schema);
std::string to_key() const;
int64_t mem_size() const { return _mem_size; }
-
size_t row_size() const;
int32_t field_index(const std::string& field_name) const;
+ int32_t field_index(const vectorized::PathInData& path) const;
int32_t field_index(int32_t col_unique_id) const;
const TabletColumn& column(size_t ordinal) const;
const TabletColumn& column(const std::string& field_name) const;
Status have_column(const std::string& field_name) const;
const TabletColumn& column_by_uid(int32_t col_unique_id) const;
const std::vector<TabletColumn>& columns() const;
+ std::vector<TabletColumn>& mutable_columns();
size_t num_columns() const { return _num_columns; }
size_t num_key_columns() const { return _num_key_columns; }
size_t num_null_columns() const { return _num_null_columns; }
size_t num_short_key_columns() const { return _num_short_key_columns; }
size_t num_rows_per_row_block() const { return _num_rows_per_row_block; }
+ size_t num_variant_columns() const { return _num_variant_columns; };
KeysType keys_type() const { return _keys_type; }
SortType sort_type() const { return _sort_type; }
size_t sort_col_num() const { return _sort_col_num; }
@@ -305,6 +330,27 @@ public:
str += "]";
return str;
}
+
+ // Dump [(name, type, is_nullable), ...]
+ string dump_structure() const {
+ string str = "[";
+ for (auto p : _field_name_to_index) {
+ if (str.size() > 1) {
+ str += ", ";
+ }
+ str += "(";
+ str += p.first;
+ str += ", ";
+ str +=
TabletColumn::get_string_by_field_type(_cols[p.second].type());
+ str += ", ";
+ str += "is_nullable:";
+ str += (_cols[p.second].is_nullable() ? "true" : "false");
+ str += ")";
+ }
+ str += "]";
+ return str;
+ }
+
vectorized::Block create_missing_columns_block();
vectorized::Block create_update_columns_block();
void set_partial_update_info(bool is_partial_update,
@@ -317,8 +363,8 @@ public:
}
void set_is_strict_mode(bool is_strict_mode) { _is_strict_mode =
is_strict_mode; }
bool is_strict_mode() const { return _is_strict_mode; }
- std::vector<uint32_t> get_missing_cids() { return _missing_cids; }
- std::vector<uint32_t> get_update_cids() { return _update_cids; }
+ std::vector<uint32_t> get_missing_cids() const { return _missing_cids; }
+ std::vector<uint32_t> get_update_cids() const { return _update_cids; }
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
@@ -331,7 +377,10 @@ private:
std::vector<TabletIndex> _indexes;
std::unordered_map<std::string, int32_t> _field_name_to_index;
std::unordered_map<int32_t, int32_t> _field_id_to_index;
+ std::unordered_map<vectorized::PathInData, int32_t,
vectorized::PathInData::Hash>
+ _field_path_to_index;
size_t _num_columns = 0;
+ size_t _num_variant_columns = 0;
size_t _num_key_columns = 0;
size_t _num_null_columns = 0;
size_t _num_short_key_columns = 0;
diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp
index 102f00c7e1..2c92bd3f2c 100644
--- a/be/src/olap/types.cpp
+++ b/be/src/olap/types.cpp
@@ -96,8 +96,9 @@ const TypeInfo* get_scalar_type_info(FieldType field_type) {
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_DECIMAL64>(),
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>(),
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_JSONB>(),
- nullptr,
- get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_AGG_STATE>()};
+ get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_VARIANT>(),
+ get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_AGG_STATE>(),
+ nullptr};
return field_type_array[int(field_type)];
}
@@ -168,6 +169,7 @@ const TypeInfo* get_array_type_info(FieldType leaf_type,
int32_t iterations) {
INIT_ARRAY_TYPE_INFO_LIST(FieldType::OLAP_FIELD_TYPE_DECIMAL64),
INIT_ARRAY_TYPE_INFO_LIST(FieldType::OLAP_FIELD_TYPE_DECIMAL128I),
INIT_ARRAY_TYPE_INFO_LIST(FieldType::OLAP_FIELD_TYPE_JSONB),
+ INIT_ARRAY_TYPE_INFO_LIST(FieldType::OLAP_FIELD_TYPE_VARIANT),
{nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr},
INIT_ARRAY_TYPE_INFO_LIST(FieldType::OLAP_FIELD_TYPE_AGG_STATE)};
return array_type_Info_arr[int(leaf_type)][iterations];
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 40d812a160..bb54959aee 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -731,6 +731,10 @@ struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_JSONB> {
using CppType = Slice;
};
template <>
+struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_VARIANT> {
+ using CppType = Slice;
+};
+template <>
struct CppTypeTraits<FieldType::OLAP_FIELD_TYPE_HLL> {
using CppType = Slice;
};
@@ -1385,6 +1389,15 @@ struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_JSONB>
}
};
+template <>
+struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_VARIANT>
+ : public FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_JSONB> {
+ static int cmp(const void* left, const void* right) {
+ LOG(WARNING) << "can not compare VARIANT values";
+ return -1; // always update ?
+ }
+};
+
template <>
struct FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_HLL>
: public FieldTypeTraits<FieldType::OLAP_FIELD_TYPE_VARCHAR> {};
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index 73bde2ba9d..5b58ff2af5 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -64,6 +64,7 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc)
_is_materialized(tdesc.isMaterialized),
_is_key(tdesc.is_key),
_need_materialize(tdesc.need_materialize),
+ _column_paths(tdesc.column_paths),
_is_auto_increment(tdesc.__isset.is_auto_increment ?
tdesc.is_auto_increment : false) {}
SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc)
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index af59b287c4..18d9bb62f3 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -108,6 +108,7 @@ public:
bool is_key() const { return _is_key; }
bool need_materialize() const { return _need_materialize; }
+ const std::vector<std::string>& column_paths() const { return
_column_paths; };
bool is_auto_increment() const { return _is_auto_increment; }
@@ -143,6 +144,7 @@ private:
const bool _is_key;
const bool _need_materialize;
+ const std::vector<std::string> _column_paths;
const bool _is_auto_increment;
diff --git a/be/src/runtime/primitive_type.cpp
b/be/src/runtime/primitive_type.cpp
index f45e13b745..82a189107d 100644
--- a/be/src/runtime/primitive_type.cpp
+++ b/be/src/runtime/primitive_type.cpp
@@ -152,6 +152,8 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
case TPrimitiveType::AGG_STATE:
return TYPE_AGG_STATE;
+ case TPrimitiveType::VARIANT:
+ return TYPE_VARIANT;
default:
CHECK(false) << ", meet unknown type " << ttype;
return INVALID_TYPE;
@@ -217,6 +219,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
case TYPE_JSONB:
return TPrimitiveType::JSONB;
+ case TYPE_VARIANT:
+ return TPrimitiveType::VARIANT;
+
case TYPE_BINARY:
return TPrimitiveType::BINARY;
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index 1b8870286e..4cb3d3ef5b 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -92,13 +92,6 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>&
types, int* idx)
}
break;
}
- case TTypeNodeType::VARIANT: {
- DCHECK(!node.__isset.scalar_type);
- // variant column must be the last column
- DCHECK_EQ(*idx, types.size() - 1);
- type = TYPE_VARIANT;
- break;
- }
case TTypeNodeType::MAP: {
//TODO(xy): handle contains_null[0] for key and [1] for value
DCHECK(!node.__isset.scalar_type);
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index 19766933e8..bb030b66d6 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -227,8 +227,7 @@ struct TypeDescriptor {
}
bool is_complex_type() const {
- return type == TYPE_STRUCT || type == TYPE_ARRAY || type == TYPE_MAP ||
- type == TYPE_VARIANT;
+ return type == TYPE_STRUCT || type == TYPE_ARRAY || type == TYPE_MAP;
}
bool is_collection_type() const { return type == TYPE_ARRAY || type ==
TYPE_MAP; }
diff --git a/be/src/vec/data_types/data_type_factory.cpp
b/be/src/vec/data_types/data_type_factory.cpp
index 971b37eb8f..4ab836141b 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -37,6 +37,7 @@
#include "common/exception.h"
#include "common/status.h"
#include "data_type_time.h"
+#include "gen_cpp/segment_v2.pb.h"
#include "olap/field.h"
#include "olap/olap_common.h"
#include "runtime/define_primitive_type.h"
@@ -103,7 +104,7 @@ DataTypePtr DataTypeFactory::create_data_type(const
TabletColumn& col_desc, bool
}
if ((is_nullable || col_desc.is_nullable()) && nested) {
- return std::make_shared<DataTypeNullable>(nested);
+ return make_nullable(nested);
}
return nested;
}
@@ -152,6 +153,8 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeDescriptor& col_desc, bo
case TYPE_DOUBLE:
nested = std::make_shared<vectorized::DataTypeFloat64>();
break;
+ case TYPE_VARIANT:
+ return std::make_shared<vectorized::DataTypeObject>("", true);
case TYPE_STRING:
case TYPE_CHAR:
case TYPE_VARCHAR:
@@ -220,9 +223,6 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeDescriptor& col_desc, bo
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
}
- case TYPE_VARIANT:
- // ColumnObject always none nullable
- return std::make_shared<vectorized::DataTypeObject>("json", true);
case INVALID_TYPE:
default:
throw Exception(ErrorCode::INTERNAL_ERROR, "invalid PrimitiveType:
{}", (int)col_desc.type);
@@ -230,7 +230,7 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeDescriptor& col_desc, bo
}
if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
+ return make_nullable(nested);
}
return nested;
}
@@ -286,6 +286,8 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeIndex& type_index, bool
case TypeIndex::String:
nested = std::make_shared<vectorized::DataTypeString>();
break;
+ case TypeIndex::VARIANT:
+ return std::make_shared<vectorized::DataTypeObject>("", true);
case TypeIndex::Decimal32:
nested =
std::make_shared<DataTypeDecimal<Decimal32>>(BeConsts::MAX_DECIMAL32_PRECISION,
0);
break;
@@ -322,7 +324,7 @@ DataTypePtr DataTypeFactory::create_data_type(const
TypeIndex& type_index, bool
}
if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
+ return make_nullable(nested);
}
return nested;
}
@@ -372,6 +374,8 @@ DataTypePtr
DataTypeFactory::_create_primitive_data_type(const FieldType& type,
case FieldType::OLAP_FIELD_TYPE_STRING:
result = std::make_shared<vectorized::DataTypeString>();
break;
+ case FieldType::OLAP_FIELD_TYPE_VARIANT:
+ return std::make_shared<vectorized::DataTypeObject>("", true);
case FieldType::OLAP_FIELD_TYPE_JSONB:
result = std::make_shared<vectorized::DataTypeJsonb>();
break;
@@ -442,6 +446,8 @@ DataTypePtr DataTypeFactory::create_data_type(const
PColumnMeta& pcolumn) {
case PGenericType::STRING:
nested = std::make_shared<DataTypeString>();
break;
+ case PGenericType::VARIANT:
+ return std::make_shared<DataTypeObject>("", true);
case PGenericType::JSONB:
nested = std::make_shared<DataTypeJsonb>();
break;
@@ -506,10 +512,6 @@ DataTypePtr DataTypeFactory::create_data_type(const
PColumnMeta& pcolumn) {
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
}
- case PGenericType::VARIANT: {
- nested = std::make_shared<DataTypeObject>("object", true);
- break;
- }
case PGenericType::QUANTILE_STATE: {
nested = std::make_shared<DataTypeQuantileState>();
break;
@@ -535,7 +537,42 @@ DataTypePtr DataTypeFactory::create_data_type(const
PColumnMeta& pcolumn) {
}
if (nested && pcolumn.is_nullable() > 0) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
+ return make_nullable(nested);
+ }
+ return nested;
+}
+
+DataTypePtr DataTypeFactory::create_data_type(const segment_v2::ColumnMetaPB&
pcolumn) {
+ DataTypePtr nested = nullptr;
+ if (pcolumn.type() == static_cast<int>(FieldType::OLAP_FIELD_TYPE_ARRAY)) {
+ // Item subcolumn and length subcolumn
+ DCHECK_GE(pcolumn.children_columns().size(), 2) <<
pcolumn.DebugString();
+ nested =
std::make_shared<DataTypeArray>(create_data_type(pcolumn.children_columns(0)));
+ } else if (pcolumn.type() ==
static_cast<int>(FieldType::OLAP_FIELD_TYPE_MAP)) {
+ DCHECK_GE(pcolumn.children_columns().size(), 2) <<
pcolumn.DebugString();
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(pcolumn.children_columns(0)),
+ create_data_type(pcolumn.children_columns(1)));
+ } else if (pcolumn.type() ==
static_cast<int>(FieldType::OLAP_FIELD_TYPE_STRUCT)) {
+ DCHECK_GE(pcolumn.children_columns().size(), 1);
+ size_t col_size = pcolumn.children_columns().size();
+ DataTypes dataTypes;
+ Strings names;
+ dataTypes.reserve(col_size);
+ names.reserve(col_size);
+ for (size_t i = 0; i < col_size; i++) {
+ dataTypes.push_back(create_data_type(pcolumn.children_columns(i)));
+ names.push_back("");
+ }
+ nested = std::make_shared<DataTypeStruct>(dataTypes, names);
+ } else {
+ // TODO add precision and frac
+ nested =
_create_primitive_data_type(static_cast<FieldType>(pcolumn.type()),
+ pcolumn.precision(),
pcolumn.frac());
+ }
+
+ if (pcolumn.is_nullable() && nested) {
+ return make_nullable(nested);
}
return nested;
}
@@ -623,7 +660,7 @@ DataTypePtr DataTypeFactory::create_data_type(const
arrow::DataType* type, bool
}
if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
+ return make_nullable(nested);
}
return nested;
}
diff --git a/be/src/vec/data_types/data_type_factory.hpp
b/be/src/vec/data_types/data_type_factory.hpp
index 6bef55938f..0439be0277 100644
--- a/be/src/vec/data_types/data_type_factory.hpp
+++ b/be/src/vec/data_types/data_type_factory.hpp
@@ -38,6 +38,10 @@ class Field;
class PColumnMeta;
enum class FieldType;
+namespace segment_v2 {
+class ColumnMetaPB;
+}
+
namespace vectorized {
enum class TypeIndex;
} // namespace vectorized
@@ -62,6 +66,7 @@ public:
DataTypePtr create_data_type(const TypeDescriptor& col_desc, bool
is_nullable = true);
DataTypePtr create_data_type(const PColumnMeta& pcolumn);
+ DataTypePtr create_data_type(const segment_v2::ColumnMetaPB& pcolumn);
DataTypePtr create_data_type(const arrow::DataType* type, bool
is_nullable);
diff --git a/be/src/vec/json/path_in_data.cpp b/be/src/vec/json/path_in_data.cpp
index a05542e189..3eab57f781 100644
--- a/be/src/vec/json/path_in_data.cpp
+++ b/be/src/vec/json/path_in_data.cpp
@@ -45,6 +45,15 @@ PathInData::PathInData(const Parts& parts_) {
PathInData::PathInData(const PathInData& other) : path(other.path) {
build_parts(other.get_parts());
}
+
+PathInData::PathInData(const std::vector<std::string>& paths) {
+ PathInDataBuilder path_builder;
+ for (size_t i = 0; i < paths.size(); ++i) {
+ path_builder.append(paths[i], false);
+ }
+ build_parts(path_builder.get_parts());
+}
+
PathInData& PathInData::operator=(const PathInData& other) {
if (this != &other) {
path = other.path;
@@ -92,10 +101,66 @@ void PathInData::build_parts(const Parts& other_parts) {
begin += part.key.length() + 1;
}
}
+
+void PathInData::from_protobuf(const segment_v2::ColumnPathInfo& pb) {
+ parts.clear();
+ path = pb.path();
+ has_nested = pb.has_has_nested();
+ parts.reserve(pb.path_part_infos().size());
+ for (const segment_v2::ColumnPathPartInfo& part_info :
pb.path_part_infos()) {
+ Part part;
+ part.is_nested = part_info.is_nested();
+ part.anonymous_array_level = part_info.anonymous_array_level();
+ part.key = part_info.key();
+ parts.push_back(part);
+ }
+}
+
+std::string PathInData::to_jsonpath() const {
+ std::string jsonpath = "$.";
+ if (parts.empty()) {
+ return jsonpath;
+ }
+ auto it = parts.begin();
+ jsonpath += it->key;
+ ++it;
+ for (; it != parts.end(); ++it) {
+ jsonpath += ".";
+ jsonpath += it->key;
+ }
+ return jsonpath;
+}
+
+void PathInData::to_protobuf(segment_v2::ColumnPathInfo* pb, int32_t
parent_col_unique_id) const {
+ pb->set_path(path);
+ pb->set_has_nested(has_nested);
+ pb->set_parrent_column_unique_id(parent_col_unique_id);
+
+ // set parts info
+ for (const Part& part : parts) {
+ segment_v2::ColumnPathPartInfo& part_info = *pb->add_path_part_infos();
+ part_info.set_key(std::string(part.key.data(), part.key.size()));
+ part_info.set_is_nested(part.is_nested);
+ part_info.set_anonymous_array_level(part.anonymous_array_level);
+ }
+}
+
size_t PathInData::Hash::operator()(const PathInData& value) const {
auto hash = get_parts_hash(value.parts);
return hash.low ^ hash.high;
}
+
+PathInData PathInData::pop_front() const {
+ PathInData new_path;
+ Parts new_parts;
+ if (!parts.empty()) {
+ std::copy(parts.begin() + 1, parts.end(),
std::back_inserter(new_parts));
+ }
+ new_path.build_path(new_parts);
+ new_path.build_parts(new_parts);
+ return new_path;
+}
+
PathInDataBuilder& PathInDataBuilder::append(std::string_view key, bool
is_array) {
if (parts.empty()) {
current_anonymous_array_level += is_array;
@@ -125,9 +190,11 @@ PathInDataBuilder& PathInDataBuilder::append(const
PathInData::Parts& path, bool
}
return *this;
}
+
void PathInDataBuilder::pop_back() {
parts.pop_back();
}
+
void PathInDataBuilder::pop_back(size_t n) {
assert(n <= parts.size());
parts.resize(parts.size() - n);
diff --git a/be/src/vec/json/path_in_data.h b/be/src/vec/json/path_in_data.h
index 47da9c85cb..87278c1c92 100644
--- a/be/src/vec/json/path_in_data.h
+++ b/be/src/vec/json/path_in_data.h
@@ -27,6 +27,7 @@
#include <string_view>
#include <vector>
+#include "gen_cpp/segment_v2.pb.h"
#include "vec/common/uint128.h"
#include "vec/core/field.h"
#include "vec/core/types.h"
@@ -59,6 +60,7 @@ public:
PathInData() = default;
explicit PathInData(std::string_view path_);
explicit PathInData(const Parts& parts_);
+ explicit PathInData(const std::vector<std::string>& paths);
PathInData(const PathInData& other);
PathInData& operator=(const PathInData& other);
static UInt128 get_parts_hash(const Parts& parts_);
@@ -71,6 +73,11 @@ public:
struct Hash {
size_t operator()(const PathInData& value) const;
};
+ std::string to_jsonpath() const;
+
+ PathInData pop_front() const;
+ void to_protobuf(segment_v2::ColumnPathInfo* pb, int32_t
parent_col_unique_id) const;
+ void from_protobuf(const segment_v2::ColumnPathInfo& pb);
private:
/// Creates full path from parts.
@@ -90,8 +97,10 @@ public:
const PathInData::Parts& get_parts() const { return parts; }
PathInDataBuilder& append(std::string_view key, bool is_array);
PathInDataBuilder& append(const PathInData::Parts& path, bool is_array);
+ PathInDataBuilder& append(const std::vector<std::string>& parts);
void pop_back();
void pop_back(size_t n);
+ PathInData build() { return PathInData(parts); }
private:
PathInData::Parts parts;
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index 50fddda0f3..5f34bb65c9 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -128,6 +128,7 @@ public enum PrimitiveType {
builder.put(NULL_TYPE, VARCHAR);
builder.put(NULL_TYPE, STRING);
builder.put(NULL_TYPE, JSONB);
+ builder.put(NULL_TYPE, VARIANT);
builder.put(NULL_TYPE, BITMAP); //TODO(weixiang):why null type can
cast to bitmap?
builder.put(NULL_TYPE, TIME);
builder.put(NULL_TYPE, TIMEV2);
@@ -414,6 +415,7 @@ public enum PrimitiveType {
builder.put(VARCHAR, DECIMAL128);
builder.put(VARCHAR, VARCHAR);
builder.put(VARCHAR, JSONB);
+ builder.put(VARCHAR, VARIANT);
builder.put(VARCHAR, STRING);
builder.put(VARCHAR, TIME);
builder.put(VARCHAR, TIMEV2);
@@ -437,6 +439,7 @@ public enum PrimitiveType {
builder.put(STRING, DECIMAL128);
builder.put(STRING, VARCHAR);
builder.put(STRING, JSONB);
+ builder.put(STRING, VARIANT);
builder.put(STRING, STRING);
builder.put(STRING, TIME);
builder.put(STRING, TIMEV2);
@@ -503,8 +506,26 @@ public enum PrimitiveType {
builder.put(DECIMAL128, STRING);
// JSONB
+ builder.put(JSONB, BOOLEAN);
+ builder.put(JSONB, TINYINT);
+ builder.put(JSONB, SMALLINT);
+ builder.put(JSONB, INT);
+ builder.put(JSONB, BIGINT);
+ builder.put(JSONB, LARGEINT);
+ builder.put(JSONB, FLOAT);
+ builder.put(JSONB, DOUBLE);
+ builder.put(JSONB, DECIMALV2);
+ builder.put(JSONB, DECIMAL32);
+ builder.put(JSONB, DECIMAL64);
+ builder.put(JSONB, DECIMAL128);
builder.put(JSONB, VARCHAR);
builder.put(JSONB, STRING);
+ builder.put(JSONB, VARIANT);
+
+ // VARIANT
+ builder.put(VARIANT, VARCHAR);
+ builder.put(VARIANT, STRING);
+ builder.put(VARIANT, JSONB);
// HLL
builder.put(HLL, HLL);
@@ -568,6 +589,7 @@ public enum PrimitiveType {
supportedTypes.add(DOUBLE);
supportedTypes.add(VARCHAR);
supportedTypes.add(JSONB);
+ supportedTypes.add(VARIANT);
supportedTypes.add(STRING);
supportedTypes.add(HLL);
supportedTypes.add(CHAR);
@@ -585,7 +607,6 @@ public enum PrimitiveType {
supportedTypes.add(ARRAY);
supportedTypes.add(MAP);
supportedTypes.add(QUANTILE_STATE);
- supportedTypes.add(VARIANT);
supportedTypes.add(AGG_STATE);
}
@@ -641,6 +662,7 @@ public enum PrimitiveType {
compatibilityMatrix[NULL_TYPE.ordinal()][CHAR.ordinal()] = CHAR;
compatibilityMatrix[NULL_TYPE.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[NULL_TYPE.ordinal()][JSONB.ordinal()] = JSONB;
+ compatibilityMatrix[NULL_TYPE.ordinal()][VARIANT.ordinal()] = VARIANT;
compatibilityMatrix[NULL_TYPE.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[NULL_TYPE.ordinal()][DECIMALV2.ordinal()] =
DECIMALV2;
compatibilityMatrix[NULL_TYPE.ordinal()][DECIMAL32.ordinal()] =
DECIMAL32;
@@ -667,6 +689,7 @@ public enum PrimitiveType {
compatibilityMatrix[BOOLEAN.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[BOOLEAN.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][DECIMALV2.ordinal()] =
DECIMALV2;
compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL32.ordinal()] =
DECIMAL32;
@@ -689,6 +712,7 @@ public enum PrimitiveType {
compatibilityMatrix[TINYINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[TINYINT.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][DECIMALV2.ordinal()] =
DECIMALV2;
compatibilityMatrix[TINYINT.ordinal()][DECIMAL32.ordinal()] =
DECIMAL32;
@@ -710,6 +734,7 @@ public enum PrimitiveType {
compatibilityMatrix[SMALLINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][JSONB.ordinal()] =
INVALID_TYPE;
+ compatibilityMatrix[SMALLINT.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][DECIMALV2.ordinal()] =
DECIMALV2;
compatibilityMatrix[SMALLINT.ordinal()][DECIMAL32.ordinal()] =
DECIMAL32;
@@ -730,6 +755,7 @@ public enum PrimitiveType {
compatibilityMatrix[INT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[INT.ordinal()][VARIANT.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[INT.ordinal()][DECIMAL32.ordinal()] = DECIMAL32;
@@ -749,6 +775,7 @@ public enum PrimitiveType {
compatibilityMatrix[BIGINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[BIGINT.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[BIGINT.ordinal()][DECIMAL32.ordinal()] = DECIMAL32;
@@ -767,6 +794,7 @@ public enum PrimitiveType {
compatibilityMatrix[LARGEINT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][JSONB.ordinal()] =
INVALID_TYPE;
+ compatibilityMatrix[LARGEINT.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][STRING.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][DECIMALV2.ordinal()] =
DECIMALV2;
compatibilityMatrix[LARGEINT.ordinal()][DECIMAL32.ordinal()] =
DECIMAL32;
@@ -784,6 +812,7 @@ public enum PrimitiveType {
compatibilityMatrix[FLOAT.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[FLOAT.ordinal()][VARIANT.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[FLOAT.ordinal()][DECIMAL32.ordinal()] = DECIMAL32;
@@ -800,6 +829,7 @@ public enum PrimitiveType {
compatibilityMatrix[DOUBLE.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[DOUBLE.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][DECIMALV2.ordinal()] = DECIMALV2;
compatibilityMatrix[DOUBLE.ordinal()][DECIMAL32.ordinal()] = DECIMAL32;
@@ -816,6 +846,7 @@ public enum PrimitiveType {
compatibilityMatrix[DATE.ordinal()][VARCHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][JSONB.ordinal()] = INVALID_TYPE;
+ compatibilityMatrix[DATE.ordinal()][VARIANT.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][DECIMALV2.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][DECIMAL32.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][DECIMAL64.ordinal()] =
INVALID_TYPE;
@@ -842,6 +873,7 @@ public enum PrimitiveType {
compatibilityMatrix[DATETIME.ordinal()][CHAR.ordinal()] = INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][VARCHAR.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][JSONB.ordinal()] =
INVALID_TYPE;
+ compatibilityMatrix[DATETIME.ordinal()][VARIANT.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][DECIMALV2.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][DECIMAL32.ordinal()] =
INVALID_TYPE;
@@ -865,6 +897,7 @@ public enum PrimitiveType {
compatibilityMatrix[CHAR.ordinal()][CHAR.ordinal()] = CHAR;
compatibilityMatrix[CHAR.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[CHAR.ordinal()][JSONB.ordinal()] = CHAR;
+ compatibilityMatrix[CHAR.ordinal()][VARIANT.ordinal()] = CHAR;
compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[CHAR.ordinal()][DECIMALV2.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][DECIMAL32.ordinal()] =
INVALID_TYPE;
@@ -876,6 +909,7 @@ public enum PrimitiveType {
compatibilityMatrix[VARCHAR.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[VARCHAR.ordinal()][JSONB.ordinal()] = VARCHAR;
+ compatibilityMatrix[VARCHAR.ordinal()][VARIANT.ordinal()] = VARCHAR;
compatibilityMatrix[VARCHAR.ordinal()][DECIMALV2.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][DECIMAL32.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][DECIMAL64.ordinal()] =
INVALID_TYPE;
@@ -889,6 +923,7 @@ public enum PrimitiveType {
compatibilityMatrix[STRING.ordinal()][DECIMAL64.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][DECIMAL128.ordinal()] =
INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][JSONB.ordinal()] = STRING;
+ compatibilityMatrix[STRING.ordinal()][VARIANT.ordinal()] = STRING;
compatibilityMatrix[STRING.ordinal()][TIME.ordinal()] = INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][TIMEV2.ordinal()] = INVALID_TYPE;
@@ -896,6 +931,13 @@ public enum PrimitiveType {
compatibilityMatrix[JSONB.ordinal()][STRING.ordinal()] = STRING;
compatibilityMatrix[JSONB.ordinal()][VARCHAR.ordinal()] = VARCHAR;
compatibilityMatrix[JSONB.ordinal()][CHAR.ordinal()] = CHAR;
+ compatibilityMatrix[JSONB.ordinal()][VARIANT.ordinal()] = VARIANT;
+
+ compatibilityMatrix[VARIANT.ordinal()][VARIANT.ordinal()] = VARIANT;
+ compatibilityMatrix[VARIANT.ordinal()][STRING.ordinal()] = STRING;
+ compatibilityMatrix[VARIANT.ordinal()][VARCHAR.ordinal()] = VARCHAR;
+ compatibilityMatrix[VARIANT.ordinal()][CHAR.ordinal()] = CHAR;
+ compatibilityMatrix[VARIANT.ordinal()][JSONB.ordinal()] = JSONB;
compatibilityMatrix[DECIMALV2.ordinal()][DECIMALV2.ordinal()] =
DECIMALV2;
compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL32.ordinal()] =
DECIMALV2;
@@ -1233,6 +1275,7 @@ public enum PrimitiveType {
case STRING:
return MysqlColType.MYSQL_TYPE_BLOB;
case JSONB:
+ case VARIANT:
return MysqlColType.MYSQL_TYPE_JSON;
case MAP:
return MysqlColType.MYSQL_TYPE_MAP;
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
index 3655a8c5c4..b8dc17d546 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -178,6 +178,8 @@ public class ScalarType extends Type {
return createStringType();
case JSONB:
return createJsonbType();
+ case VARIANT:
+ return createVariantType();
case STRING:
return createStringType();
case HLL:
@@ -247,6 +249,8 @@ public class ScalarType extends Type {
return createVarcharType();
case "JSON":
return createJsonbType();
+ case "VARIANT":
+ return createVariantType();
case "STRING":
case "TEXT":
return createStringType();
@@ -509,6 +513,13 @@ public class ScalarType extends Type {
return type;
}
+ public static ScalarType createVariantType() {
+ // length checked in analysis
+ ScalarType type = new ScalarType(PrimitiveType.VARIANT);
+ type.len = MAX_STRING_LENGTH;
+ return type;
+ }
+
public static ScalarType createVarchar(int len) {
// length checked in analysis
ScalarType type = new ScalarType(PrimitiveType.VARCHAR);
@@ -558,6 +569,8 @@ public class ScalarType extends Type {
return "TEXT";
} else if (type == PrimitiveType.JSONB) {
return "JSON";
+ } else if (type == PrimitiveType.VARIANT) {
+ return "VARIANT";
}
return type.toString();
}
@@ -679,7 +692,8 @@ public class ScalarType extends Type {
case CHAR:
case HLL:
case STRING:
- case JSONB: {
+ case JSONB:
+ case VARIANT: {
scalarType.setLen(getLength());
break;
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 77ac485c75..d48b82ef16 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -113,7 +113,7 @@ public abstract class Type {
public static final StructType GENERIC_STRUCT = new
StructType(Lists.newArrayList(
new StructField("generic_struct", new
ScalarType(PrimitiveType.NULL_TYPE))));
public static final StructType STRUCT = new StructType();
- public static final VariantType VARIANT = new VariantType();
+ public static final ScalarType VARIANT = new
ScalarType(PrimitiveType.VARIANT);
public static final AnyType ANY_STRUCT_TYPE = new AnyStructType();
public static final AnyType ANY_ELEMENT_TYPE = new AnyElementType();
@@ -172,6 +172,7 @@ public abstract class Type {
trivialTypes.add(TIME);
trivialTypes.add(TIMEV2);
trivialTypes.add(JSONB);
+ trivialTypes.add(VARIANT);
supportedTypes = Lists.newArrayList();
supportedTypes.addAll(trivialTypes);
@@ -439,19 +440,23 @@ public abstract class Type {
return isScalarType(PrimitiveType.JSONB);
}
+ public boolean isVariantType() {
+ return isScalarType(PrimitiveType.VARIANT);
+ }
+
// only metric types have the following constraint:
// 1. don't support as key column
// 2. don't support filter
// 3. don't support group by
// 4. don't support index
public boolean isOnlyMetricType() {
- return isObjectStored() || isComplexType() || isJsonbType();
+ return isObjectStored() || isComplexType() || isJsonbType() ||
isVariantType();
}
public static final String OnlyMetricTypeErrorMsg =
- "Doris hll, bitmap, array, map, struct, jsonb column must use with
specific function, and don't"
+ "Doris hll, bitmap, array, map, struct, jsonb, variant column must
use with specific function, and don't"
+ " support filter, group by or order by. please run 'help
hll' or 'help bitmap' or 'help array'"
- + " or 'help map' or 'help struct' or 'help jsonb' in your
mysql client.";
+ + " or 'help map' or 'help struct' or 'help jsonb' or
'help variant' in your mysql client.";
public boolean isHllType() {
return isScalarType(PrimitiveType.HLL);
@@ -544,10 +549,6 @@ public abstract class Type {
return isStructType() || isCollectionType();
}
- public boolean isVariantType() {
- return this instanceof VariantType;
- }
-
public boolean isCollectionType() {
return isMapType() || isArrayType() || isMultiRowType();
}
@@ -715,7 +716,10 @@ public abstract class Type {
}
public static boolean canCastTo(Type sourceType, Type targetType) {
- if (sourceType.isScalarType() && targetType.isScalarType()) {
+ if (sourceType.isVariantType() && (targetType.isScalarType() ||
targetType.isArrayType())) {
+ // variant could cast to scalar types and array
+ return true;
+ } else if (sourceType.isScalarType() && targetType.isScalarType()) {
return ScalarType.canCastTo((ScalarType) sourceType, (ScalarType)
targetType);
} else if (sourceType.isArrayType() && targetType.isArrayType()) {
return ArrayType.canCastTo((ArrayType) sourceType, (ArrayType)
targetType);
@@ -889,6 +893,8 @@ public abstract class Type {
return Type.VARCHAR;
case JSONB:
return Type.JSONB;
+ case VARIANT:
+ return Type.VARIANT;
case STRING:
return Type.STRING;
case HLL:
@@ -903,8 +909,6 @@ public abstract class Type {
return Type.BITMAP;
case QUANTILE_STATE:
return Type.QUANTILE_STATE;
- case VARIANT:
- return new VariantType();
case LAMBDA_FUNCTION:
return Type.LAMBDA_FUNCTION;
case AGG_STATE:
@@ -1260,6 +1264,7 @@ public abstract class Type {
compatibilityMatrix[BOOLEAN.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[BOOLEAN.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL32.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BOOLEAN.ordinal()][DECIMAL64.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1290,6 +1295,7 @@ public abstract class Type {
compatibilityMatrix[TINYINT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[TINYINT.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[TINYINT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1316,6 +1322,7 @@ public abstract class Type {
compatibilityMatrix[SMALLINT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[SMALLINT.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[SMALLINT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1345,6 +1352,7 @@ public abstract class Type {
compatibilityMatrix[INT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[INT.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[INT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1375,6 +1383,7 @@ public abstract class Type {
compatibilityMatrix[BIGINT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[BIGINT.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BIGINT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1397,6 +1406,7 @@ public abstract class Type {
compatibilityMatrix[LARGEINT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[LARGEINT.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[LARGEINT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1418,6 +1428,7 @@ public abstract class Type {
compatibilityMatrix[FLOAT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[FLOAT.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[FLOAT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1435,6 +1446,7 @@ public abstract class Type {
compatibilityMatrix[DOUBLE.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[DOUBLE.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][DATEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DOUBLE.ordinal()][DATETIMEV2.ordinal()] =
PrimitiveType.DOUBLE;
@@ -1457,6 +1469,7 @@ public abstract class Type {
compatibilityMatrix[DATE.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[DATE.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATE.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1476,6 +1489,7 @@ public abstract class Type {
compatibilityMatrix[DATEV2.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATEV2.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATEV2.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[DATEV2.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATEV2.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATEV2.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1494,6 +1508,7 @@ public abstract class Type {
compatibilityMatrix[DATETIME.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[DATETIME.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIME.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1512,6 +1527,7 @@ public abstract class Type {
compatibilityMatrix[DATETIMEV2.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIMEV2.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIMEV2.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[DATETIMEV2.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIMEV2.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DATETIMEV2.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1530,6 +1546,7 @@ public abstract class Type {
compatibilityMatrix[CHAR.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][STRING.ordinal()] =
PrimitiveType.STRING;
compatibilityMatrix[CHAR.ordinal()][JSONB.ordinal()] =
PrimitiveType.CHAR;
+ compatibilityMatrix[CHAR.ordinal()][VARIANT.ordinal()] =
PrimitiveType.CHAR;
compatibilityMatrix[CHAR.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[CHAR.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1546,6 +1563,7 @@ public abstract class Type {
compatibilityMatrix[VARCHAR.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][STRING.ordinal()] =
PrimitiveType.STRING;
compatibilityMatrix[VARCHAR.ordinal()][JSONB.ordinal()] =
PrimitiveType.VARCHAR;
+ compatibilityMatrix[VARCHAR.ordinal()][VARIANT.ordinal()] =
PrimitiveType.VARCHAR;
compatibilityMatrix[VARCHAR.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[VARCHAR.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1561,6 +1579,7 @@ public abstract class Type {
compatibilityMatrix[STRING.ordinal()][DECIMAL64.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][DECIMAL128.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[STRING.ordinal()][JSONB.ordinal()] =
PrimitiveType.STRING;
+ compatibilityMatrix[STRING.ordinal()][VARIANT.ordinal()] =
PrimitiveType.STRING;
compatibilityMatrix[STRING.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
//JSONB
@@ -1579,6 +1598,22 @@ public abstract class Type {
compatibilityMatrix[JSONB.ordinal()][CHAR.ordinal()] =
PrimitiveType.CHAR;
compatibilityMatrix[JSONB.ordinal()][VARCHAR.ordinal()] =
PrimitiveType.VARCHAR;
compatibilityMatrix[JSONB.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ // VARIANT
+ compatibilityMatrix[VARIANT.ordinal()][DECIMALV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][DECIMAL32.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][DECIMAL64.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][DECIMAL128.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][HLL.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][TIME.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][TIMEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][DATEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][DATETIMEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[VARIANT.ordinal()][STRING.ordinal()] =
PrimitiveType.STRING;
+ compatibilityMatrix[VARIANT.ordinal()][CHAR.ordinal()] =
PrimitiveType.CHAR;
+ compatibilityMatrix[VARIANT.ordinal()][VARCHAR.ordinal()] =
PrimitiveType.VARCHAR;
+ compatibilityMatrix[VARIANT.ordinal()][AGG_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
// DECIMALV2
compatibilityMatrix[DECIMALV2.ordinal()][HLL.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1589,6 +1624,7 @@ public abstract class Type {
compatibilityMatrix[DECIMALV2.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[DECIMALV2.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL32.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[DECIMALV2.ordinal()][DECIMAL64.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1645,6 +1681,7 @@ public abstract class Type {
compatibilityMatrix[HLL.ordinal()][BITMAP.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[HLL.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][DECIMAL32.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[HLL.ordinal()][DECIMAL64.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1657,6 +1694,7 @@ public abstract class Type {
compatibilityMatrix[BITMAP.ordinal()][TIMEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[BITMAP.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][QUANTILE_STATE.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][DATEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[BITMAP.ordinal()][DATETIMEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1667,6 +1705,7 @@ public abstract class Type {
//QUANTILE_STATE
compatibilityMatrix[QUANTILE_STATE.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[QUANTILE_STATE.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[QUANTILE_STATE.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[QUANTILE_STATE.ordinal()][DATEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[QUANTILE_STATE.ordinal()][DATETIMEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1677,6 +1716,7 @@ public abstract class Type {
//AGG_STATE
compatibilityMatrix[AGG_STATE.ordinal()][JSONB.ordinal()] =
PrimitiveType.INVALID_TYPE;
+ compatibilityMatrix[AGG_STATE.ordinal()][VARIANT.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[AGG_STATE.ordinal()][STRING.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[AGG_STATE.ordinal()][DATEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
compatibilityMatrix[AGG_STATE.ordinal()][DATETIMEV2.ordinal()] =
PrimitiveType.INVALID_TYPE;
@@ -1764,6 +1804,8 @@ public abstract class Type {
return STRING;
case JSONB:
return JSONB;
+ case VARIANT:
+ return VARIANT;
default:
return INVALID;
@@ -1806,6 +1848,19 @@ public abstract class Type {
return Type.STRING;
}
+ // VARIANT
+ if (t1ResultType == PrimitiveType.VARIANT && t2ResultType ==
PrimitiveType.VARIANT) {
+ return Type.VARIANT;
+ }
+ if ((t1ResultType == PrimitiveType.VARIANT && t2ResultType ==
PrimitiveType.VARCHAR)
+ || (t1ResultType == PrimitiveType.VARCHAR && t2ResultType ==
PrimitiveType.VARIANT)) {
+ return Type.VARCHAR;
+ }
+ if ((t1ResultType == PrimitiveType.VARIANT && t2ResultType ==
PrimitiveType.STRING)
+ || (t1ResultType == PrimitiveType.STRING && t2ResultType ==
PrimitiveType.VARIANT)) {
+ return Type.STRING;
+ }
+
// int family type and char family type should cast to char family type
if ((t1.getPrimitiveType().isFixedPointType() &&
t2.getPrimitiveType().isCharFamily())
|| (t2.getPrimitiveType().isFixedPointType() &&
t1.getPrimitiveType().isCharFamily())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaChangeExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaChangeExpr.java
deleted file mode 100644
index 84f8f4c03d..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaChangeExpr.java
+++ /dev/null
@@ -1,89 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.analysis;
-
-import org.apache.doris.catalog.PrimitiveType;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.catalog.VariantType;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.thrift.TExpr;
-import org.apache.doris.thrift.TExprNode;
-import org.apache.doris.thrift.TExprNodeType;
-import org.apache.doris.thrift.TSchemaChangeExpr;
-import org.apache.doris.thrift.TTypeDesc;
-import org.apache.doris.thrift.TTypeNode;
-
-import com.google.common.base.Preconditions;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.ArrayList;
-
-public class SchemaChangeExpr extends Expr {
- private static final Logger LOG =
LogManager.getLogger(SchemaChangeExpr.class);
- // Target table id
- private int tableId;
- private SlotRef variantSlot;
-
- public SchemaChangeExpr(SlotRef sourceSlot, int tableId) {
- super();
- Preconditions.checkNotNull(sourceSlot);
- Preconditions.checkState(sourceSlot.getType() == Type.VARIANT);
- variantSlot = sourceSlot;
- this.tableId = tableId;
- }
-
- @Override
- protected void treeToThriftHelper(TExpr container) {
- super.treeToThriftHelper(container);
- }
-
- @Override
- protected void toThrift(TExprNode msg) {
- TSchemaChangeExpr schemaInfo = new TSchemaChangeExpr();
- schemaInfo.setTableId(tableId);
- msg.setSchemaChangeExpr(schemaInfo);
- // set src variant slot
- variantSlot.toThrift(msg);
- msg.node_type = TExprNodeType.SCHEMA_CHANGE_EXPR;
- // set type info
- TTypeDesc desc = new TTypeDesc();
- desc.setTypes(new ArrayList<TTypeNode>());
- VariantType variant = new VariantType();
- variant.toThrift(desc);
- msg.setType(desc);
- }
-
- @Override
- public Expr clone() {
- return new SchemaChangeExpr(variantSlot, tableId);
- }
-
- @Override
- public String toSqlImpl() {
- return "SCHEMA_CHANGE(" + variantSlot.toSql() + ")";
- }
-
- @Override
- public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
- Type childType = variantSlot.getType();
- if (childType.getPrimitiveType() != PrimitiveType.VARIANT) {
- throw new AnalysisException("Invalid column " +
variantSlot.toSql());
- }
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
index 17398a5491..964acab5f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java
@@ -25,7 +25,7 @@ import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.NullLiteral;
-import org.apache.doris.analysis.SchemaChangeExpr;
+// import org.apache.doris.analysis.SchemaChangeExpr;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
@@ -36,7 +36,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.TableIf;
+// import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.load.BrokerFileGroup;
@@ -313,10 +313,6 @@ public class FileLoadScanNode extends FileScanNode {
String name = "jsonb_parse_" + nullable + "_error_to_null";
expr = new FunctionCallExpr(name, args);
expr.analyze(analyzer);
- } else if (dstType == PrimitiveType.VARIANT) {
- // Generate SchemaChange expr for dynamicly generating columns
- TableIf targetTbl = desc.getTable();
- expr = new SchemaChangeExpr((SlotRef) expr, (int)
targetTbl.getId());
} else {
expr = castToSlot(destSlotDesc, expr);
}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 6a21a77734..fe5f76f6b4 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -206,6 +206,8 @@ message ColumnPB {
repeated ColumnPB children_columns = 17;
repeated string children_column_names = 18;
optional bool result_is_nullable = 19;
+ // persist info for PathInData that represents path in document, e.g. JSON.
+ optional segment_v2.ColumnPathInfo column_path_info = 20;
}
enum IndexType {
diff --git a/gensrc/proto/segment_v2.proto b/gensrc/proto/segment_v2.proto
index 9bf62846ec..c71bedd6d9 100644
--- a/gensrc/proto/segment_v2.proto
+++ b/gensrc/proto/segment_v2.proto
@@ -137,6 +137,28 @@ message ZoneMapPB {
optional bool pass_all = 5 [default = false];
}
+message ColumnPathPartInfo {
+ // key represent a part of key in full parts info
+ optional string key = 1;
+ // is_nested indicates that the part if a nested array
+ optional bool is_nested = 2;
+ // anonymous_array_level indicates the nested level of array
+ optional uint32 anonymous_array_level = 3;
+}
+
+// Persist info for PathInData that represents path in document, e.g. JSON.
+// Each variant's subcolumn must persist it's path info to storage, since it
only
+// uses path info to identify a column in storage instead of column unique id
+message ColumnPathInfo {
+ // The full path string representation
+ optional string path = 1;
+ // A list of parts, each part indicate a sub path of the whole path
+ repeated ColumnPathPartInfo path_part_infos = 2;
+ optional bool has_nested = 3;
+ // The original parent variant's unique id, used to distinguish from
different variants
+ optional uint32 parrent_column_unique_id = 4;
+}
+
message ColumnMetaPB {
// column id in table schema
optional uint32 column_id = 1;
@@ -162,6 +184,13 @@ message ColumnMetaPB {
optional uint64 num_rows = 11;
repeated string children_column_names = 12;
+ // persist info for PathInData that represents path in document, e.g. JSON.
+ optional ColumnPathInfo column_path_info = 13;
+
+ // Extra type info to be compatible with tabet schema
+ optional bytes default_value = 14; // ColumnMessage.default_value ?
+ optional int32 precision = 15; // ColumnMessage.precision
+ optional int32 frac = 16; // ColumnMessag
}
message PrimaryKeyIndexMetaPB {
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index 1b9f85e2b7..fa391febda 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -60,6 +60,8 @@ struct TSlotDescriptor {
// materialize them.Used to optmize to read less data and less memory usage
13: optional bool need_materialize = true
14: optional bool is_auto_increment = false;
+ // subcolumn path info list for semi structure column(variant)
+ 15: optional list<string> column_paths
}
struct TTupleDescriptor {
diff --git a/regression-test/suites/nereids_arith_p0/bitmap.groovy
b/regression-test/suites/nereids_arith_p0/bitmap.groovy
index 8e7f6cbf54..f7826621be 100644
--- a/regression-test/suites/nereids_arith_p0/bitmap.groovy
+++ b/regression-test/suites/nereids_arith_p0/bitmap.groovy
@@ -23,6 +23,6 @@ suite('bitmap') {
sql "select id from (select BITMAP_EMPTY() as c0 from expr_test) as
ref0 where c0 = 1 order by id"
exception "can not cast from origin type BITMAP to target type=DOUBLE"
sql "select id from expr_test group by id having ktint in (select
BITMAP_EMPTY() from expr_test) order by id"
- exception "Doris hll, bitmap, array, map, struct, jsonb column must
use with specific function,"
+ exception "Doris hll, bitmap, array, map, struct, jsonb, variant
column must use with specific function,"
}
}
diff --git
a/regression-test/suites/nereids_syntax_p0/forbid_unexpected_type.groovy
b/regression-test/suites/nereids_syntax_p0/forbid_unexpected_type.groovy
index 8f7fef423d..bf11bf18e3 100644
--- a/regression-test/suites/nereids_syntax_p0/forbid_unexpected_type.groovy
+++ b/regression-test/suites/nereids_syntax_p0/forbid_unexpected_type.groovy
@@ -48,7 +48,7 @@ suite("forbid_unexpected_type") {
sql """
select min(version()) over (partition by
bitmap_union(bitmap_empty())) as c0 from forbid_unexpected_types
"""
- exception "Doris hll, bitmap, array, map, struct, jsonb column"
+ exception "Doris hll, bitmap, array, map, struct, jsonb, variant
column"
}
// window function's order by could not have bitmap
@@ -56,7 +56,7 @@ suite("forbid_unexpected_type") {
sql """
select min(version()) over (partition by 1 order by
bitmap_union(bitmap_empty())) as c0 from forbid_unexpected_types
"""
- exception "Doris hll, bitmap, array, map, struct, jsonb column"
+ exception "Doris hll, bitmap, array, map, struct, jsonb, variant
column"
}
}
diff --git
a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
index d0cf4b2384..e8e5d33d16 100644
--- a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
+++ b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy
@@ -78,7 +78,7 @@ suite("test_bitmap_filter_nereids") {
test {
sql "select k1, count(*) from ${tbl1} b1 group by k1 having k1 in
(select k2 from ${tbl2} b2) order by k1;"
- exception "Doris hll, bitmap, array, map, struct, jsonb column must
use with specific function, and don't support filter"
+ exception "Doris hll, bitmap, array, map, struct, jsonb, variant
column must use with specific function, and don't support filter"
}
explain{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
