This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 151842a1fe [feature](inverted index)WIP inverted index api: SQL syntax
and metadata (#13430)
151842a1fe is described below
commit 151842a1fe7f431a458446278a43f974e28b2883
Author: Kang <[email protected]>
AuthorDate: Tue Nov 8 23:46:53 2022 +0800
[feature](inverted index)WIP inverted index api: SQL syntax and metadata
(#13430)
Introduce a SQL syntax for creating inverted index and related metadata
changes.
```
-- create table with INVERTED index
CREATE TABLE httplogs (
ts datetime,
clientip varchar(20),
request string,
status smallint,
size int,
INDEX idx_size (size) USING INVERTED,
INDEX idx_status (status) USING INVERTED,
INDEX idx_clientip (clientip) USING INVERTED PROPERTIES("parser"="none")
)
DUPLICATE KEY(ts)
DISTRIBUTED BY RANDOM BUCKETS 10
-- add an INVERTED index to a table
CREATE INDEX idx_request ON httplogs(request) USING INVERTED
PROPERTIES("parser"="english");
```
---
be/src/olap/CMakeLists.txt | 1 +
be/src/olap/inverted_index_parser.cpp | 65 ++++++++
be/src/olap/inverted_index_parser.h | 46 ++++++
be/src/olap/tablet_meta.cpp | 35 +++++
be/src/olap/tablet_schema.cpp | 120 +++++++++++++++
be/src/olap/tablet_schema.h | 29 ++++
fe/fe-core/src/main/cup/sql_parser.cup | 15 +-
.../apache/doris/analysis/CreateIndexClause.java | 6 +-
.../org/apache/doris/analysis/CreateTableStmt.java | 5 +-
.../java/org/apache/doris/analysis/IndexDef.java | 52 ++++++-
.../apache/doris/analysis/InvertedIndexUtil.java | 56 +++++++
.../org/apache/doris/analysis/ShowIndexStmt.java | 1 +
.../main/java/org/apache/doris/catalog/Index.java | 56 ++++++-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../doris/analysis/CreateIndexClauseTest.java | 2 +-
.../org/apache/doris/analysis/IndexDefTest.java | 6 +-
.../org/apache/doris/catalog/OlapTableTest.java | 4 +-
.../persist/TableAddOrDropColumnsInfoTest.java | 2 +-
gensrc/proto/olap_file.proto | 15 ++
gensrc/thrift/Descriptors.thrift | 6 +-
.../data/index_p0/test_bitmap_index.out | 92 ++++++------
.../test_decimal_bitmap_index_multi_page.out | 2 +-
.../suites/index_p0/test_index_meta.groovy | 167 +++++++++++++++++++++
24 files changed, 710 insertions(+), 76 deletions(-)
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index 225f60dad2..f46dd6259d 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -41,6 +41,7 @@ add_library(Olap STATIC
file_helper.cpp
generic_iterators.cpp
hll.cpp
+ inverted_index_parser.cpp
bloom_filter_predicate.cpp
like_column_predicate.cpp
key_coder.cpp
diff --git a/be/src/olap/inverted_index_parser.cpp
b/be/src/olap/inverted_index_parser.cpp
new file mode 100644
index 0000000000..9407b52ee4
--- /dev/null
+++ b/be/src/olap/inverted_index_parser.cpp
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/inverted_index_parser.h"
+
+#include "util/string_util.h"
+
+namespace doris {
+
+std::string inverted_index_parser_type_to_string(InvertedIndexParserType
parser_type) {
+ switch (parser_type) {
+ case InvertedIndexParserType::PARSER_NONE:
+ return INVERTED_INDEX_PARSER_NONE;
+ case InvertedIndexParserType::PARSER_STANDARD:
+ return INVERTED_INDEX_PARSER_STANDARD;
+ case InvertedIndexParserType::PARSER_ENGLISH:
+ return INVERTED_INDEX_PARSER_ENGLISH;
+ case InvertedIndexParserType::PARSER_CHINESE:
+ return INVERTED_INDEX_PARSER_CHINESE;
+ default:
+ return INVERTED_INDEX_PARSER_UNKNOWN;
+ }
+
+ return INVERTED_INDEX_PARSER_UNKNOWN;
+}
+
+InvertedIndexParserType get_inverted_index_parser_type_from_string(const
std::string& parser_str) {
+ auto parser_str_lower = to_lower(parser_str);
+ if (parser_str_lower == INVERTED_INDEX_PARSER_NONE) {
+ return InvertedIndexParserType::PARSER_NONE;
+ } else if (parser_str_lower == INVERTED_INDEX_PARSER_STANDARD) {
+ return InvertedIndexParserType::PARSER_STANDARD;
+ } else if (parser_str_lower == INVERTED_INDEX_PARSER_ENGLISH) {
+ return InvertedIndexParserType::PARSER_ENGLISH;
+ } else if (parser_str_lower == INVERTED_INDEX_PARSER_CHINESE) {
+ return InvertedIndexParserType::PARSER_CHINESE;
+ }
+
+ return InvertedIndexParserType::PARSER_UNKNOWN;
+}
+
+std::string get_parser_string_from_properties(
+ const std::map<std::string, std::string>& properties) {
+ if (properties.find(INVERTED_INDEX_PARSER_KEY) != properties.end()) {
+ return properties.at(INVERTED_INDEX_PARSER_KEY);
+ } else {
+ return INVERTED_INDEX_PARSER_NONE;
+ }
+}
+
+} // namespace doris
diff --git a/be/src/olap/inverted_index_parser.h
b/be/src/olap/inverted_index_parser.h
new file mode 100644
index 0000000000..0c870aa355
--- /dev/null
+++ b/be/src/olap/inverted_index_parser.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <map>
+#include <string>
+
+namespace doris {
+
+enum class InvertedIndexParserType {
+ PARSER_UNKNOWN = 0,
+ PARSER_NONE = 1,
+ PARSER_STANDARD = 2,
+ PARSER_ENGLISH = 3,
+ PARSER_CHINESE = 4,
+};
+
+const std::string INVERTED_INDEX_PARSER_KEY = "parser";
+const std::string INVERTED_INDEX_PARSER_UNKNOWN = "unknown";
+const std::string INVERTED_INDEX_PARSER_NONE = "none";
+const std::string INVERTED_INDEX_PARSER_STANDARD = "standard";
+const std::string INVERTED_INDEX_PARSER_ENGLISH = "english";
+const std::string INVERTED_INDEX_PARSER_CHINESE = "chinese";
+
+std::string inverted_index_parser_type_to_string(InvertedIndexParserType
parser_type);
+
+InvertedIndexParserType get_inverted_index_parser_type_from_string(const
std::string& parser_str);
+
+std::string get_parser_string_from_properties(const std::map<std::string,
std::string>& properties);
+
+} // namespace doris
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 70348ba110..010b2217c6 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -169,6 +169,41 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
}
}
+ // copy index meta
+ if (tablet_schema.__isset.indexes) {
+ for (auto& index : tablet_schema.indexes) {
+ TabletIndexPB* index_pb = schema->add_index();
+ index_pb->set_index_id(index.index_id);
+ index_pb->set_index_name(index.index_name);
+ // init col_unique_id in index at be side, since col_unique_id may
be -1 at fe side
+ // get column unique id by name
+ for (auto column_name : index.columns) {
+ for (auto column : schema->column()) {
+ if (iequal(column.name(), column_name)) {
+ index_pb->add_col_unique_id(column.unique_id());
+ }
+ }
+ }
+ switch (index.index_type) {
+ case TIndexType::BITMAP:
+ index_pb->set_index_type(IndexType::BITMAP);
+ break;
+ case TIndexType::INVERTED:
+ index_pb->set_index_type(IndexType::INVERTED);
+ break;
+ case TIndexType::BLOOMFILTER:
+ index_pb->set_index_type(IndexType::BLOOMFILTER);
+ break;
+ }
+ if (index.__isset.properties) {
+ auto properties = index_pb->mutable_properties();
+ for (auto kv : index.properties) {
+ (*properties)[kv.first] = kv.second;
+ }
+ }
+ }
+ }
+
schema->set_next_column_unique_id(next_unique_id);
if (has_bf_columns && tablet_schema.__isset.bloom_filter_fpp) {
schema->set_bf_fpp(tablet_schema.bloom_filter_fpp);
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 9dcc7a5793..fe4b353e5a 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -469,6 +469,62 @@ vectorized::AggregateFunctionPtr
TabletColumn::get_aggregate_function(
agg_name, argument_types, {},
argument_types.back()->is_nullable());
}
+void TabletIndex::init_from_thrift(const TOlapTableIndex& index,
+ const TabletSchema& tablet_schema) {
+ _index_id = index.index_id;
+ _index_name = index.index_name;
+ // init col_unique_id in index at be side, since col_unique_id may be -1
at fe side
+ // get column unique id by name
+ std::vector<int32_t> col_unique_ids(index.columns.size());
+ for (size_t i = 0; i < index.columns.size(); i++) {
+ col_unique_ids[i] = tablet_schema.column(index.columns[i]).unique_id();
+ }
+ _col_unique_ids = std::move(col_unique_ids);
+
+ switch (index.index_type) {
+ case TIndexType::BITMAP:
+ _index_type = IndexType::BITMAP;
+ break;
+ case TIndexType::INVERTED:
+ _index_type = IndexType::INVERTED;
+ break;
+ case TIndexType::BLOOMFILTER:
+ _index_type = IndexType::BLOOMFILTER;
+ break;
+ }
+ if (index.__isset.properties) {
+ for (auto kv : index.properties) {
+ _properties[kv.first] = kv.second;
+ }
+ }
+}
+
+void TabletIndex::init_from_pb(const TabletIndexPB& index) {
+ _index_id = index.index_id();
+ _index_name = index.index_name();
+ _col_unique_ids.clear();
+ for (auto col_unique_id : index.col_unique_id()) {
+ _col_unique_ids.push_back(col_unique_id);
+ }
+ _index_type = index.index_type();
+ for (auto& kv : index.properties()) {
+ _properties[kv.first] = kv.second;
+ }
+}
+
+void TabletIndex::to_schema_pb(TabletIndexPB* index) const {
+ index->set_index_id(_index_id);
+ index->set_index_name(_index_name);
+ index->clear_col_unique_id();
+ for (auto col_unique_id : _col_unique_ids) {
+ index->add_col_unique_id(col_unique_id);
+ }
+ index->set_index_type(_index_type);
+ for (auto& kv : _properties) {
+ (*index->mutable_properties())[kv.first] = kv.second;
+ }
+}
+
void TabletSchema::append_column(TabletColumn column, bool is_dropped_column) {
if (column.is_key()) {
_num_key_columns++;
@@ -522,6 +578,11 @@ void TabletSchema::init_from_pb(const TabletSchemaPB&
schema) {
_cols.emplace_back(std::move(column));
_num_columns++;
}
+ for (auto& index_pb : schema.index()) {
+ TabletIndex index;
+ index.init_from_pb(index_pb);
+ _indexes.emplace_back(std::move(index));
+ }
_num_short_key_columns = schema.num_short_key_columns();
_num_rows_per_row_block = schema.num_rows_per_row_block();
_compress_kind = schema.compress_kind();
@@ -647,6 +708,10 @@ void TabletSchema::to_schema_pb(TabletSchemaPB*
tablet_schema_pb) const {
ColumnPB* column = tablet_schema_pb->add_column();
col.to_schema_pb(column);
}
+ for (auto& index : _indexes) {
+ auto index_pb = tablet_schema_pb->add_index();
+ index.to_schema_pb(index_pb);
+ }
tablet_schema_pb->set_num_short_key_columns(_num_short_key_columns);
tablet_schema_pb->set_num_rows_per_row_block(_num_rows_per_row_block);
tablet_schema_pb->set_compress_kind(_compress_kind);
@@ -710,11 +775,66 @@ const TabletColumn& TabletSchema::column_by_uid(int32_t
col_unique_id) const {
return _cols.at(_field_id_to_index.at(col_unique_id));
}
+void TabletSchema::update_indexes_from_thrift(const
std::vector<doris::TOlapTableIndex>& tindexes) {
+ std::vector<TabletIndex> indexes;
+ for (auto& tindex : tindexes) {
+ TabletIndex index;
+ index.init_from_thrift(tindex, *this);
+ indexes.emplace_back(std::move(index));
+ }
+ _indexes = std::move(indexes);
+}
+
const TabletColumn& TabletSchema::column(const std::string& field_name) const {
const auto& found = _field_name_to_index.find(field_name);
return _cols[found->second];
}
+std::vector<const TabletIndex*> TabletSchema::get_indexes_for_column(int32_t
col_unique_id) const {
+ std::vector<const TabletIndex*> indexes_for_column;
+
+ // TODO use more efficient impl
+ for (size_t i = 0; i < _indexes.size(); i++) {
+ for (int32_t id : _indexes[i].col_unique_ids()) {
+ if (id == col_unique_id) {
+ indexes_for_column.push_back(&(_indexes[i]));
+ }
+ }
+ }
+
+ return indexes_for_column;
+}
+
+bool TabletSchema::has_inverted_index(int32_t col_unique_id) const {
+ // TODO use more efficient impl
+ for (size_t i = 0; i < _indexes.size(); i++) {
+ if (_indexes[i].index_type() == IndexType::INVERTED) {
+ for (int32_t id : _indexes[i].col_unique_ids()) {
+ if (id == col_unique_id) {
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+}
+
+const TabletIndex* TabletSchema::get_inverted_index(int32_t col_unique_id)
const {
+ // TODO use more efficient impl
+ for (size_t i = 0; i < _indexes.size(); i++) {
+ if (_indexes[i].index_type() == IndexType::INVERTED) {
+ for (int32_t id : _indexes[i].col_unique_ids()) {
+ if (id == col_unique_id) {
+ return &(_indexes[i]);
+ }
+ }
+ }
+ }
+
+ return nullptr;
+}
+
vectorized::Block TabletSchema::create_block(
const std::vector<uint32_t>& return_columns,
const std::unordered_set<uint32_t>* tablet_columns_need_convert_null)
const {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 300b603e83..6702901e83 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -117,6 +117,28 @@ private:
bool operator==(const TabletColumn& a, const TabletColumn& b);
bool operator!=(const TabletColumn& a, const TabletColumn& b);
+class TabletSchema;
+
+class TabletIndex {
+public:
+ void init_from_thrift(const TOlapTableIndex& index, const TabletSchema&
tablet_schema);
+ void init_from_pb(const TabletIndexPB& index);
+ void to_schema_pb(TabletIndexPB* index) const;
+
+ const int64_t index_id() const { return _index_id; }
+ const std::string& index_name() const { return _index_name; }
+ const IndexType index_type() const { return _index_type; }
+ const vector<int32_t>& col_unique_ids() const { return _col_unique_ids; }
+ const std::map<string, string>& properties() const { return _properties; }
+
+private:
+ int64_t _index_id;
+ std::string _index_name;
+ IndexType _index_type;
+ std::vector<int32_t> _col_unique_ids;
+ std::map<string, string> _properties;
+};
+
class TabletSchema {
public:
// TODO(yingchun): better to make constructor as private to avoid
@@ -161,6 +183,12 @@ public:
int32_t sequence_col_idx() const { return _sequence_col_idx; }
segment_v2::CompressionTypePB compression_type() const { return
_compression_type; }
+ const std::vector<TabletIndex>& indexes() const { return _indexes; }
+ std::vector<const TabletIndex*> get_indexes_for_column(int32_t
col_unique_id) const;
+ bool has_inverted_index(int32_t col_unique_id) const;
+ const TabletIndex* get_inverted_index(int32_t col_unique_id) const;
+ void update_indexes_from_thrift(const std::vector<doris::TOlapTableIndex>&
indexes);
+
int32_t schema_version() const { return _schema_version; }
void clear_columns();
vectorized::Block create_block(
@@ -196,6 +224,7 @@ private:
SortType _sort_type = SortType::LEXICAL;
size_t _sort_col_num = 0;
std::vector<TabletColumn> _cols;
+ std::vector<TabletIndex> _indexes;
std::unordered_map<std::string, int32_t> _field_name_to_index;
std::unordered_map<int32_t, int32_t> _field_id_to_index;
size_t _num_columns = 0;
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index a9cf6b3dea..adec23db2c 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -409,6 +409,7 @@ terminal String
KW_IS,
KW_ISNULL,
KW_ISOLATION,
+ KW_INVERTED,
KW_JOB,
KW_JOIN,
KW_JSON,
@@ -1830,9 +1831,9 @@ create_stmt ::=
{:
RESULT = new CreateMultiTableMaterializedViewStmt(mvName, buildMethod,
refreshInfo, keyDesc, partitionDesc, distributionDesc, properties, query);
:}
- | KW_CREATE KW_INDEX opt_if_not_exists:ifNotExists ident:indexName KW_ON
table_name:tableName LPAREN ident_list:cols RPAREN opt_index_type:indexType
opt_comment:comment
+ | KW_CREATE KW_INDEX opt_if_not_exists:ifNotExists ident:indexName KW_ON
table_name:tableName LPAREN ident_list:cols RPAREN opt_index_type:indexType
opt_properties:properties opt_comment:comment
{:
- RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new
CreateIndexClause(tableName, new IndexDef(indexName, ifNotExists, cols,
indexType, comment), false)));
+ RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new
CreateIndexClause(tableName, new IndexDef(indexName, ifNotExists, cols,
indexType, properties, comment), false)));
:}
/* resource */
| KW_CREATE opt_external:isExternal KW_RESOURCE
opt_if_not_exists:ifNotExists ident_or_text:resourceName
opt_properties:properties
@@ -3142,9 +3143,9 @@ column_definition ::=
;
index_definition ::=
- KW_INDEX opt_if_not_exists:ifNotExists ident:indexName LPAREN
ident_list:cols RPAREN opt_index_type:indexType opt_comment:comment
+ KW_INDEX opt_if_not_exists:ifNotExists ident:indexName LPAREN
ident_list:cols RPAREN opt_index_type:indexType opt_properties:properties
opt_comment:comment
{:
- RESULT = new IndexDef(indexName, ifNotExists, cols, indexType,
comment);
+ RESULT = new IndexDef(indexName, ifNotExists, cols, indexType,
properties, comment);
:}
;
@@ -3181,6 +3182,10 @@ opt_index_type ::=
{:
RESULT = IndexDef.IndexType.BITMAP;
:}
+ | KW_USING KW_INVERTED
+ {:
+ RESULT = IndexDef.IndexType.INVERTED;
+ :}
;
opt_if_exists ::=
@@ -6489,6 +6494,8 @@ keyword ::=
{: RESULT = id; :}
| KW_INDEXES:id
{: RESULT = id; :}
+ | KW_INVERTED:id
+ {: RESULT = id; :}
| KW_ISNULL:id
{: RESULT = id; :}
| KW_ISOLATION:id
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java
index d1ff4006ba..632492191c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateIndexClause.java
@@ -18,6 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.alter.AlterOpType;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
import org.apache.doris.common.AnalysisException;
@@ -70,8 +71,9 @@ public class CreateIndexClause extends AlterTableClause {
throw new AnalysisException("index definition expected.");
}
indexDef.analyze();
- this.index = new Index(indexDef.getIndexName(), indexDef.getColumns(),
indexDef.getIndexType(),
- indexDef.getComment());
+ this.index = new Index(Env.getCurrentEnv().getNextId(),
indexDef.getIndexName(),
+ indexDef.getColumns(), indexDef.getIndexType(),
+ indexDef.getProperties(), indexDef.getComment());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 7ba06377a0..7f33428da3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -488,8 +488,9 @@ public class CreateTableStmt extends DdlStmt {
+ indexColName);
}
}
- indexes.add(new Index(indexDef.getIndexName(),
indexDef.getColumns(), indexDef.getIndexType(),
- indexDef.getComment()));
+ indexes.add(new Index(Env.getCurrentEnv().getNextId(),
indexDef.getIndexName(),
+ indexDef.getColumns(), indexDef.getIndexType(),
+ indexDef.getProperties(), indexDef.getComment()));
distinct.add(indexDef.getIndexName());
distinctCol.add(indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index fca80403c1..4df18a9879 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -24,7 +24,9 @@ import org.apache.doris.common.AnalysisException;
import com.google.common.base.Strings;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.TreeSet;
public class IndexDef {
@@ -33,8 +35,10 @@ public class IndexDef {
private List<String> columns;
private IndexType indexType;
private String comment;
+ private Map<String, String> properties;
- public IndexDef(String indexName, boolean ifNotExists, List<String>
columns, IndexType indexType, String comment) {
+ public IndexDef(String indexName, boolean ifNotExists, List<String>
columns, IndexType indexType,
+ Map<String, String> properties, String comment) {
this.indexName = indexName;
this.ifNotExists = ifNotExists;
this.columns = columns;
@@ -48,12 +52,18 @@ public class IndexDef {
} else {
this.comment = comment;
}
+ if (properties == null) {
+ this.properties = new HashMap<>();
+ } else {
+ this.properties = properties;
+ }
}
public void analyze() throws AnalysisException {
- if (indexType == IndexDef.IndexType.BITMAP) {
+ if (indexType == IndexDef.IndexType.BITMAP
+ || indexType == IndexDef.IndexType.INVERTED) {
if (columns == null || columns.size() != 1) {
- throw new AnalysisException("bitmap index can only apply to a
single column.");
+ throw new AnalysisException(indexType.toString() + " index can
only apply to a single column.");
}
if (Strings.isNullOrEmpty(indexName)) {
throw new AnalysisException("index name cannot be blank.");
@@ -93,6 +103,19 @@ public class IndexDef {
if (indexType != null) {
sb.append(" USING ").append(indexType.toString());
}
+ if (properties != null && properties.size() > 0) {
+ sb.append(" PROPERTIES(");
+ first = true;
+ for (Map.Entry<String, String> e : properties.entrySet()) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(", ");
+ }
+
sb.append("\"").append(e.getKey()).append("\"=").append("\"").append(e.getValue()).append("\"");
+ }
+ sb.append(")");
+ }
if (comment != null) {
sb.append(" COMMENT '" + comment + "'");
}
@@ -116,6 +139,10 @@ public class IndexDef {
return indexType;
}
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
public String getComment() {
return comment;
}
@@ -126,29 +153,38 @@ public class IndexDef {
public enum IndexType {
BITMAP,
+ INVERTED,
+ BLOOMFILTER,
+ }
+ public boolean isInvertedIndex() {
+ return (this.indexType == IndexType.INVERTED);
}
public void checkColumn(Column column, KeysType keysType) throws
AnalysisException {
- if (indexType == IndexType.BITMAP) {
+ if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED
|| indexType == IndexType.BLOOMFILTER) {
String indexColName = column.getName();
PrimitiveType colType = column.getDataType();
if (!(colType.isDateType() || colType.isDecimalV2Type() ||
colType.isDecimalV3Type()
|| colType.isFixedPointType() || colType.isStringType() ||
colType == PrimitiveType.BOOLEAN)) {
- throw new AnalysisException(colType + " is not supported in
bitmap index. "
+ throw new AnalysisException(colType + " is not supported in "
+ indexType.toString() + " index. "
+ "invalid column: " + indexColName);
} else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) {
- throw new AnalysisException(
- "BITMAP index only used in columns of
DUP_KEYS/UNIQUE_KEYS table or key columns of"
+ throw new AnalysisException(indexType.toString()
+ + " index only used in columns of DUP_KEYS/UNIQUE_KEYS
table or key columns of"
+ " AGG_KEYS table. invalid column: " +
indexColName);
}
+
+ if (indexType == IndexType.INVERTED) {
+ InvertedIndexUtil.checkInvertedIndexParser(indexColName,
colType, properties);
+ }
} else {
throw new AnalysisException("Unsupported index type: " +
indexType);
}
}
public void checkColumns(List<Column> columns, KeysType keysType) throws
AnalysisException {
- if (indexType == IndexType.BITMAP) {
+ if (indexType == IndexType.BITMAP || indexType == IndexType.INVERTED
|| indexType == IndexType.BLOOMFILTER) {
for (Column col : columns) {
checkColumn(col, keysType);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
new file mode 100644
index 0000000000..412935ab91
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.AnalysisException;
+
+import java.util.Map;
+
+public class InvertedIndexUtil {
+
+ public static String INVERTED_INDEX_PARSER_KEY = "parser";
+ public static String INVERTED_INDEX_PARSER_UNKNOWN = "unknown";
+ public static String INVERTED_INDEX_PARSER_NONE = "none";
+ public static String INVERTED_INDEX_PARSER_STANDARD = "standard";
+ public static String INVERTED_INDEX_PARSER_ENGLISH = "english";
+ public static String INVERTED_INDEX_PARSER_CHINESE = "chinese";
+
+ public static String getInvertedIndexParser(Map<String, String>
properties) {
+ String parser = properties == null ? null :
properties.get(INVERTED_INDEX_PARSER_KEY);
+ // default is "none" if not set
+ return parser != null ? parser : INVERTED_INDEX_PARSER_NONE;
+ }
+
+ public static void checkInvertedIndexParser(String indexColName,
PrimitiveType colType,
+ Map<String, String> properties) throws AnalysisException {
+ String parser = getInvertedIndexParser(properties);
+ if (colType.isStringType()) {
+ if (!(parser.equals(INVERTED_INDEX_PARSER_NONE)
+ || parser.equals(INVERTED_INDEX_PARSER_STANDARD)
+ || parser.equals(INVERTED_INDEX_PARSER_ENGLISH)
+ || parser.equals(INVERTED_INDEX_PARSER_CHINESE))) {
+ throw new AnalysisException("INVERTED index parser: " + parser
+ + " is invalid for column: " + indexColName + " of type "
+ colType);
+ }
+ } else if (!parser.equals(INVERTED_INDEX_PARSER_NONE)) {
+ throw new AnalysisException("INVERTED index with parser: " + parser
+ + " is not supported for column: " + indexColName + " of type
" + colType);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java
index 38468e5c7c..0a3977f03f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java
@@ -46,6 +46,7 @@ public class ShowIndexStmt extends ShowStmt {
.addColumn(new Column("Null",
ScalarType.createVarchar(80)))
.addColumn(new Column("Index_type",
ScalarType.createVarchar(80)))
.addColumn(new Column("Comment",
ScalarType.createVarchar(80)))
+ .addColumn(new Column("Properties",
ScalarType.createVarchar(200)))
.build();
private String dbName;
private TableName tableName;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
index b5536faf55..2da8ce35ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
@@ -18,8 +18,10 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
+import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TIndexType;
import org.apache.doris.thrift.TOlapTableIndex;
@@ -30,26 +32,37 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Internal representation of index, including index type, name, columns and
comments.
* This class will used in olaptable
*/
public class Index implements Writable {
+ public static final int INDEX_ID_INIT_VALUE = -1;
+
+ @SerializedName(value = "indexId")
+ private long indexId = -1; // -1 for compatibale
@SerializedName(value = "indexName")
private String indexName;
@SerializedName(value = "columns")
private List<String> columns;
@SerializedName(value = "indexType")
private IndexDef.IndexType indexType;
+ @SerializedName(value = "properties")
+ private Map<String, String> properties;
@SerializedName(value = "comment")
private String comment;
- public Index(String indexName, List<String> columns, IndexDef.IndexType
indexType, String comment) {
+ public Index(long indexId, String indexName, List<String> columns,
+ IndexDef.IndexType indexType, Map<String, String> properties,
String comment) {
+ this.indexId = indexId;
this.indexName = indexName;
this.columns = columns;
this.indexType = indexType;
+ this.properties = properties;
this.comment = comment;
}
@@ -57,9 +70,18 @@ public class Index implements Writable {
this.indexName = null;
this.columns = null;
this.indexType = null;
+ this.properties = null;
this.comment = null;
}
+ public long getIndexId() {
+ return indexId;
+ }
+
+ public void setIndexId(long indexId) {
+ this.indexId = indexId;
+ }
+
public String getIndexName() {
return indexName;
}
@@ -84,6 +106,26 @@ public class Index implements Writable {
this.indexType = indexType;
}
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ public String getPropertiesString() {
+ if (properties == null || properties.isEmpty()) {
+ return "";
+ }
+
+ return "(" + new PrintableMap(properties, "=", true, false,
",").toString() + ")";
+ }
+
+ public String getInvertedIndexParser() {
+ return InvertedIndexUtil.getInvertedIndexParser(properties);
+ }
+
public String getComment() {
return comment;
}
@@ -108,7 +150,8 @@ public class Index implements Writable {
}
public Index clone() {
- return new Index(indexName, new ArrayList<>(columns), indexType,
comment);
+ return new Index(indexId, indexName, new ArrayList<>(columns),
+ indexType, new HashMap<>(properties), comment);
}
@Override
@@ -133,6 +176,10 @@ public class Index implements Writable {
if (indexType != null) {
sb.append(" USING ").append(indexType.toString());
}
+ if (properties != null && properties.size() > 0) {
+ sb.append(" PROPERTIES");
+ sb.append(getPropertiesString());
+ }
if (comment != null) {
sb.append(" COMMENT '" + comment + "'");
}
@@ -141,11 +188,12 @@ public class Index implements Writable {
public TOlapTableIndex toThrift() {
TOlapTableIndex tIndex = new TOlapTableIndex();
+ tIndex.setIndexId(indexId);
tIndex.setIndexName(indexName);
tIndex.setColumns(columns);
tIndex.setIndexType(TIndexType.valueOf(indexType.toString()));
- if (columns != null) {
- tIndex.setComment(comment);
+ if (properties != null) {
+ tIndex.setProperties(properties);
}
return tIndex;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 0c7077a84f..f72aab6f0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -963,7 +963,7 @@ public class ShowExecutor {
for (Index index : indexes) {
rows.add(Lists.newArrayList(showStmt.getTableName().toString(), "",
index.getIndexName(),
"", String.join(",", index.getColumns()), "", "", "",
"",
- "", index.getIndexType().name(), index.getComment()));
+ "", index.getIndexType().name(), index.getComment(),
index.getPropertiesString()));
}
} finally {
table.readUnlock();
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex
b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 4a15fe3521..541322b57a 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -118,6 +118,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("bin", new Integer(SqlParserSymbols.KW_BIN));
keywordMap.put("binlog", new Integer(SqlParserSymbols.KW_BINLOG));
keywordMap.put("bitmap", new Integer(SqlParserSymbols.KW_BITMAP));
+ keywordMap.put("inverted", new Integer(SqlParserSymbols.KW_INVERTED));
keywordMap.put("bitmap_union", new
Integer(SqlParserSymbols.KW_BITMAP_UNION));
keywordMap.put("blob", new Integer(SqlParserSymbols.KW_BLOB));
keywordMap.put("boolean", new Integer(SqlParserSymbols.KW_BOOLEAN));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java
index fe46c0c4af..88aae9c4e2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java
@@ -37,7 +37,7 @@ public class CreateIndexClauseTest {
public void testNormal() throws AnalysisException {
CreateIndexClause clause = new CreateIndexClause(
new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, "db",
"table"),
- new IndexDef("index1", false, Lists.newArrayList("col1"),
IndexDef.IndexType.BITMAP, "balabala"),
+ new IndexDef("index1", false, Lists.newArrayList("col1"),
IndexDef.IndexType.BITMAP, null, "balabala"),
false);
clause.analyze(analyzer);
Assert.assertEquals("CREATE INDEX index1 ON `db`.`table` (`col1`)
USING BITMAP COMMENT 'balabala'",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
index 5f18742cb9..b1072e41fa 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
@@ -29,7 +29,7 @@ public class IndexDefTest {
@Before
public void setUp() throws Exception {
- def = new IndexDef("index1", false, Lists.newArrayList("col1"),
IndexDef.IndexType.BITMAP, "balabala");
+ def = new IndexDef("index1", false, Lists.newArrayList("col1"),
IndexDef.IndexType.BITMAP, null, "balabala");
}
@Test
@@ -46,7 +46,7 @@ public class IndexDefTest {
+
"x1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxx"
+
"xxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxx"
+ "xxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxx", false,
- Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP,
+ Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP,
null,
"balabala");
def.analyze();
Assert.fail("No exception throws.");
@@ -54,7 +54,7 @@ public class IndexDefTest {
Assert.assertTrue(e instanceof AnalysisException);
}
try {
- def = new IndexDef("", false, Lists.newArrayList("col1"),
IndexDef.IndexType.BITMAP, "balabala");
+ def = new IndexDef("", false, Lists.newArrayList("col1"),
IndexDef.IndexType.BITMAP, null, "balabala");
def.analyze();
Assert.fail("No exception throws.");
} catch (AnalysisException e) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index 871389d6dd..f547189ff1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -56,8 +56,8 @@ public class OlapTableTest {
continue;
}
OlapTable tbl = (OlapTable) table;
- tbl.setIndexes(Lists.newArrayList(new Index("index",
Lists.newArrayList("col"),
- IndexDef.IndexType.BITMAP, "xxxxxx")));
+ tbl.setIndexes(Lists.newArrayList(new Index(0, "index",
Lists.newArrayList("col"),
+ IndexDef.IndexType.BITMAP, null, "xxxxxx")));
System.out.println("orig table id: " + tbl.getId());
FastByteArrayOutputStream byteArrayOutputStream = new
FastByteArrayOutputStream();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
index a97dd27bf9..e849198a5b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java
@@ -65,7 +65,7 @@ public class TableAddOrDropColumnsInfoTest {
indexSchemaMap.put(tableId, fullSchema);
List<Index> indexes = Lists.newArrayList(
- new Index("index", Lists.newArrayList("testCol1"),
IndexDef.IndexType.BITMAP, "xxxxxx"));
+ new Index(0, "index", Lists.newArrayList("testCol1"),
IndexDef.IndexType.BITMAP, null, "xxxxxx"));
TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new
TableAddOrDropColumnsInfo(dbId, tableId,
indexSchemaMap, indexes, jobId);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index f19228b935..30b127fc78 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -201,6 +201,20 @@ message ColumnPB {
repeated string children_column_names = 18;
}
+enum IndexType {
+ BITMAP = 0;
+ INVERTED = 1;
+ BLOOMFILTER = 2;
+}
+
+message TabletIndexPB {
+ optional int64 index_id = 1;
+ optional string index_name = 2;
+ optional IndexType index_type = 3;
+ repeated int32 col_unique_id = 4;
+ map<string, string> properties = 5;
+}
+
enum SortType {
LEXICAL = 0;
ZORDER = 1;
@@ -222,6 +236,7 @@ message TabletSchemaPB {
optional segment_v2.CompressionTypePB compression_type = 13 [default=LZ4F];
optional int32 schema_version = 14;
optional bool disable_auto_compaction = 15 [default=false];
+ repeated TabletIndexPB index = 16;
}
enum TabletStatePB {
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index a3fc84571b..d133b3a27d 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -118,7 +118,9 @@ enum THdfsCompression {
}
enum TIndexType {
- BITMAP
+ BITMAP,
+ INVERTED,
+ BLOOMFILTER
}
// Mapping from names defined by Avro to the enum.
@@ -195,6 +197,8 @@ struct TOlapTableIndex {
2: optional list<string> columns
3: optional TIndexType index_type
4: optional string comment
+ 5: optional i64 index_id
+ 6: optional map<string, string> properties
}
struct TTabletLocation {
diff --git a/regression-test/data/index_p0/test_bitmap_index.out
b/regression-test/data/index_p0/test_bitmap_index.out
index 3b37588e29..ff0754c309 100644
--- a/regression-test/data/index_p0/test_bitmap_index.out
+++ b/regression-test/data/index_p0/test_bitmap_index.out
@@ -17,21 +17,21 @@ k14 DATETIMEV2(3) Yes false \N NONE
k15 DATETIMEV2(6) Yes false \N NONE
-- !sql --
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index1
k1 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index2
k2 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index3
k3 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index4
k4 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index5
k5 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index6
k6 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index7
k7 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index8
k8 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index9
k9 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index10
k10 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index11
k11 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index12
k12 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index13
k13 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index14
k14 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_dup index15
k15 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index1
k1 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index2
k2 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index3
k3 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index4
k4 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index5
k5 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index6
k6 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index7
k7 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index8
k8 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index9
k9 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index10
k10 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index11
k11 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index12
k12 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index13
k13 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index14
k14 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_dup index15
k15 BITMAP
-- !sql --
1 1 1 1 1 1 2022-05-31
2022-05-31T10:00 1 1 true 2022-05-31
2022-05-31T10:00 2022-05-31T10:00:00.111 2022-05-31T10:00:00.111111
@@ -55,21 +55,21 @@ k15 DATETIMEV2(6) Yes true \N
v1 INT Yes false \N SUM
-- !sql --
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index1
k1 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index2
k2 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index3
k3 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index4
k4 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index5
k5 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index6
k6 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index7
k7 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index8
k8 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index9
k9 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index10
k10 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index11
k11 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index12
k12 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index13
k13 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index14
k14 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_agg index15
k15 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index1
k1 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index2
k2 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index3
k3 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index4
k4 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index5
k5 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index6
k6 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index7
k7 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index8
k8 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index9
k9 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index10
k10 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index11
k11 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index12
k12 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index13
k13 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index14
k14 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_agg index15
k15 BITMAP
-- !sql --
1 1 1 1 1 1 2022-05-31
2022-05-31T10:00 1 1 true 2022-05-31
2022-05-31T10:00 2022-05-31T10:00:00.111 2022-05-31T10:00:00.111111
1
@@ -93,22 +93,22 @@ k15 DATETIMEV2(6) Yes false \N REPLACE
v1 INT Yes false \N REPLACE
-- !sql --
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index1 k1 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index2 k2 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index3 k3 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index4 k4 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index5 k5 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index6 k6 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index7 k7 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index8 k8 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index9 k9 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index10 k10 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index11 k11 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index12 k12 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index13 k13 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index14 k14 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index15 k15 BITMAP
-default_cluster:regression_test_index_p0.test_bitmap_index_unique
index16 v1 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index1 k1 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index2 k2 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index3 k3 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index4 k4 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index5 k5 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index6 k6 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index7 k7 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index8 k8 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index9 k9 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index10 k10 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index11 k11 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index12 k12 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index13 k13 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index14 k14 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index15 k15 BITMAP
+default_cluster:regression_test_index_p0.test_bitmap_index_unique
index16 v1 BITMAP
-- !sql --
1 1 1 1 1 1 2022-05-31
2022-05-31T10:00 1 1 true 2022-05-31
2022-05-31T10:00 2022-05-31T10:00:00.111 2022-05-31T10:00:00.111111
1
diff --git
a/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
b/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
index 538cbb9404..49b5a9da49 100644
--- a/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
+++ b/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out
@@ -3,7 +3,7 @@
a DECIMAL(12,6) No true \N
-- !sql --
-default_cluster:regression_test_index_p0.test_decimal_bitmap_index_multi_page
bitmap_index_multi_page a
BITMAP
+default_cluster:regression_test_index_p0.test_decimal_bitmap_index_multi_page
bitmap_index_multi_page a
BITMAP
-- !sql --
0.000001
diff --git a/regression-test/suites/index_p0/test_index_meta.groovy
b/regression-test/suites/index_p0/test_index_meta.groovy
new file mode 100644
index 0000000000..f31c125220
--- /dev/null
+++ b/regression-test/suites/index_p0/test_index_meta.groovy
@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonOutput
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("index_meta", "p0") {
+ // prepare test table
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout)
+ }
+
+ def tableName = "test_index_meta"
+
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ // create 1 replica table
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `id` INT NULL,
+ `name` STRING NULL,
+ `description` STRING NULL,
+ INDEX idx_id (`id`) USING BITMAP COMMENT 'index for id',
+ INDEX idx_name (`name`) USING INVERTED
PROPERTIES("parser"="none") COMMENT 'index for name'
+ )
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ properties("replication_num" = "1");
+ """
+
+ // set enable_vectorized_engine=true
+ sql """ SET enable_vectorized_engine=true; """
+ def var_result = sql "show variables"
+ logger.info("show variales result: " + var_result )
+
+ // show index of create table
+ def show_result = sql "show index from ${tableName}"
+ logger.info("show index from " + tableName + " result: " + show_result)
+ assertEquals(show_result.size(), 2)
+ assertEquals(show_result[0][2], "idx_id")
+ assertEquals(show_result[0][4], "id")
+ assertEquals(show_result[0][10], "BITMAP")
+ assertEquals(show_result[0][11], "index for id")
+ assertEquals(show_result[0][12], "")
+ assertEquals(show_result[1][2], "idx_name")
+ assertEquals(show_result[1][4], "name")
+ assertEquals(show_result[1][10], "INVERTED")
+ assertEquals(show_result[1][11], "index for name")
+ assertEquals(show_result[1][12], "(\"parser\" = \"none\")")
+
+ // add index on column description
+ sql "create index idx_desc on ${tableName}(description) USING INVERTED
PROPERTIES(\"parser\"=\"standard\") COMMENT 'index for description';"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // show index after add index
+ show_result = sql "show index from ${tableName}"
+ logger.info("show index from " + tableName + " result: " + show_result)
+ assertEquals(show_result.size(), 3)
+ assertEquals(show_result[0][2], "idx_id")
+ assertEquals(show_result[0][4], "id")
+ assertEquals(show_result[0][10], "BITMAP")
+ assertEquals(show_result[0][11], "index for id")
+ assertEquals(show_result[0][12], "")
+ assertEquals(show_result[1][2], "idx_name")
+ assertEquals(show_result[1][4], "name")
+ assertEquals(show_result[1][10], "INVERTED")
+ assertEquals(show_result[1][11], "index for name")
+ assertEquals(show_result[1][12], "(\"parser\" = \"none\")")
+ assertEquals(show_result[2][2], "idx_desc")
+ assertEquals(show_result[2][4], "description")
+ assertEquals(show_result[2][10], "INVERTED")
+ assertEquals(show_result[2][11], "index for description")
+ assertEquals(show_result[2][12], "(\"parser\" = \"standard\")")
+
+ // drop index
+ // add index on column description
+ sql "drop index idx_name on ${tableName}"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ show_result = sql "show index from ${tableName}"
+ logger.info("show index from " + tableName + " result: " + show_result)
+ assertEquals(show_result.size(), 2)
+ assertEquals(show_result[0][2], "idx_id")
+ assertEquals(show_result[0][4], "id")
+ assertEquals(show_result[0][10], "BITMAP")
+ assertEquals(show_result[0][11], "index for id")
+ assertEquals(show_result[0][12], "")
+ assertEquals(show_result[1][2], "idx_desc")
+ assertEquals(show_result[1][4], "description")
+ assertEquals(show_result[1][10], "INVERTED")
+ assertEquals(show_result[1][11], "index for description")
+ assertEquals(show_result[1][12], "(\"parser\" = \"standard\")")
+
+ // add index on column description
+ sql "create index idx_name on ${tableName}(name) USING INVERTED COMMENT
'new index for name';"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // show index after add index
+ show_result = sql "show index from ${tableName}"
+ logger.info("show index from " + tableName + " result: " + show_result)
+ assertEquals(show_result.size(), 3)
+ assertEquals(show_result[0][2], "idx_id")
+ assertEquals(show_result[0][4], "id")
+ assertEquals(show_result[0][10], "BITMAP")
+ assertEquals(show_result[0][11], "index for id")
+ assertEquals(show_result[0][12], "")
+ assertEquals(show_result[1][2], "idx_desc")
+ assertEquals(show_result[1][4], "description")
+ assertEquals(show_result[1][10], "INVERTED")
+ assertEquals(show_result[1][11], "index for description")
+ assertEquals(show_result[1][12], "(\"parser\" = \"standard\")")
+ assertEquals(show_result[2][2], "idx_name")
+ assertEquals(show_result[2][4], "name")
+ assertEquals(show_result[2][10], "INVERTED")
+ assertEquals(show_result[2][11], "new index for name")
+ assertEquals(show_result[2][12], "")
+
+
+ def show_tablets_result = sql "show tablets from ${tableName}"
+ logger.info("show tablets from " + tableName + " result: " +
show_tablets_result)
+ for (j in range(0, show_tablets_result.size())) {
+ String metaUrl = show_tablets_result[j][16]
+ String getMetaCommand = "curl -X GET " + metaUrl
+ def process = getMetaCommand.toString().execute()
+ int code = process.waitFor()
+ String err = IOGroovyMethods.getText(new BufferedReader(new
InputStreamReader(process.getErrorStream())));
+ String out = process.getText()
+ logger.info("get meta process result: code=" + code + ", out=" + out +
", err=" + err)
+ assertEquals(code, 0)
+ def json = parseJson(out.trim())
+ assert json.schema.index instanceof List
+ int i = 0;
+ for (Object index in (List) json.schema.index) {
+ // assertEquals(index.index_id, i);
+ assertEquals(index.index_name, show_result[i][2])
+ assertEquals(index.index_type, show_result[i][10])
+ // assertEquals(index.properties, show_result[j][12]);
+ i++;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
