This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/dev-1.1.2 by this push:
new 978f65a24c [Vectorized][Function] add orthogonal bitmap agg functions
(#10126) (#11333)
978f65a24c is described below
commit 978f65a24c5ac27fa76b6738e7369ad6aab743cd
Author: starocean999 <[email protected]>
AuthorDate: Sat Jul 30 19:22:26 2022 +0800
[Vectorized][Function] add orthogonal bitmap agg functions (#10126) (#11333)
* [Vectorized][Function] add orthogonal bitmap agg functions
save some file about orthogonal bitmap function
add some file to rebase
update functions file
* refactor union_count function
refactor orthogonal union count functions
* remove bool is_variadic
Co-authored-by: zhangstar333
<[email protected]>
---
be/src/exprs/bitmap_function.cpp | 256 +----
be/src/runtime/string_value.h | 1 +
be/src/util/bitmap_intersect.h | 245 +++++
be/src/vec/CMakeLists.txt | 2 +
.../aggregate_function_orthogonal_bitmap.cpp | 99 ++
.../aggregate_function_orthogonal_bitmap.h | 247 +++++
.../aggregate_function_simple_factory.cpp | 2 +
be/test/exprs/bitmap_function_test.cpp | 7 +-
docs/.vuepress/sidebar/en/docs.js | 1009 ++++++++++++++++++++
docs/.vuepress/sidebar/zh-CN.js | 1 +
.../bitmap-functions/intersect_count.md | 57 ++
.../bitmap-functions/intersect_count.md | 56 ++
.../apache/doris/analysis/FunctionCallExpr.java | 13 +-
.../apache/doris/catalog/AggregateFunction.java | 7 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 21 +
15 files changed, 1782 insertions(+), 241 deletions(-)
diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp
index 5e38ab8f79..e45f7244b9 100644
--- a/be/src/exprs/bitmap_function.cpp
+++ b/be/src/exprs/bitmap_function.cpp
@@ -20,141 +20,12 @@
#include "exprs/anyval_util.h"
#include "gutil/strings/numbers.h"
#include "gutil/strings/split.h"
+#include "util/bitmap_intersect.h"
#include "util/bitmap_value.h"
#include "util/string_parser.hpp"
namespace doris {
-namespace detail {
-
-const int DATETIME_PACKED_TIME_BYTE_SIZE = 8;
-const int DATETIME_TYPE_BYTE_SIZE = 4;
-
-const int DECIMAL_BYTE_SIZE = 16;
-
-// get_val start
-template <typename ValType, typename T>
-T get_val(const ValType& x) {
- DCHECK(!x.is_null);
- return x.val;
-}
-
-template <>
-StringValue get_val(const StringVal& x) {
- DCHECK(!x.is_null);
- return StringValue::from_string_val(x);
-}
-
-template <>
-DateTimeValue get_val(const DateTimeVal& x) {
- return DateTimeValue::from_datetime_val(x);
-}
-
-template <>
-DecimalV2Value get_val(const DecimalV2Val& x) {
- return DecimalV2Value::from_decimal_val(x);
-}
-// get_val end
-
-// serialize_size start
-template <typename T>
-int32_t serialize_size(const T& v) {
- return sizeof(T);
-}
-
-template <>
-int32_t serialize_size(const DateTimeValue& v) {
- return DATETIME_PACKED_TIME_BYTE_SIZE + DATETIME_TYPE_BYTE_SIZE;
-}
-
-template <>
-int32_t serialize_size(const DecimalV2Value& v) {
- return DECIMAL_BYTE_SIZE;
-}
-
-template <>
-int32_t serialize_size(const StringValue& v) {
- return v.len + 4;
-}
-// serialize_size end
-
-// write_to start
-template <typename T>
-char* write_to(const T& v, char* dest) {
- size_t type_size = sizeof(T);
- memcpy(dest, &v, type_size);
- dest += type_size;
- return dest;
-}
-
-template <>
-char* write_to(const DateTimeValue& v, char* dest) {
- DateTimeVal value;
- v.to_datetime_val(&value);
- *(int64_t*)dest = value.packed_time;
- dest += DATETIME_PACKED_TIME_BYTE_SIZE;
- *(int*)dest = value.type;
- dest += DATETIME_TYPE_BYTE_SIZE;
- return dest;
-}
-
-template <>
-char* write_to(const DecimalV2Value& v, char* dest) {
- __int128 value = v.value();
- memcpy(dest, &value, DECIMAL_BYTE_SIZE);
- dest += DECIMAL_BYTE_SIZE;
- return dest;
-}
-
-template <>
-char* write_to(const StringValue& v, char* dest) {
- *(int32_t*)dest = v.len;
- dest += 4;
- memcpy(dest, v.ptr, v.len);
- dest += v.len;
- return dest;
-}
-// write_to end
-
-// read_from start
-template <typename T>
-void read_from(const char** src, T* result) {
- size_t type_size = sizeof(T);
- memcpy(result, *src, type_size);
- *src += type_size;
-}
-
-template <>
-void read_from(const char** src, DateTimeValue* result) {
- DateTimeVal value;
- value.is_null = false;
- value.packed_time = *(int64_t*)(*src);
- *src += DATETIME_PACKED_TIME_BYTE_SIZE;
- value.type = *(int*)(*src);
- *src += DATETIME_TYPE_BYTE_SIZE;
- *result = DateTimeValue::from_datetime_val(value);
- ;
-}
-
-template <>
-void read_from(const char** src, DecimalV2Value* result) {
- __int128 v = 0;
- memcpy(&v, *src, DECIMAL_BYTE_SIZE);
- *src += DECIMAL_BYTE_SIZE;
- *result = DecimalV2Value(v);
-}
-
-template <>
-void read_from(const char** src, StringValue* result) {
- int32_t length = *(int32_t*)(*src);
- *src += 4;
- *result = StringValue((char*)*src, length);
- *src += length;
-}
-// read_from end
-
-} // namespace detail
-
static StringVal serialize(FunctionContext* ctx, BitmapValue* value) {
if (!value) {
BitmapValue empty_bitmap;
@@ -168,98 +39,6 @@ static StringVal serialize(FunctionContext* ctx,
BitmapValue* value) {
}
}
-// Calculate the intersection of two or more bitmaps
-// Usage: intersect_count(bitmap_column_to_count, filter_column, filter_values
...)
-// Example: intersect_count(user_id, event, 'A', 'B', 'C'), meaning find the
intersect count of user_id in all A/B/C 3 bitmaps
-// Todo(kks) Use Array type instead of variable arguments
-template <typename T>
-struct BitmapIntersect {
-public:
- BitmapIntersect() {}
-
- explicit BitmapIntersect(const char* src) { deserialize(src); }
-
- void add_key(const T key) {
- BitmapValue empty_bitmap;
- _bitmaps[key] = empty_bitmap;
- }
-
- void update(const T& key, const BitmapValue& bitmap) {
- if (_bitmaps.find(key) != _bitmaps.end()) {
- _bitmaps[key] |= bitmap;
- }
- }
-
- void merge(const BitmapIntersect& other) {
- for (auto& kv : other._bitmaps) {
- if (_bitmaps.find(kv.first) != _bitmaps.end()) {
- _bitmaps[kv.first] |= kv.second;
- } else {
- _bitmaps[kv.first] = kv.second;
- }
- }
- }
-
- // intersection
- BitmapValue intersect() const {
- BitmapValue result;
- auto it = _bitmaps.begin();
- result |= it->second;
- it++;
- for (; it != _bitmaps.end(); it++) {
- result &= it->second;
- }
- return result;
- }
-
- // calculate the intersection for _bitmaps's bitmap values
- int64_t intersect_count() const {
- if (_bitmaps.empty()) {
- return 0;
- }
- return intersect().cardinality();
- }
-
- // the serialize size
- size_t size() {
- size_t size = 4;
- for (auto& kv : _bitmaps) {
- size += detail::serialize_size(kv.first);
- ;
- size += kv.second.getSizeInBytes();
- }
- return size;
- }
-
- //must call size() first
- void serialize(char* dest) {
- char* writer = dest;
- *(int32_t*)writer = _bitmaps.size();
- writer += 4;
- for (auto& kv : _bitmaps) {
- writer = detail::write_to(kv.first, writer);
- kv.second.write(writer);
- writer += kv.second.getSizeInBytes();
- }
- }
-
- void deserialize(const char* src) {
- const char* reader = src;
- int32_t bitmaps_size = *(int32_t*)reader;
- reader += 4;
- for (int32_t i = 0; i < bitmaps_size; i++) {
- T key;
- detail::read_from(&reader, &key);
- BitmapValue bitmap(reader);
- reader += bitmap.getSizeInBytes();
- _bitmaps[key] = bitmap;
- }
- }
-
-private:
- std::map<T, BitmapValue> _bitmaps;
-};
-
void BitmapFunctions::init() {}
void BitmapFunctions::bitmap_init(FunctionContext* ctx, StringVal* dst) {
@@ -403,7 +182,7 @@ StringVal
BitmapFunctions::bitmap_serialize(FunctionContext* ctx, const StringVa
return result;
}
-// This is a init function for intersect_count not for bitmap_intersect.
+// This is a init function for intersect_count not for bitmap_intersect, not
for _orthogonal_bitmap_intersect(bitmap,t,t)
template <typename T, typename ValType>
void BitmapFunctions::bitmap_intersect_init(FunctionContext* ctx, StringVal*
dst) {
dst->is_null = false;
@@ -414,12 +193,14 @@ void
BitmapFunctions::bitmap_intersect_init(FunctionContext* ctx, StringVal* dst
for (int i = 2; i < ctx->get_num_constant_args(); ++i) {
DCHECK(ctx->is_arg_constant(i));
ValType* arg = reinterpret_cast<ValType*>(ctx->get_constant_arg(i));
- intersect->add_key(detail::get_val<ValType, T>(*arg));
+ intersect->add_key(detail::Helper::get_val<ValType, T>(*arg));
}
dst->ptr = (uint8_t*)intersect;
}
+// This is a update function for
intersect_count/ORTHOGONAL_BITMAP_INTERSECT_COUNT/ORTHOGONAL_BITMAP_INTERSECT(bitmap,t,t)
+// not for bitmap_intersect(Bitmap)
template <typename T, typename ValType>
void BitmapFunctions::bitmap_intersect_update(FunctionContext* ctx, const
StringVal& src,
const ValType& key, int num_key,
const ValType* keys,
@@ -427,13 +208,14 @@ void
BitmapFunctions::bitmap_intersect_update(FunctionContext* ctx, const String
auto* dst_bitmap = reinterpret_cast<BitmapIntersect<T>*>(dst->ptr);
// zero size means the src input is a agg object
if (src.len == 0) {
- dst_bitmap->update(detail::get_val<ValType, T>(key),
+ dst_bitmap->update(detail::Helper::get_val<ValType, T>(key),
*reinterpret_cast<BitmapValue*>(src.ptr));
} else {
- dst_bitmap->update(detail::get_val<ValType, T>(key),
BitmapValue((char*)src.ptr));
+ dst_bitmap->update(detail::Helper::get_val<ValType, T>(key),
BitmapValue((char*)src.ptr));
}
}
+//only for intersect_count(bitmap,t,t)
template <typename T>
void BitmapFunctions::bitmap_intersect_merge(FunctionContext* ctx, const
StringVal& src,
const StringVal* dst) {
@@ -441,6 +223,7 @@ void
BitmapFunctions::bitmap_intersect_merge(FunctionContext* ctx, const StringV
dst_bitmap->merge(BitmapIntersect<T>((char*)src.ptr));
}
+//only for intersect_count(bitmap,t,t)
template <typename T>
StringVal BitmapFunctions::bitmap_intersect_serialize(FunctionContext* ctx,
const StringVal& src) {
auto* src_bitmap = reinterpret_cast<BitmapIntersect<T>*>(src.ptr);
@@ -450,6 +233,7 @@ StringVal
BitmapFunctions::bitmap_intersect_serialize(FunctionContext* ctx, cons
return result;
}
+//only for intersect_count(bitmap,t,t)
template <typename T>
BigIntVal BitmapFunctions::bitmap_intersect_finalize(FunctionContext* ctx,
const StringVal& src) {
auto* src_bitmap = reinterpret_cast<BitmapIntersect<T>*>(src.ptr);
@@ -928,13 +712,15 @@ StringVal
BitmapFunctions::bitmap_subset_limit(FunctionContext* ctx, const Strin
return serialize(ctx, &ret_bitmap);
}
-
+// init ORTHOGONAL_BITMAP_UNION_COUNT(bitmap)
+// update bitmap_union()
void BitmapFunctions::orthogonal_bitmap_union_count_init(FunctionContext* ctx,
StringVal* dst) {
dst->is_null = false;
dst->len = sizeof(BitmapValue);
dst->ptr = (uint8_t*)new BitmapValue();
}
+// serialize for ORTHOGONAL_BITMAP_UNION_COUNT(bitmap)
StringVal BitmapFunctions::orthogonal_bitmap_count_serialize(FunctionContext*
ctx,
const StringVal&
src) {
if (src.is_null) {
@@ -950,7 +736,7 @@ StringVal
BitmapFunctions::orthogonal_bitmap_count_serialize(FunctionContext* ct
return result;
}
-// This is a init function for bitmap_intersect.
+// This is a init function for orthogonal_bitmap_intersect(bitmap,t,t).
template <typename T, typename ValType>
void BitmapFunctions::orthogonal_bitmap_intersect_init(FunctionContext* ctx,
StringVal* dst) {
// constant args start from index 2
@@ -961,7 +747,7 @@ void
BitmapFunctions::orthogonal_bitmap_intersect_init(FunctionContext* ctx, Str
for (int i = 2; i < ctx->get_num_constant_args(); ++i) {
ValType* arg =
reinterpret_cast<ValType*>(ctx->get_constant_arg(i));
- intersect->add_key(detail::get_val<ValType, T>(*arg));
+ intersect->add_key(detail::Helper::get_val<ValType, T>(*arg));
}
dst->ptr = (uint8_t*)intersect;
@@ -972,7 +758,7 @@ void
BitmapFunctions::orthogonal_bitmap_intersect_init(FunctionContext* ctx, Str
}
}
-// This is a init function for intersect_count.
+// This is a init function for orthogonal_bitmap_intersect_count(bitmap,t,t).
template <typename T, typename ValType>
void BitmapFunctions::orthogonal_bitmap_intersect_count_init(FunctionContext*
ctx, StringVal* dst) {
if (ctx->get_num_constant_args() > 1) {
@@ -983,7 +769,7 @@ void
BitmapFunctions::orthogonal_bitmap_intersect_count_init(FunctionContext* ct
// constant args start from index 2
for (int i = 2; i < ctx->get_num_constant_args(); ++i) {
ValType* arg =
reinterpret_cast<ValType*>(ctx->get_constant_arg(i));
- intersect->add_key(detail::get_val<ValType, T>(*arg));
+ intersect->add_key(detail::Helper::get_val<ValType, T>(*arg));
}
dst->ptr = (uint8_t*)intersect;
@@ -995,6 +781,9 @@ void
BitmapFunctions::orthogonal_bitmap_intersect_count_init(FunctionContext* ct
}
}
+// This is a serialize function for orthogonal_bitmap_intersect(bitmap,t,t).
+// merge is ths simple bitmap_union() function LINE(80);
+// finalize is the bitmap_serialize() function LINE(173)
template <typename T>
StringVal
BitmapFunctions::orthogonal_bitmap_intersect_serialize(FunctionContext* ctx,
const
StringVal& src) {
@@ -1014,6 +803,8 @@ BigIntVal
BitmapFunctions::orthogonal_bitmap_intersect_finalize(FunctionContext*
return result;
}
+// This is a merge function for orthogonal_bitmap_intersect_count(bitmap,t,t).
+// and merge for ORTHOGONAL_BITMAP_UNION_COUNT(bitmap)
void BitmapFunctions::orthogonal_bitmap_count_merge(FunctionContext* context,
const StringVal& src,
StringVal* dst) {
if (dst->len != sizeof(int64_t)) {
@@ -1027,6 +818,8 @@ void
BitmapFunctions::orthogonal_bitmap_count_merge(FunctionContext* context, co
*(int64_t*)dst->ptr += *(int64_t*)src.ptr;
}
+// This is a finalize function for
orthogonal_bitmap_intersect_count(bitmap,t,t).
+// finalize for ORTHOGONAL_BITMAP_UNION_COUNT(bitmap)
BigIntVal BitmapFunctions::orthogonal_bitmap_count_finalize(FunctionContext*
context,
const StringVal&
src) {
auto* pval = reinterpret_cast<int64_t*>(src.ptr);
@@ -1035,6 +828,7 @@ BigIntVal
BitmapFunctions::orthogonal_bitmap_count_finalize(FunctionContext* con
return result;
}
+// This is a serialize function for
orthogonal_bitmap_intersect_count(bitmap,t,t).
template <typename T>
StringVal
BitmapFunctions::orthogonal_bitmap_intersect_count_serialize(FunctionContext*
ctx,
const
StringVal& src) {
diff --git a/be/src/runtime/string_value.h b/be/src/runtime/string_value.h
index cdf33ab57e..9ba7d4fc34 100644
--- a/be/src/runtime/string_value.h
+++ b/be/src/runtime/string_value.h
@@ -89,6 +89,7 @@ struct StringValue {
StringValue(char* ptr, int len) : ptr(ptr), len(len) {}
StringValue(const char* ptr, int len) : ptr(const_cast<char*>(ptr)),
len(len) {}
StringValue() : ptr(nullptr), len(0) {}
+ StringValue(const StringRef& str) : ptr(const_cast<char*>(str.data)),
len(str.size) {}
/// Construct a StringValue from 's'. 's' must be valid for as long as
/// this object is valid.
diff --git a/be/src/util/bitmap_intersect.h b/be/src/util/bitmap_intersect.h
new file mode 100644
index 0000000000..dcda6ae5a5
--- /dev/null
+++ b/be/src/util/bitmap_intersect.h
@@ -0,0 +1,245 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+#include "runtime/string_value.h"
+#include "udf/udf.h"
+#include "util/bitmap_value.h"
+
+namespace doris {
+
+namespace detail {
+class Helper {
+public:
+ static const int DATETIME_PACKED_TIME_BYTE_SIZE = 8;
+ static const int DATETIME_TYPE_BYTE_SIZE = 4;
+ static const int DECIMAL_BYTE_SIZE = 16;
+
+ // get_val start
+ template <typename ValType, typename T>
+ static T get_val(const ValType& x) {
+ DCHECK(!x.is_null);
+ return x.val;
+ }
+
+ // serialize_size start
+ template <typename T>
+ static int32_t serialize_size(const T& v) {
+ return sizeof(T);
+ }
+
+ // write_to start
+ template <typename T>
+ static char* write_to(const T& v, char* dest) {
+ size_t type_size = sizeof(T);
+ memcpy(dest, &v, type_size);
+ dest += type_size;
+ return dest;
+ }
+
+ // read_from start
+ template <typename T>
+ static void read_from(const char** src, T* result) {
+ size_t type_size = sizeof(T);
+ memcpy(result, *src, type_size);
+ *src += type_size;
+ }
+};
+
+template <>
+inline StringValue Helper::get_val<StringVal>(const StringVal& x) {
+ DCHECK(!x.is_null);
+ return StringValue::from_string_val(x);
+}
+
+template <>
+inline DateTimeValue Helper::get_val<DateTimeVal>(const DateTimeVal& x) {
+ return DateTimeValue::from_datetime_val(x);
+}
+
+template <>
+inline DecimalV2Value Helper::get_val<DecimalV2Val>(const DecimalV2Val& x) {
+ return DecimalV2Value::from_decimal_val(x);
+}
+// get_val end
+
+template <>
+inline char* Helper::write_to<DateTimeValue>(const DateTimeValue& v, char*
dest) {
+ DateTimeVal value;
+ v.to_datetime_val(&value);
+ *(int64_t*)dest = value.packed_time;
+ dest += DATETIME_PACKED_TIME_BYTE_SIZE;
+ *(int*)dest = value.type;
+ dest += DATETIME_TYPE_BYTE_SIZE;
+ return dest;
+}
+
+template <>
+inline char* Helper::write_to<DecimalV2Value>(const DecimalV2Value& v, char*
dest) {
+ __int128 value = v.value();
+ memcpy(dest, &value, DECIMAL_BYTE_SIZE);
+ dest += DECIMAL_BYTE_SIZE;
+ return dest;
+}
+
+template <>
+inline char* Helper::write_to<StringValue>(const StringValue& v, char* dest) {
+ *(int32_t*)dest = v.len;
+ dest += 4;
+ memcpy(dest, v.ptr, v.len);
+ dest += v.len;
+ return dest;
+}
+// write_to end
+
+template <>
+inline int32_t Helper::serialize_size<DateTimeValue>(const DateTimeValue& v) {
+ return Helper::DATETIME_PACKED_TIME_BYTE_SIZE +
Helper::DATETIME_TYPE_BYTE_SIZE;
+}
+
+template <>
+inline int32_t Helper::serialize_size<DecimalV2Value>(const DecimalV2Value& v)
{
+ return Helper::DECIMAL_BYTE_SIZE;
+}
+
+template <>
+inline int32_t Helper::serialize_size<StringValue>(const StringValue& v) {
+ return v.len + 4;
+}
+// serialize_size end
+
+template <>
+inline void Helper::read_from<DateTimeValue>(const char** src, DateTimeValue*
result) {
+ DateTimeVal value;
+ value.is_null = false;
+ value.packed_time = *(int64_t*)(*src);
+ *src += DATETIME_PACKED_TIME_BYTE_SIZE;
+ value.type = *(int*)(*src);
+ *src += DATETIME_TYPE_BYTE_SIZE;
+ *result = DateTimeValue::from_datetime_val(value);
+}
+
+template <>
+inline void Helper::read_from<DecimalV2Value>(const char** src,
DecimalV2Value* result) {
+ __int128 v = 0;
+ memcpy(&v, *src, DECIMAL_BYTE_SIZE);
+ *src += DECIMAL_BYTE_SIZE;
+ *result = DecimalV2Value(v);
+}
+
+template <>
+inline void Helper::read_from<StringValue>(const char** src, StringValue*
result) {
+ int32_t length = *(int32_t*)(*src);
+ *src += 4;
+ *result = StringValue((char*)*src, length);
+ *src += length;
+}
+// read_from end
+
+} // namespace detail
+
+// Calculate the intersection of two or more bitmaps
+// Usage: intersect_count(bitmap_column_to_count, filter_column, filter_values
...)
+// Example: intersect_count(user_id, event, 'A', 'B', 'C'), meaning find the
intersect count of user_id in all A/B/C 3 bitmaps
+// Todo(kks) Use Array type instead of variable arguments
+template <typename T>
+struct BitmapIntersect {
+public:
+ BitmapIntersect() = default;
+
+ explicit BitmapIntersect(const char* src) { deserialize(src); }
+
+ void add_key(const T key) {
+ BitmapValue empty_bitmap;
+ _bitmaps[key] = empty_bitmap;
+ }
+
+ void update(const T& key, const BitmapValue& bitmap) {
+ if (_bitmaps.find(key) != _bitmaps.end()) {
+ _bitmaps[key] |= bitmap;
+ }
+ }
+
+ void merge(const BitmapIntersect& other) {
+ for (auto& kv : other._bitmaps) {
+ if (_bitmaps.find(kv.first) != _bitmaps.end()) {
+ _bitmaps[kv.first] |= kv.second;
+ } else {
+ _bitmaps[kv.first] = kv.second;
+ }
+ }
+ }
+
+ // intersection
+ BitmapValue intersect() const {
+ BitmapValue result;
+ auto it = _bitmaps.begin();
+ result |= it->second;
+ it++;
+ for (; it != _bitmaps.end(); it++) {
+ result &= it->second;
+ }
+ return result;
+ }
+
+ // calculate the intersection for _bitmaps's bitmap values
+ int64_t intersect_count() const {
+ if (_bitmaps.empty()) {
+ return 0;
+ }
+ return intersect().cardinality();
+ }
+
+ // the serialize size
+ size_t size() {
+ size_t size = 4;
+ for (auto& kv : _bitmaps) {
+ size += detail::Helper::serialize_size(kv.first);
+ size += kv.second.getSizeInBytes();
+ }
+ return size;
+ }
+
+ //must call size() first
+ void serialize(char* dest) {
+ char* writer = dest;
+ *(int32_t*)writer = _bitmaps.size();
+ writer += 4;
+ for (auto& kv : _bitmaps) {
+ writer = detail::Helper::write_to(kv.first, writer);
+ kv.second.write(writer);
+ writer += kv.second.getSizeInBytes();
+ }
+ }
+
+ void deserialize(const char* src) {
+ const char* reader = src;
+ int32_t bitmaps_size = *(int32_t*)reader;
+ reader += 4;
+ for (int32_t i = 0; i < bitmaps_size; i++) {
+ T key;
+ detail::Helper::read_from(&reader, &key);
+ BitmapValue bitmap(reader);
+ reader += bitmap.getSizeInBytes();
+ _bitmaps[key] = bitmap;
+ }
+ }
+
+private:
+ std::map<T, BitmapValue> _bitmaps;
+};
+
+} // namespace doris
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index 5a81b42ccd..3b18357086 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -39,6 +39,8 @@ set(VEC_FILES
aggregate_functions/aggregate_function_group_concat.cpp
aggregate_functions/aggregate_function_percentile_approx.cpp
aggregate_functions/aggregate_function_simple_factory.cpp
+ aggregate_functions/aggregate_function_java_udaf.h
+ aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
columns/collator.cpp
columns/column.cpp
columns/column_const.cpp
diff --git
a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
new file mode 100644
index 0000000000..470a6c8388
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.cpp
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h"
+
+#include <memory>
+
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/aggregate_functions/helpers.h"
+#include "vec/data_types/data_type_string.h"
+
+namespace doris::vectorized {
+
+template <template <typename> class Impl>
+AggregateFunctionPtr create_aggregate_function_orthogonal(const std::string&
name,
+ const DataTypes&
argument_types,
+ const Array& params,
+ const bool
result_is_nullable) {
+ if (argument_types.empty()) {
+ LOG(WARNING) << "Incorrect number of arguments for aggregate function
" << name;
+ return nullptr;
+ } else if (argument_types.size() == 1) {
+ // only used at AGGREGATE (merge finalize) for variadic function
+ // and for orthogonal_bitmap_union_count function
+ return
std::make_shared<AggFunctionOrthBitmapFunc<Impl<StringValue>>>(argument_types);
+ } else {
+ const IDataType& argument_type = *argument_types[1];
+ AggregateFunctionPtr
res(create_with_numeric_type<AggFunctionOrthBitmapFunc, Impl>(
+ argument_type, argument_types));
+
+ WhichDataType which(argument_type);
+
+ if (res) {
+ return res;
+ } else if (which.is_string_or_fixed_string()) {
+ return
std::make_shared<AggFunctionOrthBitmapFunc<Impl<StringValue>>>(argument_types);
+ }
+ LOG(WARNING) << "Incorrect Type " << argument_type.get_name()
+ << " of arguments for aggregate function " << name;
+ return nullptr;
+ }
+}
+
+AggregateFunctionPtr create_aggregate_function_orthogonal_bitmap_intersect(
+ const std::string& name, const DataTypes& argument_types, const Array&
parameters,
+ bool result_is_nullable) {
+ return create_aggregate_function_orthogonal<AggOrthBitMapIntersect>(
+ name, argument_types, parameters, result_is_nullable);
+}
+
+AggregateFunctionPtr
create_aggregate_function_orthogonal_bitmap_intersect_count(
+ const std::string& name, const DataTypes& argument_types, const Array&
parameters,
+ bool result_is_nullable) {
+ return create_aggregate_function_orthogonal<AggOrthBitMapIntersectCount>(
+ name, argument_types, parameters, result_is_nullable);
+}
+
+AggregateFunctionPtr create_aggregate_function_intersect_count(const
std::string& name,
+ const
DataTypes& argument_types,
+ const Array&
parameters,
+ bool
result_is_nullable) {
+ return create_aggregate_function_orthogonal<AggIntersectCount>(name,
argument_types, parameters,
+
result_is_nullable);
+}
+
+AggregateFunctionPtr create_aggregate_function_orthogonal_bitmap_union_count(
+ const std::string& name, const DataTypes& argument_types, const Array&
parameters,
+ const bool result_is_nullable) {
+ return create_aggregate_function_orthogonal<OrthBitmapUnionCountData>(
+ name, argument_types, parameters, result_is_nullable);
+}
+
+void
register_aggregate_function_orthogonal_bitmap(AggregateFunctionSimpleFactory&
factory) {
+ factory.register_function("orthogonal_bitmap_intersect",
+
create_aggregate_function_orthogonal_bitmap_intersect);
+
+ factory.register_function("orthogonal_bitmap_intersect_count",
+
create_aggregate_function_orthogonal_bitmap_intersect_count);
+
+ factory.register_function("orthogonal_bitmap_union_count",
+
create_aggregate_function_orthogonal_bitmap_union_count);
+
+ factory.register_function("intersect_count",
create_aggregate_function_intersect_count);
+}
+} // namespace doris::vectorized
\ No newline at end of file
diff --git
a/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
new file mode 100644
index 0000000000..4f1fb69ec4
--- /dev/null
+++ b/be/src/vec/aggregate_functions/aggregate_function_orthogonal_bitmap.h
@@ -0,0 +1,247 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/bitmap_function.h"
+#include "util/bitmap_intersect.h"
+#include "util/bitmap_value.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_complex.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_vector.h"
+#include "vec/common/assert_cast.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/io/io_helper.h"
+
+namespace doris::vectorized {
+
+template <typename T>
+struct AggOrthBitmapBaseData {
+public:
+ using ColVecData = std::conditional_t<IsNumber<T>, ColumnVector<T>,
ColumnString>;
+
+ void add(const IColumn** columns, size_t row_num) {
+ const auto& bitmap_col = static_cast<const ColumnBitmap&>(*columns[0]);
+ const auto& data_col = static_cast<const ColVecData&>(*columns[1]);
+ const auto& bitmap_value = bitmap_col.get_element(row_num);
+
+ if constexpr (IsNumber<T>) {
+ bitmap.update(data_col.get_element(row_num), bitmap_value);
+ } else {
+ bitmap.update(StringValue(data_col.get_data_at(row_num)),
bitmap_value);
+ }
+ }
+
+ void init_add_key(const IColumn** columns, size_t row_num, int
argument_size) {
+ if (first_init) {
+ DCHECK(argument_size > 1);
+ for (int idx = 2; idx < argument_size; ++idx) {
+ const auto& col = static_cast<const
ColVecData&>(*columns[idx]);
+ if constexpr (IsNumber<T>) {
+ bitmap.add_key(col.get_element(row_num));
+ } else {
+ bitmap.add_key(StringValue(col.get_data_at(row_num)));
+ }
+ }
+ first_init = false;
+ }
+ }
+
+protected:
+ doris::BitmapIntersect<T> bitmap;
+ bool first_init = true;
+};
+
+template <typename T>
+struct AggOrthBitMapIntersect : public AggOrthBitmapBaseData<T> {
+public:
+ static constexpr auto name = "orthogonal_bitmap_intersect";
+
+ static DataTypePtr get_return_type() { return
std::make_shared<DataTypeBitMap>(); }
+
+ void merge(const AggOrthBitMapIntersect& rhs) {
+ if (rhs.first_init) {
+ return;
+ }
+ result |= rhs.result;
+ }
+
+ void write(BufferWritable& buf) {
+ write_binary(AggOrthBitmapBaseData<T>::first_init, buf);
+ result = AggOrthBitmapBaseData<T>::bitmap.intersect();
+ DataTypeBitMap::serialize_as_stream(result, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ read_binary(AggOrthBitmapBaseData<T>::first_init, buf);
+ DataTypeBitMap::deserialize_as_stream(result, buf);
+ }
+
+ void get(IColumn& to) const {
+ auto& column = static_cast<ColumnBitmap&>(to);
+ column.get_data().emplace_back(result);
+ }
+
+private:
+ BitmapValue result;
+};
+
+template <typename T>
+struct AggIntersectCount : public AggOrthBitmapBaseData<T> {
+public:
+ static constexpr auto name = "intersect_count";
+
+ static DataTypePtr get_return_type() { return
std::make_shared<DataTypeInt64>(); }
+
+ void merge(const AggIntersectCount& rhs) {
+ if (rhs.first_init) {
+ return;
+ }
+ AggOrthBitmapBaseData<T>::bitmap.merge(rhs.bitmap);
+ }
+
+ void write(BufferWritable& buf) {
+ write_binary(AggOrthBitmapBaseData<T>::first_init, buf);
+ std::string data;
+ data.resize(AggOrthBitmapBaseData<T>::bitmap.size());
+ AggOrthBitmapBaseData<T>::bitmap.serialize(data.data());
+ write_binary(data, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ read_binary(AggOrthBitmapBaseData<T>::first_init, buf);
+ std::string data;
+ read_binary(data, buf);
+ AggOrthBitmapBaseData<T>::bitmap.deserialize(data.data());
+ }
+
+ void get(IColumn& to) const {
+ auto& column = static_cast<ColumnVector<Int64>&>(to);
+
column.get_data().emplace_back(AggOrthBitmapBaseData<T>::bitmap.intersect_count());
+ }
+};
+
+template <typename T>
+struct AggOrthBitMapIntersectCount : public AggOrthBitmapBaseData<T> {
+public:
+ static constexpr auto name = "orthogonal_bitmap_intersect_count";
+
+ static DataTypePtr get_return_type() { return
std::make_shared<DataTypeInt64>(); }
+
+ void merge(const AggOrthBitMapIntersectCount& rhs) {
+ if (rhs.first_init) {
+ return;
+ }
+ result += rhs.result;
+ }
+
+ void write(BufferWritable& buf) {
+ write_binary(AggOrthBitmapBaseData<T>::first_init, buf);
+ result = AggOrthBitmapBaseData<T>::bitmap.intersect_count();
+ write_binary(result, buf);
+ }
+
+ void read(BufferReadable& buf) {
+ read_binary(AggOrthBitmapBaseData<T>::first_init, buf);
+ read_binary(result, buf);
+ }
+
+ void get(IColumn& to) const {
+ auto& column = static_cast<ColumnVector<Int64>&>(to);
+ column.get_data().emplace_back(result);
+ }
+
+private:
+ Int64 result = 0;
+};
+
+template <typename T>
+struct OrthBitmapUnionCountData {
+ static constexpr auto name = "orthogonal_bitmap_union_count";
+
+ static DataTypePtr get_return_type() { return
std::make_shared<DataTypeInt64>(); }
+ // Here no need doing anything, so only given an function declaration
+ void init_add_key(const IColumn** columns, size_t row_num, int
argument_size) {}
+
+ void add(const IColumn** columns, size_t row_num) {
+ const auto& column = static_cast<const ColumnBitmap&>(*columns[0]);
+ value |= column.get_data()[row_num];
+ }
+ void merge(const OrthBitmapUnionCountData& rhs) { result += rhs.result; }
+
+ void write(BufferWritable& buf) {
+ result = value.cardinality();
+ write_binary(result, buf);
+ }
+
+ void read(BufferReadable& buf) { read_binary(result, buf); }
+
+ void get(IColumn& to) const {
+ auto& column = static_cast<ColumnVector<Int64>&>(to);
+ column.get_data().emplace_back(result ? result : value.cardinality());
+ }
+
+private:
+ BitmapValue value;
+ int64_t result = 0;
+};
+
+template <typename Impl>
+class AggFunctionOrthBitmapFunc final
+ : public IAggregateFunctionDataHelper<Impl,
AggFunctionOrthBitmapFunc<Impl>> {
+public:
+ String get_name() const override { return Impl::name; }
+
+ AggFunctionOrthBitmapFunc(const DataTypes& argument_types_)
+ : IAggregateFunctionDataHelper<Impl,
AggFunctionOrthBitmapFunc<Impl>>(argument_types_,
+
{}),
+ _argument_size(argument_types_.size()) {}
+
+ DataTypePtr get_return_type() const override { return
Impl::get_return_type(); }
+
+ void add(AggregateDataPtr __restrict place, const IColumn** columns,
size_t row_num,
+ Arena*) const override {
+ this->data(place).init_add_key(columns, row_num, _argument_size);
+ this->data(place).add(columns, row_num);
+ }
+
+ void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs,
+ Arena*) const override {
+ this->data(place).merge(this->data(rhs));
+ }
+
+ void serialize(ConstAggregateDataPtr __restrict place, BufferWritable&
buf) const override {
+ this->data(const_cast<AggregateDataPtr>(place)).write(buf);
+ }
+
+ void deserialize(AggregateDataPtr __restrict place, BufferReadable& buf,
+ Arena*) const override {
+ this->data(place).read(buf);
+ }
+
+ void insert_result_into(ConstAggregateDataPtr __restrict place, IColumn&
to) const override {
+ this->data(place).get(to);
+ }
+
+private:
+ int _argument_size;
+};
+} // namespace doris::vectorized
\ No newline at end of file
diff --git
a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
index 6315fd6600..3fa99b908d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_simple_factory.cpp
@@ -46,6 +46,7 @@ void
register_aggregate_function_group_concat(AggregateFunctionSimpleFactory& fa
void register_aggregate_function_percentile(AggregateFunctionSimpleFactory&
factory);
void register_aggregate_function_window_funnel(AggregateFunctionSimpleFactory&
factory);
void
register_aggregate_function_percentile_approx(AggregateFunctionSimpleFactory&
factory);
+void
register_aggregate_function_orthogonal_bitmap(AggregateFunctionSimpleFactory&
factory);
AggregateFunctionSimpleFactory& AggregateFunctionSimpleFactory::instance() {
static std::once_flag oc;
static AggregateFunctionSimpleFactory instance;
@@ -67,6 +68,7 @@ AggregateFunctionSimpleFactory&
AggregateFunctionSimpleFactory::instance() {
register_aggregate_function_percentile(instance);
register_aggregate_function_percentile_approx(instance);
register_aggregate_function_window_funnel(instance);
+ register_aggregate_function_orthogonal_bitmap(instance);
// if you only register function with no nullable, and wants to add
nullable automatically, you should place function above this line
register_aggregate_function_combinator_null(instance);
diff --git a/be/test/exprs/bitmap_function_test.cpp
b/be/test/exprs/bitmap_function_test.cpp
index 334ba7aa73..34c3155551 100644
--- a/be/test/exprs/bitmap_function_test.cpp
+++ b/be/test/exprs/bitmap_function_test.cpp
@@ -27,6 +27,7 @@
#include "exprs/aggregate_functions.h"
#include "exprs/anyval_util.h"
#include "testutil/function_utils.h"
+#include "util/bitmap_intersect.h"
#include "util/bitmap_value.h"
#include "util/logging.h"
@@ -266,10 +267,10 @@ void test_bitmap_intersect(FunctionContext* ctx, ValType
key1, ValType key2) {
BitmapIntersect<ValueType> intersect2;
for (size_t i = 2; i < const_vals.size(); i++) {
ValType* arg = reinterpret_cast<ValType*>(const_vals[i]);
- intersect2.add_key(detail::get_val<ValType, ValueType>(*arg));
+ intersect2.add_key(detail::Helper::get_val<ValType, ValueType>(*arg));
}
- intersect2.update(detail::get_val<ValType, ValueType>(key1), bitmap1);
- intersect2.update(detail::get_val<ValType, ValueType>(key2), bitmap2);
+ intersect2.update(detail::Helper::get_val<ValType, ValueType>(key1),
bitmap1);
+ intersect2.update(detail::Helper::get_val<ValType, ValueType>(key2),
bitmap2);
StringVal expected = convert_bitmap_intersect_to_string(ctx, intersect2);
ASSERT_EQ(expected, intersect1);
diff --git a/docs/.vuepress/sidebar/en/docs.js
b/docs/.vuepress/sidebar/en/docs.js
new file mode 100644
index 0000000000..f7ad6c58df
--- /dev/null
+++ b/docs/.vuepress/sidebar/en/docs.js
@@ -0,0 +1,1009 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = [
+ {
+ title: "Getting Started",
+ directoryPath: "get-starting/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "get-starting"
+ ],
+ },
+ {
+ title: "Doris Architecture",
+ directoryPath: "summary/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "basic-summary",
+ "system-architecture"
+ ],
+ },
+ {
+ title: "Install And Deploy",
+ directoryPath: "install/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "install-deploy",
+ {
+ title: "Compile",
+ directoryPath: "source-install/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "compilation",
+ "compilation-with-ldb-toolchain",
+ "compilation-arm"
+ ],
+ sidebarDepth: 2,
+ },
+ ]
+ },
+ {
+ title: "Table Design",
+ directoryPath: "data-table/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "data-model",
+ "data-partition",
+ "basic-usage",
+ "advance-usage",
+ "hit-the-rollup",
+ "best-practice",
+ {
+ title: "Index",
+ directoryPath: "index/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "bloomfilter",
+ "prefix-index",
+ "bitmap-index"
+ ],
+ },
+ ],
+ },
+ {
+ title: "Data Operation",
+ directoryPath: "data-operate/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Import",
+ directoryPath: "import/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "load-manual",
+ {
+ title: "Import Scenes",
+ directoryPath: "import-scenes/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "local-file-load",
+ "external-storage-load",
+ "kafka-load",
+ "external-table-load",
+ "jdbc-load",
+ "load-atomicity",
+ "load-data-convert",
+ "load-strict-mode",
+ ],
+ },
+ {
+ title: "Import Way",
+ directoryPath: "import-way/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "binlog-load-manual",
+ "broker-load-manual",
+ "routine-load-manual",
+ "spark-load-manual",
+ "stream-load-manual",
+ "s3-load-manual",
+ "insert-into-manual",
+ "load-json-format",
+ ],
+ },
+ ],
+ },
+ {
+ title: "Export",
+ directoryPath: "export/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "export-manual",
+ "outfile",
+ "export_with_mysql_dump",
+ ],
+ },
+ {
+ title: "Update and Delete",
+ directoryPath: "update-delete/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "update",
+ "sequence-column-manual",
+ "delete-manual",
+ "batch-delete-manual"
+ ],
+ },
+ ],
+ },
+ {
+ title: "Advanced Usage",
+ directoryPath: "advanced/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "materialized-view",
+ {
+ title: "Alter Table",
+ directoryPath: "alter-table/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "schema-change",
+ "replace-table"
+ ],
+ },
+ {
+ title: "Doris Partition",
+ directoryPath: "partition/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "dynamic-partition",
+ "table-temp-partition"
+ ],
+ },
+ {
+ title: "Join Optimization",
+ directoryPath: "join-optimization/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "bucket-shuffle-join",
+ "colocation-join",
+ "runtime-filter",
+ "doris-join-optimization"
+ ],
+ },
+ {
+ title: "Date Cache",
+ directoryPath: "cache/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "partition-cache"
+ ],
+ },
+ "vectorized-execution-engine",
+ "broker",
+ "resource",
+ "orthogonal-bitmap-manual",
+ "using-hll",
+ "variables",
+ "time-zone",
+ "small-file-mgr",
+ {
+ title: "Best Practice",
+ directoryPath: "best-practice/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "query-analysis",
+ "import-analysis",
+ "debug-log"
+ ]
+ }
+ ],
+ },
+ {
+ title: "Ecosystem",
+ directoryPath: "ecosystem/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Expansion table",
+ directoryPath: "external-table/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "doris-on-es",
+ "odbc-of-doris",
+ "hive-of-doris",
+ "iceberg-of-doris",
+ "hudi-external-table"
+ ],
+ },
+ "audit-plugin",
+ "flink-doris-connector",
+ "spark-doris-connector",
+ "datax",
+ "logstash",
+ {
+ title: "Doris Manager",
+ directoryPath: "doris-manager/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "compiling-deploying",
+ "initializing",
+ "cluster-managenent",
+ "space-list",
+ "space-management",
+ "system-settings"
+ ],
+ },
+ {
+ title: "SeaTunnel",
+ directoryPath: "seatunnel/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "flink-sink",
+ "spark-sink"
+ ],
+ },
+ {
+ title: "UDF",
+ directoryPath: "udf/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "native-user-defined-function",
+ "remote-user-defined-function",
+ "contribute-udf",
+ "java-user-defined-function"
+ ],
+ },
+ ],
+ },
+ {
+ title: "SQL Manual",
+ directoryPath: "sql-manual/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "SQL Functions",
+ directoryPath: "sql-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Date Functions",
+ directoryPath: "date-time-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "convert_tz",
+ "curdate",
+ "current_timestamp",
+ "curtime",
+ "date_add",
+ "date_format",
+ "date_sub",
+ "datediff",
+ "day",
+ "dayname",
+ "dayofmonth",
+ "dayofweek",
+ "dayofyear",
+ "from_days",
+ "from_unixtime",
+ "hour",
+ "makedate",
+ "minute",
+ "month",
+ "monthname",
+ "now",
+ "second",
+ "str_to_date",
+ "time_round",
+ "timediff",
+ "timestampadd",
+ "timestampdiff",
+ "to_days",
+ "unix_timestamp",
+ "utc_timestamp",
+ "week",
+ "weekday",
+ "weekofyear",
+ "year",
+ "yearweek",
+ ],
+ },
+ {
+ title: "GIS Functions",
+ directoryPath: "spatial-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "st_astext",
+ "st_circle",
+ "st_contains",
+ "st_distance_sphere",
+ "st_geometryfromtext",
+ "st_linefromtext",
+ "st_point",
+ "st_polygon",
+ "st_x",
+ "st_y",
+ ],
+ },
+ {
+ title: "String Functions",
+ directoryPath: "string-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "append_trailing_char_if_absent",
+ "ascii",
+ "bit_length",
+ "char_length",
+ "concat",
+ "concat_ws",
+ "ends_with",
+ "find_in_set",
+ "hex",
+ "instr",
+ "lcase",
+ "left",
+ "length",
+ "locate",
+ "lower",
+ "lpad",
+ "ltrim",
+ "money_format",
+ "null_or_empty",
+ "repeat",
+ "replace",
+ "reverse",
+ "right",
+ "rpad",
+ "split_part",
+ "starts_with",
+ "strleft",
+ "strright",
+ "substring",
+ "unhex",
+ {
+ title: "Fuzzy Match",
+ directoryPath: "like/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "like",
+ "not_like",
+ ],
+ },
+ {
+ title: "Regular Match",
+ directoryPath: "regexp/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "regexp",
+ "regexp_extract",
+ "regexp_replace",
+ "not_regexp",
+ ],
+ },
+ ],
+ },
+ {
+ title: "Aggregate Functions",
+ directoryPath: "aggregate-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "approx_count_distinct",
+ "avg",
+ "bitmap_union",
+ "count",
+ "group_concat",
+ "hll_union_agg",
+ "max",
+ "max_by",
+ "min",
+ "min_by",
+ "percentile",
+ "percentile_approx",
+ "stddev",
+ "stddev_samp",
+ "sum",
+ "topn",
+ "var_samp",
+ "variance",
+ ],
+ },
+ {
+ title: "Bitmap Functions",
+ directoryPath: "bitmap-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "bitmap_and",
+ "bitmap_contains",
+ "bitmap_empty",
+ "bitmap_from_string",
+ "bitmap_has_any",
+ "bitmap_has_all",
+ "bitmap_hash",
+ "bitmap_intersect",
+ "bitmap_or",
+ "bitmap_and_count",
+ "bitmap_or_count",
+ "bitmap_xor",
+ "bitmap_xor_count",
+ "bitmap_not",
+ "bitmap_and_not",
+ "bitmap_and_not_count",
+ "bitmap_subset_in_range",
+ "bitmap_subset_limit",
+ "sub_bitmap",
+ "bitmap_to_string",
+ "bitmap_union",
+ "bitmap_xor",
+ "to_bitmap",
+ "bitmap_max",
+ "intersect_count",
+ "orthogonal_bitmap_intersect",
+ "orthogonal_bitmap_intersect_count",
+ "orthogonal_bitmap_union_count",
+ ],
+ },
+ {
+ title: "Bitwise Functions",
+ directoryPath: "bitwise-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "bitand",
+ "bitor",
+ "bitxor",
+ "bitnot"
+ ],
+ },
+ {
+ title: "Condition Functions",
+ directoryPath: "conditional-functions/",
+ children: [
+ "case",
+ "coalesce",
+ "if",
+ "ifnull",
+ "nvl",
+ "nullif"
+ ],
+ },
+ {
+ title: "JSON Functions",
+ directoryPath: "json-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "get_json_double",
+ "get_json_int",
+ "get_json_string",
+ "json_array",
+ "json_object",
+ "json_quote",
+ ],
+ },
+ {
+ title: "Hash Functions",
+ directoryPath: "hash-functions/",
+ initialOpenGroupIndex: -1,
+ children: ["murmur_hash3_32"],
+ },
+ {
+ title: "Math Functions",
+ directoryPath: "math-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "conv",
+ "pmod"
+ ],
+ },
+ {
+ title: "Encryption Functions",
+ directoryPath: "encrypt-digest-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "aes",
+ "md5",
+ "md5sum",
+ "sm4",
+ "sm3",
+ "sm3sum"
+ ],
+ },
+ {
+ title: "Table Functions",
+ directoryPath: "table-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "explode-bitmap",
+ "explode-split",
+ "explode-json-array",
+ "outer-combinator"
+ ],
+ },
+ {
+ title: "Analytic(Window) Functions",
+ directoryPath: "window-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "WINDOW-FUNCTION",
+ "WINDOW-FUNCTION-SUM",
+ "WINDOW-FUNCTION-AVG",
+ "WINDOW-FUNCTION-COUNT",
+ "WINDOW-FUNCTION-MIN",
+ "WINDOW-FUNCTION-MAX",
+ "WINDOW-FUNCTION-LEAD",
+ "WINDOW-FUNCTION-LAG",
+ "WINDOW-FUNCTION-RANK",
+ "WINDOW-FUNCTION-DENSE-RANK",
+ "WINDOW-FUNCTION-FIRST-VALUE",
+ "WINDOW-FUNCTION-LAST-VALUE",
+ "WINDOW-FUNCTION-ROW-NUMBER",
+ "WINDOW-FUNCTION-NTILE",
+ ],
+ },
+ {
+ title: "Array Functions",
+ directoryPath: "array-functions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "array_contains",
+ "array_position",
+ "element_at",
+ ],
+ },
+ "cast",
+ "digital-masking",
+ ],
+ },
+ {
+ title: "SQL Reference",
+ directoryPath: "sql-reference/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Account Management",
+ directoryPath: "Account-Management-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "CREATE-USER",
+ "CREATE-ROLE",
+ "DROP-ROLE",
+ "DROP-USER",
+ "GRANT",
+ "REVOKE",
+ "SET-PASSWORD",
+ "SET-PROPERTY",
+ "LDAP",
+ ],
+ },
+ {
+ title: "Cluster management",
+ directoryPath: "Cluster-Management-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "ALTER-SYSTEM-ADD-BACKEND",
+ "ALTER-SYSTEM-ADD-FOLLOWER",
+ "ALTER-SYSTEM-ADD-OBSERVER",
+ "ALTER-SYSTEM-DECOMMISSION-BACKEND",
+ "ALTER-SYSTEM-DROP-BACKEND",
+ "ALTER-SYSTEM-DROP-FOLLOWER",
+ "ALTER-SYSTEM-DROP-OBSERVER",
+ "ALTER-SYSTEM-MODIFY-BROKER",
+ "CANCEL-ALTER-SYSTEM",
+ ],
+ },
+ {
+ title: "DDL",
+ directoryPath: "Data-Definition-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Alter",
+ directoryPath: "Alter/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "ALTER-DATABASE",
+ "ALTER-RESOURCE",
+ "ALTER-SQL-BLOCK-RULE",
+ "ALTER-TABLE-COLUMN",
+ "ALTER-TABLE-PARTITION",
+ "ALTER-TABLE-PROPERTY",
+ "ALTER-TABLE-RENAME",
+ "ALTER-TABLE-REPLACE",
+ "ALTER-TABLE-ROLLUP",
+ "ALTER-VIEW",
+ "CANCEL-ALTER-TABLE",
+ ],
+ },
+ {
+ title: "Backup and Restore",
+ directoryPath: "Backup-and-Restore/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "BACKUP",
+ "CANCEL-BACKUP",
+ "CANCEL-RESTORE",
+ "CREATE-REPOSITORY",
+ "DROP-REPOSITORY",
+ "RESTORE",
+ ],
+ },
+ {
+ title: "Create",
+ directoryPath: "Create/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "CREATE-DATABASE",
+ "CREATE-ENCRYPT-KEY",
+ "CREATE-FILE",
+ "CREATE-FUNCTION",
+ "CREATE-INDEX",
+ "CREATE-MATERIALIZED-VIEW",
+ "CREATE-POLICY",
+ "CREATE-RESOURCE",
+ "CREATE-SQL-BLOCK-RULE",
+ "CREATE-TABLE-LIKE",
+ "CREATE-TABLE-AS-SELECT",
+ "CREATE-TABLE",
+ "CREATE-VIEW",
+ "CREATE-EXTERNAL-TABLE",
+ ],
+ },
+ {
+ title: "Drop",
+ directoryPath: "Drop/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "DROP-DATABASE",
+ "DROP-ENCRYPT-KEY",
+ "DROP-FILE",
+ "DROP-FUNCTION",
+ "DROP-INDEX",
+ "DROP-MATERIALIZED-VIEW",
+ "DROP-RESOURCE",
+ "DROP-SQL-BLOCK-RULE",
+ "DROP-TABLE",
+ "TRUNCATE-TABLE",
+ ],
+ },
+ ],
+ },
+ {
+ title: "DML",
+ directoryPath: "Data-Manipulation-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Load",
+ directoryPath: "Load/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "ALTER-ROUTINE-LOAD",
+ "BROKER-LOAD",
+ "CANCEL-LOAD",
+ "CREATE-ROUTINE-LOAD",
+ "PAUSE-ROUTINE-LOAD",
+ "RESUME-ROUTINE-LOAD",
+ "STOP-ROUTINE-LOAD",
+ "STREAM-LOAD",
+ "PAUSE-SYNC-JOB",
+ "RESUME-SYNC-JOB",
+ "STOP-SYNC-JOB",
+ "CREATE-SYNC-JOB",
+ ],
+ },
+ {
+ title: "Manipulation",
+ directoryPath: "Manipulation/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "DELETE",
+ "INSERT",
+ "UPDATE",
+ "SELECT",
+ "EXPORT"
+ ],
+ },
+ "OUTFILE"
+ ],
+ },
+ {
+ title: "Database Administration",
+ directoryPath: "Database-Administration-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "ADMIN-CANCEL-REPAIR",
+ "ADMIN-CHECK-TABLET",
+ "ADMIN-REPAIR-TABLE",
+ "ADMIN-SET-CONFIG",
+ "ADMIN-SET-REPLICA-STATUS",
+ "ADMIN-SHOW-CONFIG",
+ "ADMIN-SHOW-REPLICA-DISTRIBUTION",
+ "ADMIN-SHOW-REPLICA-STATUS",
+ "ENABLE-FEATURE",
+ "INSTALL-PLUGIN",
+ "KILL",
+ "RECOVER",
+ "SET-VARIABLE",
+ "UNINSTALL-PLUGIN",
+ ],
+ },
+ {
+ title: "Show",
+ directoryPath: "Show-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "SHOW-ALTER",
+ "SHOW-BACKENDS",
+ "SHOW-BACKUP",
+ "SHOW-BROKER",
+ "SHOW-CHARSET",
+ "SHOW-COLLATION",
+ "SHOW-COLUMNS",
+ "SHOW-CREATE-DATABASE",
+ "SHOW-CREATE-FUNCTION",
+ "SHOW-CREATE-ROUTINE-LOAD",
+ "SHOW-CREATE-TABLE",
+ "SHOW-CREATE-MATERIALIZED-VIEW",
+ "SHOW-DATA",
+ "SHOW-DATABASE-ID",
+ "SHOW-DATABASES",
+ "SHOW-DELETE",
+ "SHOW-DYNAMIC-PARTITION",
+ "SHOW-ENCRYPT-KEY",
+ "SHOW-ENGINES",
+ "SHOW-EVENTS",
+ "SHOW-EXPORT",
+ "SHOW-FRONTENDS",
+ "SHOW-FUNCTIONS",
+ "SHOW-GRANTS",
+ "SHOW-INDEX",
+ "SHOW-LOAD-PROFILE",
+ "SHOW-LOAD-WARNINGS",
+ "SHOW-LOAD",
+ "SHOW-MIGRATIONS",
+ "SHOW-OPEN-TABLES",
+ "SHOW-PARTITION-ID",
+ "SHOW-PARTITIONS",
+ "SHOW-PLUGINS",
+ "SHOW-PROC",
+ "SHOW-PROCEDURE",
+ "SHOW-PROCESSLIST",
+ "SHOW-PROPERTY",
+ "SHOW-QUERY-PROFILE",
+ "SHOW-REPOSITORIES",
+ "SHOW-RESOURCES",
+ "SHOW-RESTORE",
+ "SHOW-ROLES",
+ "SHOW-ROLLUP",
+ "SHOW-ROUTINE-LOAD-TASK",
+ "SHOW-ROUTINE-LOAD",
+ "SHOW-SMALL-FILES",
+ "SHOW-SNAPSHOT",
+ "SHOW-SQL-BLOCK-RULE",
+ "SHOW-STATUS",
+ "SHOW-STREAM-LOAD",
+ "SHOW-SYNC-JOB",
+ "SHOW-TABLE-ID",
+ "SHOW-TABLES",
+ "SHOW-TABLE-STATUS",
+ "SHOW-TABLET",
+ "SHOW-TRANSACTION",
+ "SHOW-TRIGGERS",
+ "SHOW-TRASH",
+ // "SHOW-USER",
+ "SHOW-VARIABLES",
+ "SHOW-VIEW",
+ "SHOW-WARNING",
+ "SHOW-WHITE-LIST",
+ ],
+ },
+ {
+ title: "Data Types",
+ directoryPath: "Data-Types/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "BIGINT",
+ "BITMAP",
+ "BOOLEAN",
+ "CHAR",
+ "DATE",
+ "DATETIME",
+ "DECIMAL",
+ "DOUBLE",
+ "FLOAT",
+ "HLL",
+ "INT",
+ "LARGEINT",
+ "SMALLINT",
+ "STRING",
+ "TINYINT",
+ "VARCHAR",
+ "ARRAY",
+ ],
+ },
+ {
+ title: "Utility",
+ directoryPath: "Utility-Statements/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "DESCRIBE",
+ "HELP",
+ "USE",
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ {
+ title: "Admin Manual",
+ directoryPath: "admin-manual/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "cluster management",
+ directoryPath: "cluster-management/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "upgrade",
+ "elastic-expansion",
+ "load-balancing"
+ ],
+ },
+ {
+ title: "Data Admin",
+ directoryPath: "data-admin/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "backup",
+ "restore",
+ "delete-recover"
+ ],
+ },
+ "sql-interception",
+ "query-profile",
+ "optimization",
+ {
+ title: "Maintenance and Monitor",
+ directoryPath: "maint-monitor/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "Monitor Metrics",
+ directoryPath: "monitor-metrics/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "fe-metrics",
+ "be-metrics"
+ ],
+ },
+ "disk-capacity",
+ "metadata-operation",
+ "tablet-meta-tool",
+ "tablet-repair-and-balance",
+ "tablet-restore-tool",
+ "monitor-alert",
+ "doris-error-code",
+ "be-olap-error-code"
+ ],
+ },
+ {
+ title: "Config",
+ directoryPath: "config/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "fe-config",
+ "be-config",
+ "user-property"
+ ],
+ },
+ {
+ title: "User Privilege and Ldap",
+ directoryPath: "privilege-ldap/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "user-privilege",
+ "ldap"
+ ],
+ },
+ "multi-tenant",
+ {
+ title: "HTTP API",
+ directoryPath: "http-actions/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "FE",
+ directoryPath: "fe/",
+ initialOpenGroupIndex: -1,
+ children: [
+ {
+ title: "MANAGER",
+ directoryPath: "manager/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "cluster-action",
+ "node-action",
+ "query-profile-action",
+ ],
+ },
+ "backends-action",
+ "bootstrap-action",
+ "cancel-load-action",
+ "check-decommission-action",
+ "check-storage-type-action",
+ "config-action",
+ "connection-action",
+ "get-ddl-stmt-action",
+ "get-load-info-action",
+ "get-load-state",
+ "get-log-file-action",
+ "get-small-file",
+ "ha-action",
+ "hardware-info-action",
+ "health-action",
+ "log-action",
+ "logout-action",
+ "meta-action",
+ "meta-info-action",
+ "meta-replay-state-action",
+ "profile-action",
+ "query-detail-action",
+ "query-profile-action",
+ "row-count-action",
+ "session-action",
+ "set-config-action",
+ "show-data-action",
+ "show-meta-info-action",
+ "show-proc-action",
+ "show-runtime-info-action",
+ "statement-execution-action",
+ "system-action",
+ "table-query-plan-action",
+ "table-row-count-action",
+ "table-schema-action",
+ "upload-action",
+ ],
+ },
+ "cancel-label",
+ "check-reset-rpc-cache",
+ "compaction-action",
+ "connection-action",
+ "fe-get-log-file",
+ "get-load-state",
+ "get-tablets",
+ "profile-action",
+ "query-detail-action",
+ "restore-tablet",
+ "show-data-action",
+ "tablet-migration-action",
+ "tablets_distribution",
+ ],
+ sidebarDepth: 1,
+ },
+ ],
+ },
+ {
+ title: "FAQ",
+ directoryPath: "faq/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "install-faq",
+ "data-faq",
+ "sql-faq"
+ ],
+ },
+ {
+ title: "Benchmark",
+ directoryPath: "benchmark/",
+ initialOpenGroupIndex: -1,
+ children: [
+ "ssb",
+ "tpch"
+ ],
+ }
+];
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index 8a804841cb..ac30752025 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -469,6 +469,7 @@ module.exports = [
"bitmap_xor",
"to_bitmap",
"bitmap_max",
+ "intersect_count",
"orthogonal_bitmap_intersect",
"orthogonal_bitmap_intersect_count",
"orthogonal_bitmap_union_count",
diff --git
a/docs/en/sql-reference/sql-functions/bitmap-functions/intersect_count.md
b/docs/en/sql-reference/sql-functions/bitmap-functions/intersect_count.md
new file mode 100644
index 0000000000..938865d3ac
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/bitmap-functions/intersect_count.md
@@ -0,0 +1,57 @@
+---
+{
+"title": "intersect_count",
+"language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## intersect_count
+### description
+#### Syntax
+
+`BITMAP INTERSECT_COUNT(bitmap_column, column_to_filter, filter_values)`
+Calculate the intersection of two or more bitmaps
+Usage: intersect_count(bitmap_column_to_count, filter_column, filter_values
...)
+Example: intersect_count(user_id, event, 'A', 'B', 'C'), meaning find the
intersect count of user_id in all A/B/C 3 bitmaps
+
+### example
+
+```
+MySQL [test_query_qa]> select dt,bitmap_to_string(user_id) from pv_bitmap
where dt in (3,4);
++------+-----------------------------+
+| dt | bitmap_to_string(`user_id`) |
++------+-----------------------------+
+| 4 | 1,2,3 |
+| 3 | 1,2,3,4,5 |
++------+-----------------------------+
+2 rows in set (0.012 sec)
+
+MySQL [test_query_qa]> select intersect_count(user_id,dt,3,4) from pv_bitmap;
++----------------------------------------+
+| intersect_count(`user_id`, `dt`, 3, 4) |
++----------------------------------------+
+| 3 |
++----------------------------------------+
+1 row in set (0.014 sec)
+```
+
+### keywords
+
+ INTERSECT_COUNT,BITMAP
diff --git
a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/intersect_count.md
b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/intersect_count.md
new file mode 100644
index 0000000000..41f58f6da0
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/intersect_count.md
@@ -0,0 +1,56 @@
+---
+{
+"title": "intersect_count",
+"language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## intersect_count
+### description
+#### Syntax
+
+`BITMAP INTERSECT_COUNT(bitmap_column, column_to_filter, filter_values)`
+聚合函数，求bitmap交集大小的函数, 不要求数据分布正交
+第一个参数是Bitmap列，第二个参数是用来过滤的维度列，第三个参数是变长参数，含义是过滤维度列的不同取值
+
+### example
+
+```
+MySQL [test_query_qa]> select dt,bitmap_to_string(user_id) from pv_bitmap
where dt in (3,4);
++------+-----------------------------+
+| dt | bitmap_to_string(`user_id`) |
++------+-----------------------------+
+| 4 | 1,2,3 |
+| 3 | 1,2,3,4,5 |
++------+-----------------------------+
+2 rows in set (0.012 sec)
+
+MySQL [test_query_qa]> select intersect_count(user_id,dt,3,4) from pv_bitmap;
++----------------------------------------+
+| intersect_count(`user_id`, `dt`, 3, 4) |
++----------------------------------------+
+| 3 |
++----------------------------------------+
+1 row in set (0.014 sec)
+```
+
+### keywords
+
+ INTERSECT_COUNT,BITMAP
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index ae2414a695..80a7d631f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -501,20 +501,23 @@ public class FunctionCallExpr extends Expr {
throw new AnalysisException("BITMAP_UNION_INT params only support
TINYINT or SMALLINT or INT");
}
- if
(fnName.getFunction().equalsIgnoreCase(FunctionSet.INTERSECT_COUNT)) {
+ if (fnName.getFunction().equalsIgnoreCase(FunctionSet.INTERSECT_COUNT)
|| fnName.getFunction()
+ .equalsIgnoreCase(FunctionSet.ORTHOGONAL_BITMAP_INTERSECT) ||
fnName.getFunction()
+
.equalsIgnoreCase(FunctionSet.ORTHOGONAL_BITMAP_INTERSECT_COUNT)) {
if (children.size() <= 2) {
- throw new AnalysisException("intersect_count(bitmap_column,
column_to_filter, filter_values) " +
- "function requires at least three parameters");
+ throw new AnalysisException(fnName + "(bitmap_column,
column_to_filter, filter_values) "
+ + "function requires at least three parameters");
}
Type inputType = getChild(0).getType();
if (!inputType.isBitmapType()) {
- throw new AnalysisException("intersect_count function first
argument should be of BITMAP type, but was " + inputType);
+ throw new AnalysisException(
+ fnName + "function first argument should be of BITMAP
type, but was " + inputType);
}
for (int i = 2; i < children.size(); i++) {
if (!getChild(i).isConstant()) {
- throw new AnalysisException("intersect_count function
filter_values arg must be constant");
+ throw new AnalysisException(fnName + " function
filter_values arg must be constant");
}
}
return;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
index 8f35805523..8a8de8214f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateFunction.java
@@ -48,8 +48,11 @@ public class AggregateFunction extends Function {
private static final Logger LOG =
LogManager.getLogger(AggregateFunction.class);
- public static ImmutableSet<String>
NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
- ImmutableSet.of("row_number", "rank", "dense_rank",
"multi_distinct_count", "multi_distinct_sum", "hll_union_agg", "hll_union",
"bitmap_union", "bitmap_intersect", FunctionSet.COUNT, "approx_count_distinct",
"ndv", FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT,
"ndv_no_finalize", FunctionSet.WINDOW_FUNNEL);
+ public static ImmutableSet<String>
NOT_NULLABLE_AGGREGATE_FUNCTION_NAME_SET = ImmutableSet.of("row_number", "rank",
+ "dense_rank", "multi_distinct_count", "multi_distinct_sum",
"hll_union_agg", "hll_union", "bitmap_union",
+ "bitmap_intersect", "orthogonal_bitmap_intersect",
"orthogonal_bitmap_intersect_count", "intersect_count",
+ "orthogonal_bitmap_union_count", FunctionSet.COUNT,
"approx_count_distinct", "ndv",
+ FunctionSet.BITMAP_UNION_INT, FunctionSet.BITMAP_UNION_COUNT,
"ndv_no_finalize", FunctionSet.WINDOW_FUNNEL);
public static ImmutableSet<String>
ALWAYS_NULLABLE_AGGREGATE_FUNCTION_NAME_SET =
ImmutableSet.of("stddev_samp", "variance_samp", "var_samp",
"percentile_approx");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index d0ff5711f3..d040f46715 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -1810,6 +1810,14 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
BITMAP_INTERSECT_FINALIZE_SYMBOL.get(t),
true, false, true));
+ // VEC_INTERSECT_COUNT
+ addBuiltin(
+ AggregateFunction.createBuiltin(INTERSECT_COUNT,
Lists.newArrayList(Type.BITMAP, t, t), Type.BIGINT,
+ Type.VARCHAR, true,
BITMAP_INTERSECT_INIT_SYMBOL.get(t),
+ BITMAP_INTERSECT_UPDATE_SYMBOL.get(t),
BITMAP_INTERSECT_MERGE_SYMBOL.get(t),
+ BITMAP_INTERSECT_SERIALIZE_SYMBOL.get(t), null,
null,
+ BITMAP_INTERSECT_FINALIZE_SYMBOL.get(t), true,
false, true, true));
+
// HLL_UNION_AGG
addBuiltin(AggregateFunction.createBuiltin("hll_union_agg",
Lists.newArrayList(t), Type.BIGINT, Type.VARCHAR,
@@ -2168,6 +2176,15 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
"",
"_ZN5doris15BitmapFunctions32orthogonal_bitmap_count_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE",
true, false, true));
+
+ //vec ORTHOGONAL_BITMAP_INTERSECT and
ORTHOGONAL_BITMAP_INTERSECT_COUNT
+ addBuiltin(
+
AggregateFunction.createBuiltin(ORTHOGONAL_BITMAP_INTERSECT,
Lists.newArrayList(Type.BITMAP, t, t),
+ Type.BITMAP, Type.BITMAP, true, "", "", "", "",
"", "", "", true, false, true, true));
+
+
addBuiltin(AggregateFunction.createBuiltin(ORTHOGONAL_BITMAP_INTERSECT_COUNT,
+ Lists.newArrayList(Type.BITMAP, t, t), Type.BIGINT,
Type.BITMAP, true, "", "", "", "", "", "", "",
+ true, false, true, true));
}
// bitmap
addBuiltin(AggregateFunction.createBuiltin(BITMAP_UNION,
Lists.newArrayList(Type.BITMAP),
@@ -2226,6 +2243,10 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
null,
"_ZN5doris15BitmapFunctions32orthogonal_bitmap_count_finalizeEPN9doris_udf15FunctionContextERKNS1_9StringValE",
true, true, true));
+ // ORTHOGONAL_BITMAP_UNION_COUNT vectorized
+
addBuiltin(AggregateFunction.createBuiltin(ORTHOGONAL_BITMAP_UNION_COUNT,
Lists.newArrayList(Type.BITMAP),
+ Type.BIGINT, Type.BITMAP, "", "", "", "", null, null, "",
true, true, true, true));
+
// TODO(ml): supply function symbol
addBuiltin(AggregateFunction.createBuiltin(BITMAP_INTERSECT,
Lists.newArrayList(Type.BITMAP),
Type.BITMAP, Type.VARCHAR,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
