This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 632029669db [function](bitmap) support bitmap_remove (#24190) (#24519)
632029669db is described below
commit 632029669db0c68b05bc3363f623059321ba4090
Author: TengJianPing <[email protected]>
AuthorDate: Mon Sep 18 11:23:44 2023 +0800
[function](bitmap) support bitmap_remove (#24190) (#24519)
---
be/src/vec/functions/function_bitmap.cpp | 39 ++++++++++++
be/test/vec/function/function_bitmap_test.cpp | 17 ++++++
.../bitmap-functions/bitmap-remove.md | 55 +++++++++++++++++
.../bitmap-functions/bitmap-remove.md | 55 +++++++++++++++++
.../doris/catalog/BuiltinScalarFunctions.java | 2 +
.../expressions/functions/scalar/BitmapRemove.java | 69 ++++++++++++++++++++++
.../expressions/visitor/ScalarFunctionVisitor.java | 5 ++
gensrc/script/doris_builtins_functions.py | 1 +
.../bitmap_functions/test_bitmap_function.groovy | 4 ++
9 files changed, 247 insertions(+)
diff --git a/be/src/vec/functions/function_bitmap.cpp
b/be/src/vec/functions/function_bitmap.cpp
index ab3cbca1295..038741c3947 100644
--- a/be/src/vec/functions/function_bitmap.cpp
+++ b/be/src/vec/functions/function_bitmap.cpp
@@ -845,6 +845,42 @@ struct BitmapContains {
}
};
+struct NameBitmapRemove {
+ static constexpr auto name = "bitmap_remove";
+};
+
+template <typename LeftDataType, typename RightDataType>
+struct BitmapRemove {
+ using ResultDataType = DataTypeBitMap;
+ using T0 = typename LeftDataType::FieldType;
+ using T1 = typename RightDataType::FieldType;
+ using LTData = std::vector<BitmapValue>;
+ using RTData = typename ColumnVector<T1>::Container;
+ using ResTData = std::vector<BitmapValue>;
+
+ static void vector_vector(const LTData& lvec, const RTData& rvec,
ResTData& res) {
+ size_t size = lvec.size();
+ for (size_t i = 0; i < size; ++i) {
+ res[i] = lvec[i];
+ res[i].remove(rvec[i]);
+ }
+ }
+ static void vector_scalar(const LTData& lvec, const T1& rval, ResTData&
res) {
+ size_t size = lvec.size();
+ for (size_t i = 0; i < size; ++i) {
+ res[i] = lvec[i];
+ res[i].remove(rval);
+ }
+ }
+ static void scalar_vector(const BitmapValue& lval, const RTData& rvec,
ResTData& res) {
+ size_t size = rvec.size();
+ for (size_t i = 0; i < size; ++i) {
+ res[i] = lval;
+ res[i].remove(rvec[i]);
+ }
+ }
+};
+
struct NameBitmapHasAny {
static constexpr auto name = "bitmap_has_any";
};
@@ -1234,6 +1270,8 @@ using FunctionBitmapAndNot =
FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapAndNot,
NameBitmapAndNot>;
using FunctionBitmapContains =
FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapContains,
NameBitmapContains>;
+using FunctionBitmapRemove =
+ FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapRemove,
NameBitmapRemove>;
using FunctionBitmapHasAny =
FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapHasAny,
NameBitmapHasAny>;
@@ -1261,6 +1299,7 @@ void register_function_bitmap(SimpleFunctionFactory&
factory) {
factory.register_function<FunctionBitmapAndNot>();
factory.register_function<FunctionBitmapAndNotCount>();
factory.register_function<FunctionBitmapContains>();
+ factory.register_function<FunctionBitmapRemove>();
factory.register_function<FunctionBitmapHasAny>();
factory.register_function<FunctionBitmapHasAll>();
factory.register_function<FunctionSubBitmap>();
diff --git a/be/test/vec/function/function_bitmap_test.cpp
b/be/test/vec/function/function_bitmap_test.cpp
index 312a10ca4ec..75bbe32527c 100644
--- a/be/test/vec/function/function_bitmap_test.cpp
+++ b/be/test/vec/function/function_bitmap_test.cpp
@@ -82,6 +82,23 @@ TEST(function_bitmap_test, function_bitmap_to_string_test) {
check_function<DataTypeString, true>(func_name, input_types, data_set);
}
+TEST(function_bitmap_test, function_bitmap_remove) {
+ std::string func_name = "bitmap_remove";
+ InputTypeSet input_types = {TypeIndex::BitMap, TypeIndex::Int64};
+
+ BitmapValue bitmap1({1, 3});
+ BitmapValue bitmap2({1, 3, 5});
+
+ BitmapValue bitmap1_res(1);
+ BitmapValue bitmap2_res({1, 3, 5});
+ {
+ DataSet data_set = {{{&bitmap1, (int64_t)3}, bitmap1_res},
+ {{&bitmap2, (int64_t)6}, bitmap2_res},
+ {{&bitmap1, Null()}, Null()}};
+
+ check_function<DataTypeBitMap, true>(func_name, input_types, data_set);
+ }
+}
namespace doris {
namespace config {
DECLARE_Bool(enable_set_in_bitmap_value);
diff --git
a/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
new file mode 100644
index 00000000000..32e699c55b9
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
@@ -0,0 +1,55 @@
+---
+{
+ "title": "BITMAP_REMOVE",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## bitmap_remove
+### description
+#### Syntax
+
+`BITMAP BITMAP_REMOVE(BITMAP bitmap, BIGINT input)`
+
+Remove the specified value from bitmap.
+
+### example
+
+```
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2,
3'), 3)) res;
++------+
+| res |
++------+
+| 1,2 |
++------+
+
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2,
3'), null)) res;
++------+
+| res |
++------+
+| NULL |
++------+
+```
+
+### keywords
+
+ BITMAP_REMOVE,BITMAP
diff --git
a/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
b/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
new file mode 100644
index 00000000000..53d7cbee08b
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-functions/bitmap-functions/bitmap-remove.md
@@ -0,0 +1,55 @@
+---
+{
+ "title": "BITMAP_REMOVE",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## bitmap_remove
+### description
+#### Syntax
+
+`BITMAP BITMAP_REMOVE(BITMAP bitmap, BIGINT input)`
+
+从Bitmap列中删除指定的值。
+
+### example
+
+```
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2,
3'), 3)) res;
++------+
+| res |
++------+
+| 1,2 |
++------+
+
+mysql [(none)]>select bitmap_to_string(bitmap_remove(bitmap_from_string('1, 2,
3'), null)) res;
++------+
+| res |
++------+
+| NULL |
++------+
+```
+
+### keywords
+
+ BITMAP_REMOVE,BITMAP
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
index 5fde24724a4..78f4026873f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
@@ -72,6 +72,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapMin;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapRemove;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetInRange;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetLimit;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToArray;
@@ -417,6 +418,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(BitmapNot.class, "bitmap_not"),
scalar(BitmapOr.class, "bitmap_or"),
scalar(BitmapOrCount.class, "bitmap_or_count"),
+ scalar(BitmapRemove.class, "bitmap_remove"),
scalar(BitmapSubsetInRange.class, "bitmap_subset_in_range"),
scalar(BitmapSubsetLimit.class, "bitmap_subset_limit"),
scalar(BitmapToArray.class, "bitmap_to_array"),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapRemove.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapRemove.java
new file mode 100644
index 00000000000..d9c0d8b3709
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapRemove.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
+import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.BitmapType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * ScalarFunction 'bitmap_remove'. This class is generated by GenerateFunction.
+ */
+public class BitmapRemove extends ScalarFunction
+ implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+
+ public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+
FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE,
BigIntType.INSTANCE)
+ );
+
+ /**
+ * constructor with 2 arguments.
+ */
+ public BitmapRemove(Expression arg0, Expression arg1) {
+ super("bitmap_remove", arg0, arg1);
+ }
+
+ /**
+ * withChildren.
+ */
+ @Override
+ public BitmapRemove withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == 2,
String.format("children.size() is %d", children.size()));
+ return new BitmapRemove(children.get(0), children.get(1));
+ }
+
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return SIGNATURES;
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitBitmapRemove(this, context);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index 99b198e74b1..5dcc58fea85 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -76,6 +76,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapMin;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapRemove;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetInRange;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapSubsetLimit;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapToArray;
@@ -577,6 +578,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(bitmapOrCount, context);
}
+ default R visitBitmapRemove(BitmapRemove bitmapRemove, C context) {
+ return visitScalarFunction(bitmapRemove, context);
+ }
+
default R visitBitmapSubsetInRange(BitmapSubsetInRange
bitmapSubsetInRange, C context) {
return visitScalarFunction(bitmapSubsetInRange, context);
}
diff --git a/gensrc/script/doris_builtins_functions.py
b/gensrc/script/doris_builtins_functions.py
index efeaa0f4d4c..0ee7f342ccc 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -1803,6 +1803,7 @@ visible_functions = {
[['bitmap_from_array'], 'BITMAP', ['ARRAY_INT'], 'ALWAYS_NULLABLE'],
[['bitmap_from_array'], 'BITMAP', ['ARRAY_BIGINT'], 'ALWAYS_NULLABLE'],
[['bitmap_contains'], 'BOOLEAN', ['BITMAP','BIGINT'], ''],
+ [['bitmap_remove'], 'BITMAP', ['BITMAP','BIGINT'], ''],
[['bitmap_has_any'], 'BOOLEAN', ['BITMAP','BITMAP'], ''],
[['bitmap_has_all'], 'BOOLEAN', ['BITMAP','BITMAP'], ''],
[['bitmap_min'], 'BIGINT', ['BITMAP'], 'ALWAYS_NULLABLE'],
diff --git
a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
index 67f283044d9..709d2897212 100644
---
a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
+++
b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
@@ -833,6 +833,8 @@ suite("test_bitmap_function") {
// orthogonal_bitmap_intersect_count(id_bitmap, tag, 0) as
count2_bitmap from test_orthog_bitmap_intersect;
// """
+ /////////////////////////////
+ // test bitmap base64
sql """ set experimental_enable_nereids_planner=true; """
qt_sql_bitmap_base64_nereids0 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(null))); """
qt_sql_bitmap_base64_nereids1 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string(""))));
"""
@@ -844,6 +846,7 @@ suite("test_bitmap_function") {
qt_sql_bitmap_base64_nereids7 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296"))))
"""
qt_sql_bitmap_base64_nereids8 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1)))); """
+ // test nullable
sql """ DROP TABLE IF EXISTS test_bitmap_base64 """
sql """
CREATE TABLE test_bitmap_base64 (
@@ -883,6 +886,7 @@ suite("test_bitmap_function") {
qt_sql_bitmap_base64_9 """ select
bitmap_to_string(bitmap_from_base64(bitmap_to_base64(id))) s from
test_bitmap_base64 order by s; """
+ // test not nullable
sql """ set experimental_enable_nereids_planner=true; """
sql """ DROP TABLE IF EXISTS test_bitmap_base64_not_null """
sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
