This is an automated email from the ASF dual-hosted git repository.
gabriellee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 78de4ad0c80 [function](feature) Implement CRC32 function (#38204)
78de4ad0c80 is described below
commit 78de4ad0c8025109e26faf5bf87e49f8ebcf16a7
Author: Gabriel <[email protected]>
AuthorDate: Tue Jul 23 11:48:02 2024 +0800
[function](feature) Implement CRC32 function (#38204)
---
be/src/vec/functions/function_string.cpp | 28 ++++++++-
.../doris/catalog/BuiltinScalarFunctions.java | 2 +
.../trees/expressions/functions/scalar/Crc32.java | 71 ++++++++++++++++++++++
.../expressions/visitor/ScalarFunctionVisitor.java | 5 ++
gensrc/script/doris_builtins_functions.py | 2 +
.../string_functions/test_string_function_like.out | 12 ++++
.../test_string_function_like.groovy | 4 ++
7 files changed, 122 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/functions/function_string.cpp
b/be/src/vec/functions/function_string.cpp
index f649726997b..955468f1a16 100644
--- a/be/src/vec/functions/function_string.cpp
+++ b/be/src/vec/functions/function_string.cpp
@@ -82,7 +82,7 @@ struct NameQuoteImpl {
}
};
-struct NameStringLenght {
+struct NameStringLength {
static constexpr auto name = "length";
};
@@ -104,6 +104,28 @@ struct StringLengthImpl {
}
};
+struct NameCrc32 {
+ static constexpr auto name = "crc32";
+};
+
+struct Crc32Impl {
+ using ReturnType = DataTypeInt64;
+ static constexpr auto TYPE_INDEX = TypeIndex::String;
+ using Type = String;
+ using ReturnColumnType = ColumnVector<Int64>;
+
+ static Status vector(const ColumnString::Chars& data, const
ColumnString::Offsets& offsets,
+ PaddedPODArray<Int64>& res) {
+ auto size = offsets.size();
+ res.resize(size);
+ for (int i = 0; i < size; ++i) {
+ res[i] = crc32_z(0L, (const unsigned char*)data.data() + offsets[i
- 1],
+ offsets[i] - offsets[i - 1]);
+ }
+ return Status::OK();
+ }
+};
+
struct NameStringUtf8Length {
static constexpr auto name = "char_length";
};
@@ -942,7 +964,8 @@ using StringFindInSetImpl =
StringFunctionImpl<LeftDataType, RightDataType, Find
// ready for regist function
using FunctionStringASCII = FunctionUnaryToType<StringASCII, NameStringASCII>;
-using FunctionStringLength = FunctionUnaryToType<StringLengthImpl,
NameStringLenght>;
+using FunctionStringLength = FunctionUnaryToType<StringLengthImpl,
NameStringLength>;
+using FunctionCrc32 = FunctionUnaryToType<Crc32Impl, NameCrc32>;
using FunctionStringUTF8Length = FunctionUnaryToType<StringUtf8LengthImpl,
NameStringUtf8Length>;
using FunctionStringSpace = FunctionUnaryToType<StringSpace, NameStringSpace>;
using FunctionStringStartsWith =
@@ -977,6 +1000,7 @@ using FunctionStringRPad = FunctionStringPad<StringRPad>;
void register_function_string(SimpleFunctionFactory& factory) {
factory.register_function<FunctionStringASCII>();
factory.register_function<FunctionStringLength>();
+ factory.register_function<FunctionCrc32>();
factory.register_function<FunctionStringUTF8Length>();
factory.register_function<FunctionStringSpace>();
factory.register_function<FunctionStringStartsWith>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
index bd1d10c7ec9..f57840ffd76 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
@@ -125,6 +125,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Cos;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.CosineDistance;
import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32;
import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct;
@@ -732,6 +733,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(Least.class, "least"),
scalar(Left.class, "left"),
scalar(Length.class, "length"),
+ scalar(Crc32.class, "crc32"),
scalar(Like.class, "like"),
scalar(Ln.class, "ln"),
scalar(Locate.class, "locate"),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java
new file mode 100644
index 00000000000..036807062fa
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Crc32.java
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
+import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.StringType;
+import org.apache.doris.nereids.types.VarcharType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * ScalarFunction 'crc32'. This class is generated by GenerateFunction.
+ */
+public class Crc32 extends ScalarFunction
+ implements UnaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+
+ public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+
FunctionSignature.ret(BigIntType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT),
+
FunctionSignature.ret(BigIntType.INSTANCE).args(StringType.INSTANCE)
+ );
+
+ /**
+ * constructor with 1 argument.
+ */
+ public Crc32(Expression arg) {
+ super("crc32", arg);
+ }
+
+ /**
+ * withChildren.
+ */
+ @Override
+ public Crc32 withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == 1);
+ return new Crc32(children.get(0));
+ }
+
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return SIGNATURES;
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitCrc32(this, context);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index e81924b7e7c..12fd3f119b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -132,6 +132,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Cos;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.CosineDistance;
import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32;
import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.CreateNamedStruct;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.CreateStruct;
@@ -1477,6 +1478,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(length, context);
}
+ default R visitCrc32(Crc32 crc32, C context) {
+ return visitScalarFunction(crc32, context);
+ }
+
default R visitLike(Like like, C context) {
return visitStringRegexPredicate(like, context);
}
diff --git a/gensrc/script/doris_builtins_functions.py
b/gensrc/script/doris_builtins_functions.py
index 15986dbf5af..4bd3c1c4943 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -1619,6 +1619,7 @@ visible_functions = {
[['rpad'], 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'],
'ALWAYS_NULLABLE'],
[['append_trailing_char_if_absent'], 'VARCHAR', ['VARCHAR',
'VARCHAR'], 'ALWAYS_NULLABLE'],
[['length'], 'INT', ['VARCHAR'], ''],
+ [['crc32'], 'BIGINT', ['VARCHAR'], ''],
[['bit_length'], 'INT', ['VARCHAR'], ''],
[['char_length', 'character_length'], 'INT', ['VARCHAR'], ''],
@@ -1685,6 +1686,7 @@ visible_functions = {
[['rpad'], 'STRING', ['STRING', 'INT', 'STRING'], 'ALWAYS_NULLABLE'],
[['append_trailing_char_if_absent'], 'STRING', ['STRING', 'STRING'],
'ALWAYS_NULLABLE'],
[['length'], 'INT', ['STRING'], ''],
+ [['crc32'], 'BIGINT', ['STRING'], ''],
[['bit_length'], 'INT', ['STRING'], ''],
[['char_length', 'character_length'], 'INT', ['STRING'], ''],
diff --git
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out
index 51fbfc68af1..9fcfc2d6ee9 100644
---
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out
+++
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function_like.out
@@ -245,3 +245,15 @@ bb
-- !sql --
+-- !crc32_1 --
+348606243
+
+-- !crc32_2 --
+130583814
+
+-- !crc32_3 --
+2707236321
+
+-- !crc32_4 --
+\N
+
diff --git
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy
index 9c5deed8651..e092526b03c 100644
---
a/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy
+++
b/regression-test/suites/query_p0/sql_functions/string_functions/test_string_function_like.groovy
@@ -90,4 +90,8 @@ suite("test_string_function_like") {
qt_sql "SELECT k FROM ${tbName} WHERE NOT LIKE(k, \"%\") ORDER BY k;"
// sql "DROP TABLE ${tbName};"
+ qt_crc32_1 "select crc32(\"DORIS\");"
+ qt_crc32_2 "select crc32(\"APACHE DORIS\");"
+ qt_crc32_3 "select crc32(10);"
+ qt_crc32_4 "select crc32(NULL);"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
