This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1358e259c1e [feature](function) support ip function named
ipv4_cidr_to_range(addr, cidr) (#29819)
1358e259c1e is described below
commit 1358e259c1e3d6590c0d2c1973eb112ead6fa5ff
Author: nanfeng <[email protected]>
AuthorDate: Tue Jan 23 23:08:43 2024 +0800
[feature](function) support ip function named ipv4_cidr_to_range(addr,
cidr) (#29819)
* support ip function ipv4_cidr_to_range
* fix ipv4_cidr_to_range function only support ipv4 type
---
be/src/vec/functions/function_ip.cpp | 1 +
be/src/vec/functions/function_ip.h | 97 ++++++++++++++++++++++
.../doris/catalog/BuiltinScalarFunctions.java | 2 +
.../functions/scalar/Ipv4CIDRToRange.java | 75 +++++++++++++++++
.../expressions/visitor/ScalarFunctionVisitor.java | 5 ++
gensrc/script/doris_builtins_functions.py | 2 +
.../ip_functions/test_ipv4_cidr_to_range.out | 28 +++++++
.../ip_functions/test_ipv4_cidr_to_range.groovy | 58 +++++++++++++
8 files changed, 268 insertions(+)
diff --git a/be/src/vec/functions/function_ip.cpp
b/be/src/vec/functions/function_ip.cpp
index 7eed9c13364..700fb897b44 100644
--- a/be/src/vec/functions/function_ip.cpp
+++ b/be/src/vec/functions/function_ip.cpp
@@ -37,6 +37,7 @@ void register_function_ip(SimpleFunctionFactory& factory) {
factory.register_function<FunctionIsIPString<IPv4>>();
factory.register_function<FunctionIsIPString<IPv6>>();
factory.register_function<FunctionIsIPAddressInRange>();
+ factory.register_function<FunctionIPv4CIDRToRange>();
factory.register_function<FunctionIPv6CIDRToRange>();
factory.register_function<FunctionToIP<IPExceptionMode::Throw, IPv4>>();
factory.register_function<FunctionToIP<IPExceptionMode::Default, IPv4>>();
diff --git a/be/src/vec/functions/function_ip.h
b/be/src/vec/functions/function_ip.h
index 359cafe7e34..65075bcaa2b 100644
--- a/be/src/vec/functions/function_ip.h
+++ b/be/src/vec/functions/function_ip.h
@@ -21,14 +21,25 @@
#pragma once
#include <glog/logging.h>
+#include <cstddef>
+#include <memory>
+#include <vector>
+
#include "vec/columns/column.h"
+#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
#include "vec/columns/column_struct.h"
#include "vec/columns/column_vector.h"
+#include "vec/columns/columns_number.h"
#include "vec/common/format_ip.h"
#include "vec/common/ipv6_to_binary.h"
#include "vec/core/column_with_type_and_name.h"
+#include "vec/core/columns_with_type_and_name.h"
+#include "vec/core/types.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_ipv4.h"
#include "vec/data_types/data_type_ipv6.h"
+#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_string.h"
#include "vec/data_types/data_type_struct.h"
@@ -706,6 +717,92 @@ public:
}
};
+class FunctionIPv4CIDRToRange : public IFunction {
+public:
+ static constexpr auto name = "ipv4_cidr_to_range";
+ static FunctionPtr create() { return
std::make_shared<FunctionIPv4CIDRToRange>(); }
+
+ String get_name() const override { return name; }
+
+ size_t get_number_of_arguments() const override { return 2; }
+
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
+ WhichDataType first_arg_type = arguments[0];
+ if (!(first_arg_type.is_ipv4())) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Illegal type {} of first argument of function {},
expected IPv4",
+ arguments[0]->get_name(), get_name());
+ }
+
+ WhichDataType second_arg_type = arguments[1];
+ if (!(second_arg_type.is_int16())) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "Illegal type {} of second argument of function
{}, expected Int16",
+ arguments[1]->get_name(), get_name());
+ }
+
+ DataTypePtr element = std::make_shared<DataTypeIPv4>();
+
+ return std::make_shared<DataTypeStruct>(DataTypes {element, element},
+ Strings {"min", "max"});
+ }
+
+ Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ size_t result, size_t input_rows_count) const override
{
+ ColumnWithTypeAndName& ip_column = block.get_by_position(arguments[0]);
+ ColumnWithTypeAndName& cidr_column =
block.get_by_position(arguments[1]);
+
+ const ColumnPtr& ip_column_ptr = ip_column.column;
+ const ColumnPtr& cidr_column_ptr = cidr_column.column;
+
+ const auto* col_ip_column =
check_and_get_column<ColumnVector<IPv4>>(ip_column_ptr.get());
+ const auto* col_cidr_column =
+
check_and_get_column<ColumnVector<Int16>>(cidr_column_ptr.get());
+
+ const typename ColumnVector<IPv4>::Container& vec_ip_input =
col_ip_column->get_data();
+ const ColumnInt16::Container& vec_cidr_input =
col_cidr_column->get_data();
+ auto col_lower_range_output = ColumnIPv4::create(input_rows_count, 0);
+ auto col_upper_range_output = ColumnIPv4::create(input_rows_count, 0);
+
+ ColumnIPv4::Container& vec_lower_range_output =
col_lower_range_output->get_data();
+ ColumnIPv4::Container& vec_upper_range_output =
col_upper_range_output->get_data();
+
+ static constexpr UInt8 max_cidr_mask = IPV4_BINARY_LENGTH * 8;
+
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ auto ip = vec_ip_input[i];
+ auto cidr = vec_cidr_input[i];
+ if (0 <= cidr && cidr <= max_cidr_mask) {
+ auto range = apply_cidr_mask(ip, cidr);
+ vec_lower_range_output[i] = range.first;
+ vec_upper_range_output[i] = range.second;
+ } else {
+ return Status::InvalidArgument("Invalid row {}, cidr is out of
range", i);
+ }
+ }
+
+ block.replace_by_position(
+ result, ColumnStruct::create(Columns
{std::move(col_lower_range_output),
+
std::move(col_upper_range_output)}));
+ return Status::OK();
+ }
+
+private:
+ static inline std::pair<UInt32, UInt32> apply_cidr_mask(UInt32 src, UInt8
bits_to_keep) {
+ if (bits_to_keep >= 8 * sizeof(UInt32)) {
+ return {src, src};
+ }
+ if (bits_to_keep == 0) {
+ return {static_cast<UInt32>(0), static_cast<UInt32>(-1)};
+ }
+ UInt32 mask = static_cast<UInt32>(-1) << (8 * sizeof(UInt32) -
bits_to_keep);
+ UInt32 lower = src & mask;
+ UInt32 upper = lower | ~mask;
+
+ return {lower, upper};
+ }
+};
+
class FunctionIPv6CIDRToRange : public IFunction {
public:
static constexpr auto name = "ipv6_cidr_to_range";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
index 1a40365b523..94a1aaaaace 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
@@ -196,6 +196,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ignore;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Initcap;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProduct;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Instr;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4CIDRToRange;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4NumToString;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4StringToNum;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4StringToNumOrDefault;
@@ -628,6 +629,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(IsIpv4String.class, "is_ipv4_string"),
scalar(IsIpv6String.class, "is_ipv6_string"),
scalar(IsIpAddressInRange.class, "is_ip_address_in_range"),
+ scalar(Ipv4CIDRToRange.class, "ipv4_cidr_to_range"),
scalar(Ipv6CIDRToRange.class, "ipv6_cidr_to_range"),
scalar(ToIpv4.class, "to_ipv4"),
scalar(ToIpv4OrDefault.class, "to_ipv4_or_default"),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4CIDRToRange.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4CIDRToRange.java
new file mode 100644
index 00000000000..236e2371213
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Ipv4CIDRToRange.java
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
+import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.IPv4Type;
+import org.apache.doris.nereids.types.SmallIntType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * scalar function IPv4CIDRToRange
+ */
+public class Ipv4CIDRToRange extends ScalarFunction
+ implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+
+ public static final List<FunctionSignature> SIGNATURES;
+
+ static {
+ ImmutableList.Builder<StructField> structFields =
ImmutableList.builder();
+ structFields.add(new StructField("min", IPv4Type.INSTANCE, false, ""));
+ structFields.add(new StructField("max", IPv4Type.INSTANCE, false, ""));
+ StructType retType = new StructType(structFields.build());
+ SIGNATURES = ImmutableList.of(
+ FunctionSignature.ret(retType).args(IPv4Type.INSTANCE,
SmallIntType.INSTANCE));
+ }
+
+ public Ipv4CIDRToRange(Expression arg0, Expression arg1) {
+ super("ipv4_cidr_to_range", arg0, arg1);
+ }
+
+ @Override
+ public Ipv4CIDRToRange withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == 2,
+ "ipv4_cidr_to_range accept 2 args, but got %s (%s)",
+ children.size(),
+ children);
+ return new Ipv4CIDRToRange(children.get(0), children.get(1));
+ }
+
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return SIGNATURES;
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitIpv4CIDRToRange(this, context);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index d10c384c55e..9a1b970b72f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -192,6 +192,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ignore;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Initcap;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.InnerProduct;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Instr;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4CIDRToRange;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4NumToString;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4StringToNum;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv4StringToNumOrDefault;
@@ -1174,6 +1175,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(ipv6StringToNumOrDefault, context);
}
+ default R visitIpv4CIDRToRange(Ipv4CIDRToRange ipv4CIDRToRange, C context)
{
+ return visitScalarFunction(ipv4CIDRToRange, context);
+ }
+
default R visitIpv6StringToNumOrNull(Ipv6StringToNumOrNull
ipv6StringToNumOrNull, C context) {
return visitScalarFunction(ipv6StringToNumOrNull, context);
}
diff --git a/gensrc/script/doris_builtins_functions.py
b/gensrc/script/doris_builtins_functions.py
index 9faf1e092ec..f0f36a3caf0 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -2033,6 +2033,8 @@ visible_functions = {
[['is_ipv6_string'], 'BOOLEAN', ['STRING'], ''],
[['is_ip_address_in_range'], 'BOOLEAN', ['VARCHAR', 'VARCHAR'],
'ALWAYS_NOT_NULLABLE'],
[['is_ip_address_in_range'], 'BOOLEAN', ['STRING', 'STRING'],
'ALWAYS_NOT_NULLABLE'],
+ [['ipv4_cidr_to_range'], 'STRUCT<IPV4, IPV4>', ['IPV4', 'SMALLINT'],
''],
+ [['ipv6_cidr_to_range'], 'STRUCT<IPV6, IPV6>', ['IPV6', 'SMALLINT'],
''],
[['ipv6_cidr_to_range'], 'STRUCT<IPV6, IPV6>', ['VARCHAR',
'SMALLINT'], ''],
[['ipv6_cidr_to_range'], 'STRUCT<IPV6, IPV6>', ['STRING', 'SMALLINT'],
''],
[['to_ipv4'], 'IPV4', ['VARCHAR'], 'ALWAYS_NOT_NULLABLE'],
diff --git
a/regression-test/data/query_p0/sql_functions/ip_functions/test_ipv4_cidr_to_range.out
b/regression-test/data/query_p0/sql_functions/ip_functions/test_ipv4_cidr_to_range.out
new file mode 100644
index 00000000000..035426afe54
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/ip_functions/test_ipv4_cidr_to_range.out
@@ -0,0 +1,28 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 \N \N
+2 \N \N
+3 127.0.0.1 127.0.0.1
+4 127.0.0.0 127.0.0.255
+5 127.0.0.0 127.0.255.255
+6 127.0.0.0 127.255.255.255
+7 0.0.0.0 255.255.255.255
+
+-- !sql --
+\N
+
+-- !sql --
+\N
+
+-- !sql --
+{"min": "127.0.0.1", "max": "127.0.0.1"}
+
+-- !sql --
+{"min": "127.0.0.0", "max": "127.0.255.255"}
+
+-- !sql --
+{"min": "127.0.0.0", "max": "127.255.255.255"}
+
+-- !sql --
+{"min": "0.0.0.0", "max": "255.255.255.255"}
+
diff --git
a/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv4_cidr_to_range.groovy
b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv4_cidr_to_range.groovy
new file mode 100644
index 00000000000..eda4174de99
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv4_cidr_to_range.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_ipv4_cidr_to_range") {
+ sql """ DROP TABLE IF EXISTS test_ipv4_cidr_to_range """
+
+ sql """ SET enable_nereids_planner=true """
+ sql """ SET enable_fallback_to_original_planner=false """
+
+ sql """
+ CREATE TABLE test_ipv4_cidr_to_range (
+ `id` int,
+ `addr` ipv4,
+ `cidr` int
+ ) ENGINE=OLAP
+ UNIQUE KEY (`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into test_ipv4_cidr_to_range values
+ (1, null, 0),
+ (2, '127.0.0.1', null),
+ (3, '127.0.0.1', 32),
+ (4, '127.0.0.1', 24),
+ (5, '127.0.0.1', 16),
+ (6, '127.0.0.1', 8),
+ (7, '127.0.0.1', 0)
+ """
+
+ qt_sql "select id, struct_element(ipv4_cidr_to_range(addr, cidr), 'min')
as min_range, struct_element(ipv4_cidr_to_range(addr, cidr), 'max') as
max_range from test_ipv4_cidr_to_range order by id"
+
+ sql """ DROP TABLE IF EXISTS test_ipv4_cidr_to_range """
+
+ qt_sql "select ipv4_cidr_to_range(null, 0)"
+ qt_sql "select ipv4_cidr_to_range('127.0.0.1', null)"
+ qt_sql "select ipv4_cidr_to_range('127.0.0.1', 32)"
+ qt_sql "select ipv4_cidr_to_range('127.0.0.1', 16)"
+ qt_sql "select ipv4_cidr_to_range('127.0.0.1', 8)"
+ qt_sql "select ipv4_cidr_to_range('127.0.0.1', 0)"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
