This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d194014914b [fix](ip) fix ip type in IndexDef support (#50637)
d194014914b is described below
commit d194014914b1e254ce8ae562f7bac01dfbc1e205
Author: amory <[email protected]>
AuthorDate: Fri Jun 20 19:51:38 2025 +0800
[fix](ip) fix ip type in IndexDef support (#50637)
### What problem does this PR solve?
only in branch-2.1 we do not support old indexDef to support Ip type
and fix array_contains && arrays_overlap for with type array<ip> which
backport: https://github.com/apache/doris/pull/50637
Issue Number: close #xxx
---
be/src/vec/functions/array/function_array_index.h | 10 +++
.../vec/functions/array/function_arrays_overlap.h | 6 ++
.../java/org/apache/doris/analysis/IndexDef.java | 2 +-
.../org/apache/doris/analysis/IndexDefTest.java | 63 +++++++++++++
.../test_array_ip_with_inverted_index.out | Bin 0 -> 4757 bytes
.../test_array_ip_with_inverted_index.groovy | 97 +++++++++++++++++++++
6 files changed, 177 insertions(+), 1 deletion(-)
diff --git a/be/src/vec/functions/array/function_array_index.h
b/be/src/vec/functions/array/function_array_index.h
index b313c8385d9..3f65d523f66 100644
--- a/be/src/vec/functions/array/function_array_index.h
+++ b/be/src/vec/functions/array/function_array_index.h
@@ -458,6 +458,16 @@ private:
offsets, nested_null_map, *nested_column,
*right_column,
right_nested_null_map, array_null_map);
}
+ } else if (is_ip(right_type) && is_ip(left_element_type)) {
+ if (left_which_type.is_ipv4()) {
+ return_column = _execute_number_expanded<ColumnIPv4>(
+ offsets, nested_null_map, *nested_column,
*right_column,
+ right_nested_null_map, array_null_map);
+ } else if (left_which_type.is_ipv6()) {
+ return_column = _execute_number_expanded<ColumnIPv6>(
+ offsets, nested_null_map, *nested_column,
*right_column,
+ right_nested_null_map, array_null_map);
+ }
}
if (return_column) {
diff --git a/be/src/vec/functions/array/function_arrays_overlap.h
b/be/src/vec/functions/array/function_arrays_overlap.h
index d641b78e655..8e319c8709f 100644
--- a/be/src/vec/functions/array/function_arrays_overlap.h
+++ b/be/src/vec/functions/array/function_arrays_overlap.h
@@ -341,6 +341,12 @@ public:
ret = _execute_internal<ColumnDecimal256>(left_exec_data,
right_exec_data,
dst_null_map_data,
dst_nested_col->get_data().data());
+ } else if (left_which_type.is_ipv4()) {
+ ret = _execute_internal<ColumnIPv4>(left_exec_data,
right_exec_data, dst_null_map_data,
+
dst_nested_col->get_data().data());
+ } else if (left_which_type.is_ipv6()) {
+ ret = _execute_internal<ColumnIPv6>(left_exec_data,
right_exec_data, dst_null_map_data,
+
dst_nested_col->get_data().data());
}
if (ret.ok()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index 9d2233690e9..145bccb5434 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -225,7 +225,7 @@ public class IndexDef {
}
if (!(colType.isDateType() || colType.isDecimalV2Type() ||
colType.isDecimalV3Type()
|| colType.isFixedPointType() || colType.isStringType() ||
colType == PrimitiveType.BOOLEAN
- || colType.isVariantType()) || colType.isIPType()) {
+ || colType.isVariantType() || colType.isIPType())) {
throw new AnalysisException(colType + " is not supported in "
+ indexType.toString() + " index. "
+ "invalid index: " + indexName);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
index 56c78e2430a..eb30163b7ce 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java
@@ -17,9 +17,13 @@
package org.apache.doris.analysis;
+import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructType;
import org.apache.doris.common.AnalysisException;
import com.google.common.collect.Lists;
@@ -75,6 +79,65 @@ public class IndexDefTest {
}
}
+ @Test
+ public void testIndexType() {
+ // support types
+ PrimitiveType[] supportedTypes = new PrimitiveType[] {
+ PrimitiveType.DATE, PrimitiveType.DATETIME,
PrimitiveType.DATEV2, PrimitiveType.DATETIMEV2,
+ PrimitiveType.DECIMALV2, PrimitiveType.DECIMAL32,
PrimitiveType.DECIMAL64, PrimitiveType.DECIMAL128, PrimitiveType.DECIMAL256,
+ PrimitiveType.TINYINT, PrimitiveType.SMALLINT,
PrimitiveType.INT, PrimitiveType.BIGINT, PrimitiveType.LARGEINT,
+ PrimitiveType.VARCHAR, PrimitiveType.CHAR,
PrimitiveType.STRING,
+ PrimitiveType.BOOLEAN,
+ PrimitiveType.IPV4, PrimitiveType.IPV6
+ };
+ for (PrimitiveType type : supportedTypes) {
+ try {
+ IndexDef def = new IndexDef("idx", false,
Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "");
+ Column col = new Column("col1", type);
+ def.checkColumn(col, KeysType.DUP_KEYS, true, true);
+ // check in array
+ def.checkColumn(new Column("col1",
ArrayType.create(ScalarType.createType(type), true)), KeysType.DUP_KEYS, true,
true);
+ } catch (AnalysisException e) {
+ Assert.fail("Should support type: " + type + ", but got: " +
e.getMessage());
+ }
+ }
+ // not support types
+ PrimitiveType[] unsupportedTypes = new PrimitiveType[] {
+ PrimitiveType.FLOAT, PrimitiveType.DOUBLE,
+ PrimitiveType.INVALID_TYPE, PrimitiveType.NULL_TYPE,
+ PrimitiveType.HLL, PrimitiveType.BITMAP,
PrimitiveType.QUANTILE_STATE,
+ PrimitiveType.TIME, PrimitiveType.TIMEV2, PrimitiveType.JSONB,
+ PrimitiveType.LAMBDA_FUNCTION, PrimitiveType.ALL
+ };
+ for (PrimitiveType type : unsupportedTypes) {
+ try {
+ IndexDef def = new IndexDef("idx", false,
Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "");
+ Column col = new Column("col1", type);
+ def.checkColumn(col, KeysType.DUP_KEYS, true, true);
+ Assert.fail("Should not support type: " + type);
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("is not supported"));
+ }
+ }
+ // check in map/struct
+ try {
+ IndexDef def = new IndexDef("idx", false,
Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "");
+ Column col = new Column("col1", new
MapType(ScalarType.createType(PrimitiveType.INT),
ScalarType.createType(PrimitiveType.STRING)));
+ def.checkColumn(col, KeysType.DUP_KEYS, true, true);
+ Assert.fail("Should not support map type");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("is not supported"));
+ }
+ try {
+ IndexDef def = new IndexDef("idx", false,
Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "");
+ Column col = new Column("col1", new
StructType(ScalarType.createType(PrimitiveType.INT),
ScalarType.createType(PrimitiveType.STRING)), true);
+ def.checkColumn(col, KeysType.DUP_KEYS, true, true);
+ Assert.fail("Should not support map type");
+ } catch (AnalysisException e) {
+ Assert.assertTrue(e.getMessage().contains("is not supported"));
+ }
+ }
+
@Test
public void toSql() {
Assert.assertEquals("INDEX `index1` (`col1`) USING INVERTED COMMENT
'balabala'", def.toSql());
diff --git
a/regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out
b/regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out
new file mode 100644
index 00000000000..ccc09bee8a0
Binary files /dev/null and
b/regression-test/data/inverted_index_p0/test_array_ip_with_inverted_index.out
differ
diff --git
a/regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy
b/regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy
new file mode 100644
index 00000000000..5de728d85cc
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/test_array_ip_with_inverted_index.groovy
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_ip_with_inverted_index") {
+ def tbl = "tai_ip_test"
+ sql "DROP TABLE IF EXISTS ${tbl}"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbl} (
+ id int,
+ ips_v4 array<ipv4> NULL,
+ ips_v6 array<ipv6> NULL,
+ ipv4_1 ipv4 NULL,
+ ipv4_2 ipv4 NULL,
+ ipv6_1 ipv6 NULL,
+ ipv6_2 ipv6 NULL,
+ INDEX idx_ips_v4(ips_v4) USING INVERTED COMMENT '',
+ INDEX idx_ips_v6(ips_v6) USING INVERTED COMMENT '',
+ INDEX idx_ipv4_1(ipv4_1) USING INVERTED COMMENT '',
+ INDEX idx_ipv6_1(ipv6_1) USING INVERTED COMMENT ''
+ ) ENGINE=OLAP
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ // insert some data
+ sql """ INSERT INTO ${tbl} VALUES (1, ['192.168.0.1', '10.0.0.1'],
['2001:db8::1', 'fe80::1'], '192.168.0.1', '10.0.0.1', '2001:db8::1',
'fe80::1'); """
+ sql """ INSERT INTO ${tbl} VALUES (2, ['172.16.0.1'], ['::1'],
'172.16.0.1', '1.1.1.1', '::1', '::2'); """
+ sql """ INSERT INTO ${tbl} VALUES (3, NULL, NULL, NULL, NULL, NULL, NULL);
"""
+ sql """ INSERT INTO ${tbl} VALUES (4, ['1.1.1.1'], ['2001:db8::2'],
'1.1.1.1', NULL, '2001:db8::2', NULL); """
+
+ // for array<ipv4> 和 array<ipv6> query array_contains/arrays_overlap
+ def ipv4_test_values = ["'192.168.0.1'", "'10.0.0.1'", "'1.1.1.1'", null]
+ for (def p : ipv4_test_values) {
+ order_qt_sql """ select * from ${tbl} where array_contains(ips_v4,
${p}) order by id; """
+ order_qt_sql """ select * from ${tbl} where !array_contains(ips_v4,
${p}) order by id; """
+ }
+ def ipv6_test_values = ["'2001:db8::1'", "'fe80::1'", "'::1'",
"'2001:db8::2'", null]
+ for (def p : ipv6_test_values) {
+ order_qt_sql """ select * from ${tbl} where array_contains(ips_v6,
${p}) order by id; """
+ order_qt_sql """ select * from ${tbl} where !array_contains(ips_v6,
${p}) order by id; """
+ }
+ def ipv4_overlap = ["['192.168.0.1','10.0.0.1']", "['1.1.1.1']", "[]",
null]
+ for (def p : ipv4_overlap) {
+ order_qt_sql """ select * from ${tbl} where arrays_overlap(ips_v4,
${p}) order by id; """
+ order_qt_sql """ select * from ${tbl} where !arrays_overlap(ips_v4,
${p}) order by id; """
+ }
+ def ipv6_overlap = ["['2001:db8::1','fe80::1']", "['::1']", "[]", null]
+ for (def p : ipv6_overlap) {
+ order_qt_sql """ select * from ${tbl} where arrays_overlap(ips_v6,
${p}) order by id; """
+ order_qt_sql """ select * from ${tbl} where !arrays_overlap(ips_v6,
${p}) order by id; """
+ }
+
+ // for ipv4_1/ipv4_2/ipv6_1/ipv6_2 query =、IN、IS NULL
+ order_qt_sql """ select * from ${tbl} where ipv4_1 = '192.168.0.1' order
by id; """
+ order_qt_sql """ select * from ${tbl} where ipv4_2 = '10.0.0.1' order by
id; """
+ order_qt_sql """ select * from ${tbl} where ipv6_1 = '2001:db8::1' order
by id; """
+ order_qt_sql """ select * from ${tbl} where ipv6_2 = 'fe80::1' order by
id; """
+ order_qt_sql """ select * from ${tbl} where ipv4_1 in ('192.168.0.1',
'1.1.1.1') order by id; """
+ order_qt_sql """ select * from ${tbl} where ipv6_2 is null order by id; """
+
+ // alter for inverted index
+ sql """ ALTER TABLE ${tbl} ADD INDEX idx_ipv4_2(ipv4_2) USING INVERTED
COMMENT ''; """
+ waitForSchemaChangeDone {
+ sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tbl}' ORDER BY
createtime DESC LIMIT 1 """
+ time 600
+ }
+ sql """ ALTER TABLE ${tbl} ADD INDEX idx_ipv6_2(ipv6_2) USING INVERTED
COMMENT ''; """
+ waitForSchemaChangeDone {
+ sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tbl}' ORDER BY
createtime DESC LIMIT 1 """
+ time 600
+ }
+
+
+ order_qt_sql """ select * from ${tbl} where ipv4_2 = '10.0.0.1' order by
id; """
+ order_qt_sql """ select * from ${tbl} where ipv4_2 in ('10.0.0.1',
'1.1.1.1') order by id; """
+ order_qt_sql """ select * from ${tbl} where ipv6_2 = 'fe80::1' order by
id; """
+ order_qt_sql """ select * from ${tbl} where ipv6_2 is null order by id; """
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
