This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 196e3437330 [improve](ip) update ip for bloom_filter index (#39253)
196e3437330 is described below
commit 196e3437330d880089e2d994f635e8da3f2114e0
Author: amory <[email protected]>
AuthorDate: Thu Aug 15 12:07:42 2024 +0800
[improve](ip) update ip for bloom_filter index (#39253)
---
.../segment_v2/bloom_filter_index_writer.cpp | 4 +-
.../data/bloom_filter_p0/test_bloom_filter_hit.out | 7 ++
.../bloom_filter_p0/test_bloom_filter_hit.groovy | 78 ++++++++++++++++++++++
3 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
index 3e80d2c6e61..98669ccb141 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
@@ -320,9 +320,11 @@ Status BloomFilterIndexWriter::create(const
BloomFilterOptions& bf_options,
M(FieldType::OLAP_FIELD_TYPE_DECIMAL64)
M(FieldType::OLAP_FIELD_TYPE_DECIMAL128I)
M(FieldType::OLAP_FIELD_TYPE_DECIMAL256)
+ M(FieldType::OLAP_FIELD_TYPE_IPV4)
+ M(FieldType::OLAP_FIELD_TYPE_IPV6)
#undef M
default:
- return Status::NotSupported("unsupported type for bitmap index: {}",
+ return Status::NotSupported("unsupported type for bloom filter index:
{}",
std::to_string(int(type)));
}
return Status::OK();
diff --git a/regression-test/data/bloom_filter_p0/test_bloom_filter_hit.out
b/regression-test/data/bloom_filter_p0/test_bloom_filter_hit.out
new file mode 100644
index 00000000000..5df8af4f2fd
--- /dev/null
+++ b/regression-test/data/bloom_filter_p0/test_bloom_filter_hit.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+4 4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8 192.168.11.1
+
+-- !sql --
+4 4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8 192.168.11.1
+
diff --git
a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
index 34c4042954c..f62c02a4205 100644
--- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
+++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy
@@ -16,6 +16,8 @@
// under the License.
import groovy.json.JsonSlurper
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
suite("test_bloom_filter_hit") {
def tableName = "test_bloom_filter_hit"
@@ -94,4 +96,80 @@ suite("test_bloom_filter_hit") {
//———————— clean table and disable profile ————————
sql """ SET enable_profile = false """
// sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ // test ipv for bloom filter
+ sql """ DROP TABLE IF EXISTS test_ip_bf """
+ sql """
+ CREATE TABLE IF NOT EXISTS test_ip_bf (
+ `id` int,
+ `ip_v6` ipv6,
+ `ip_v4` ipv4
+ )
+ ENGINE = OLAP DUPLICATE KEY (`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1",
+ "bloom_filter_columns" = "ip_v6, ip_v4"
+ );
+ """
+ sql """
+ insert into test_ip_bf values
+ (1, '::', '::'),
+ (2, 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff',
"192.168.1.1"),
+ (3, 'ef8d:3d6a:869b:2582:7200:aa46:4dcd:2bd4',
"192.168.1.11"),
+ (4, '4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8',
"192.168.11.1"),
+ (5, 'b374:22dc:814a:567b:6013:78a7:032d:05c8',
'127.0.0.1'),
+ (6, '1326:d47e:2417:83c0:bd35:fc82:34dc:953a',
'192.168.104.0'),
+ (7, '8ffa:65cb:6554:5c3e:fb87:3f91:29da:2891',
'192.168.103.255'),
+ (8, 'def7:1488:6fb7:0c70:aa66:df25:6a43:5d89',
'192.168.100.1'),
+ (9, 'd3fa:09a9:af08:0c8b:44ab:8f75:0b11:e997',
'192.168.107.1'),
+ (10, NULL, NULL); """
+
+
+ sql """ SET enable_profile = true """
+ sql """ set parallel_scan_min_rows_per_scanner = 2097152; """
+
+ // bf filter
+ qt_sql """ select * from test_ip_bf where
ip_v6='4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8' """
+ profiles = httpGet(profileUrl)
+ log.debug("profiles:{}", profiles);
+ profiles = new JsonSlurper().parseText(profiles)
+ assertEquals(0, profiles.code)
+
+ def profileIdIpv6 = null;
+ for (def profile in profiles["data"]["rows"]) {
+ if (profile["Sql Statement"].contains("""select * from test_ip_bf
where ip_v6='4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8'""")) {
+ profileIdIpv6 = profile["Profile ID"]
+ break;
+ }
+ }
+ log.info("profileId:{}", profileIdIpv6);
+ profileDetail = httpGet("/rest/v1/query_profile/" + profileIdIpv6)
+ // log.info("filter ipv6 profile:{}", profileDetail)
+ String regex = "RowsBloomFilterFiltered: (\\d+)"
+ Pattern pattern = Pattern.compile(regex)
+ Matcher matcher = pattern.matcher(profileDetail)
+
+ while (matcher.find()) {
+ int number = Integer.parseInt(matcher.group(1))
+ log.info("filter ipv6 number:{}", number)
+ assertTrue(number > 0)
+ }
+
+ qt_sql """ select * from test_ip_bf where ip_v4='192.168.11.1' """
+ def profileIdIpv4 = null;
+ for (def profile in profiles["data"]["rows"]) {
+ if (profile["Sql Statement"].contains("""select * from test_ip_bf
where ip_v4='192.168.11.1'""")) {
+ profileIdIpv4 = profile["Profile ID"]
+ break;
+ }
+ }
+ log.info("profileId:{}", profileIdIpv4);
+ profileDetail = httpGet("/rest/v1/query_profile/" + profileIdIpv4)
+ //log.info("filter ipv4 profile:{}", profileDetail)
+ Matcher matcherIpv4 = pattern.matcher(profileDetail)
+ while (matcherIpv4.find()) {
+ int number = Integer.parseInt(matcherIpv4.group(1))
+ log.info("filter ipv4 number:{}", number)
+ assertTrue(number > 0)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
