This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a0961070f7d [Configurations](multi-catalog) Add
`enable_parquet_filter_by_min_max` and `enable_orc_filter_by_min_max` Session
variables. (#35290)
a0961070f7d is described below
commit a0961070f7d5f3b217d0ab008984d5f416c8f80b
Author: Qi Chen <[email protected]>
AuthorDate: Tue May 28 14:06:57 2024 +0800
[Configurations](multi-catalog) Add `enable_parquet_filter_by_min_max` and
`enable_orc_filter_by_min_max` Session variables. (#35290)
backport #35012 #35320
---
be/src/vec/exec/format/orc/vorc_reader.cpp | 5 +-
be/src/vec/exec/format/orc/vorc_reader.h | 1 +
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 18 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 1 +
.../java/org/apache/doris/qe/SessionVariable.java | 39 +++
gensrc/thrift/PaloInternalService.thrift | 3 +
.../hive/test_hive_basic_type.out | 262 +++++++++++++++++++++
.../hive/test_hive_basic_type.groovy | 6 +
8 files changed, 329 insertions(+), 6 deletions(-)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp
b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 6bf9bbe7dd4..a188c2e3eb5 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -150,6 +150,8 @@ OrcReader::OrcReader(RuntimeProfile* profile, RuntimeState*
state,
_is_hive(params.__isset.slot_name_to_schema_pos),
_io_ctx(io_ctx),
_enable_lazy_mat(enable_lazy_mat),
+ _enable_filter_by_min_max(
+ state == nullptr ? true :
state->query_options().enable_orc_filter_by_min_max),
_dict_cols_has_converted(false),
_unsupported_pushdown_types(unsupported_pushdown_types) {
TimezoneUtils::find_cctz_time_zone(ctz, _time_zone);
@@ -171,6 +173,7 @@ OrcReader::OrcReader(const TFileScanRangeParams& params,
const TFileRangeDesc& r
_file_system(nullptr),
_io_ctx(io_ctx),
_enable_lazy_mat(enable_lazy_mat),
+ _enable_filter_by_min_max(true),
_dict_cols_has_converted(false) {
_init_system_properties();
_init_file_description();
@@ -648,7 +651,7 @@ bool static
build_search_argument(std::vector<OrcPredicate>& predicates, int ind
bool OrcReader::_init_search_argument(
std::unordered_map<std::string, ColumnValueRangeType>*
colname_to_value_range) {
- if (colname_to_value_range->empty()) {
+ if ((!_enable_filter_by_min_max) || colname_to_value_range->empty()) {
return false;
}
std::vector<OrcPredicate> predicates;
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h
b/be/src/vec/exec/format/orc/vorc_reader.h
index 928c2ee7579..b87ccb823fc 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -535,6 +535,7 @@ private:
io::IOContext* _io_ctx;
bool _enable_lazy_mat = true;
+ bool _enable_filter_by_min_max = true;
std::vector<DecimalScaleParams> _decimal_scale_params;
size_t _decimal_scale_params_index;
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index ccdbee91353..a57004e6608 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -81,7 +81,10 @@ ParquetReader::ParquetReader(RuntimeProfile* profile, const
TFileScanRangeParams
_io_ctx(io_ctx),
_state(state),
_meta_cache(meta_cache),
- _enable_lazy_mat(enable_lazy_mat) {
+ _enable_lazy_mat(enable_lazy_mat),
+ _enable_filter_by_min_max(
+ state == nullptr ? true
+ :
state->query_options().enable_parquet_filter_by_min_max) {
_init_profile();
_init_system_properties();
_init_file_description();
@@ -94,7 +97,10 @@ ParquetReader::ParquetReader(const TFileScanRangeParams&
params, const TFileRang
_scan_range(range),
_io_ctx(io_ctx),
_state(state),
- _enable_lazy_mat(enable_lazy_mat) {
+ _enable_lazy_mat(enable_lazy_mat),
+ _enable_filter_by_min_max(
+ state == nullptr ? true
+ :
state->query_options().enable_parquet_filter_by_min_max) {
_init_system_properties();
_init_file_description();
}
@@ -757,8 +763,9 @@ Status ParquetReader::_process_page_index(const
tparquet::RowGroup& row_group,
_statistics.read_rows += row_group.num_rows;
};
- if (_lazy_read_ctx.has_complex_type || _lazy_read_ctx.conjuncts.empty() ||
- _colname_to_value_range == nullptr ||
_colname_to_value_range->empty()) {
+ if ((!_enable_filter_by_min_max) || _lazy_read_ctx.has_complex_type ||
+ _lazy_read_ctx.conjuncts.empty() || _colname_to_value_range == nullptr
||
+ _colname_to_value_range->empty()) {
read_whole_row_group();
return Status::OK();
}
@@ -867,7 +874,8 @@ Status ParquetReader::_process_row_group_filter(const
tparquet::RowGroup& row_gr
Status ParquetReader::_process_column_stat_filter(const
std::vector<tparquet::ColumnChunk>& columns,
bool* filter_group) {
- if (_colname_to_value_range == nullptr ||
_colname_to_value_range->empty()) {
+ if ((!_enable_filter_by_min_max) || _colname_to_value_range == nullptr ||
+ _colname_to_value_range->empty()) {
return Status::OK();
}
auto& schema_desc = _file_metadata->schema();
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h
b/be/src/vec/exec/format/parquet/vparquet_reader.h
index b46dae1e29c..679b06fa296 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -268,6 +268,7 @@ private:
// Maybe null if not used
FileMetaCache* _meta_cache = nullptr;
bool _enable_lazy_mat = true;
+ bool _enable_filter_by_min_max = true;
const TupleDescriptor* _tuple_descriptor;
const RowDescriptor* _row_descriptor;
const std::unordered_map<std::string, int>* _colname_to_slot_id;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 57b6fe9fecb..840603dc3ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -377,6 +377,10 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_ORC_LAZY_MAT =
"enable_orc_lazy_materialization";
+ public static final String ENABLE_PARQUET_FILTER_BY_MIN_MAX =
"enable_parquet_filter_by_min_max";
+
+ public static final String ENABLE_ORC_FILTER_BY_MIN_MAX =
"enable_orc_filter_by_min_max";
+
public static final String INLINE_CTE_REFERENCED_THRESHOLD =
"inline_cte_referenced_threshold";
public static final String ENABLE_CTE_MATERIALIZE =
"enable_cte_materialize";
@@ -1208,6 +1212,24 @@ public class SessionVariable implements Serializable,
Writable {
needForward = true)
public boolean enableOrcLazyMat = true;
+
+ @VariableMgr.VarAttr(
+ name = ENABLE_PARQUET_FILTER_BY_MIN_MAX,
+ description = {"控制 parquet reader 是否启用 min-max 值过滤。默认为 true。",
+ "Controls whether to filter by min-max values in parquet
reader. "
+ + "The default value is true."},
+ needForward = true)
+ public boolean enableParquetFilterByMinMax = true;
+
+
+ @VariableMgr.VarAttr(
+ name = ENABLE_ORC_FILTER_BY_MIN_MAX,
+ description = {"控制 orc reader 是否启用 min-max 值过滤。默认为 true。",
+ "Controls whether to filter by min-max values in orc
reader. "
+ + "The default value is true."},
+ needForward = true)
+ public boolean enableOrcFilterByMinMax = true;
+
@VariableMgr.VarAttr(
name = EXTERNAL_TABLE_ANALYZE_PART_NUM,
description = {"收集外表统计信息行数时选取的采样分区数，默认-1表示全部分区",
@@ -2148,6 +2170,21 @@ public class SessionVariable implements Serializable,
Writable {
this.enableOrcLazyMat = enableOrcLazyMat;
}
+ public boolean isEnableParquetFilterByMinMax() {
+ return enableParquetFilterByMinMax;
+ }
+
+ public void setEnableParquetFilterByMinMax(boolean
enableParquetFilterByMinMax) {
+ this.enableParquetFilterByMinMax = enableParquetFilterByMinMax;
+ }
+
+ public boolean isEnableOrcFilterByMinMax() {
+ return enableOrcFilterByMinMax;
+ }
+
+ public void setEnableOrcFilterByMinMax(boolean enableOrcFilterByMinMax) {
+ this.enableOrcFilterByMinMax = enableOrcFilterByMinMax;
+ }
/**
* getInsertVisibleTimeoutMs.
@@ -2587,6 +2624,8 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setEnableParquetLazyMat(enableParquetLazyMat);
tResult.setEnableOrcLazyMat(enableOrcLazyMat);
+ tResult.setEnableParquetFilterByMinMax(enableParquetFilterByMinMax);
+ tResult.setEnableOrcFilterByMinMax(enableOrcFilterByMinMax);
tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns);
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 37449a4d638..4185e8a4276 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -257,6 +257,9 @@ struct TQueryOptions {
90: optional bool enable_inverted_index_compound_inlist = false;
// For emergency use, skip missing version when reading rowsets
91: optional bool skip_missing_version = false;
+
+ 92: optional bool enable_parquet_filter_by_min_max = true
+ 93: optional bool enable_orc_filter_by_min_max = true
}
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
b/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
index 94de65a4979..774bc3d24c1 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
@@ -262,3 +262,265 @@ true
123.456
123.456
+-- !2 --
+1 7706 1 155190 17.00 21168.23 0.04 0.02 N
O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON
TRUCK egular courts above the cn beijing
+
+-- !3 --
+6374628540732951412 -77 -65 -70 -107 -215 65 0
-526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685
-228568 1633079 -2725524 6163040 -10491702 697237 74565050
127767368 93532213 -209675435 -32116110 -3624917040
-2927805617 15581947241 21893441661 24075494509 -116822110531
-59683724667 -146210393388 114424524398 1341560771667 -1638742564263
520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266
140803519083304 -294675355729619 -868076759504942 181128508165910
-91 [...]
+
+-- !4 --
+false 5 5 5 50 5.5 50.5 7295 12/31/10
5 2010-12-31T12:05:13.600 2010 12
+false 7 7 7 70 7.7 70.7 7297 12/31/10
7 2010-12-31T12:07:13.710 2010 12
+false 9 9 9 90 9.9 90.89999999999999 7299
12/31/10 9 2010-12-31T12:09:13.860 2010 12
+true 6 6 6 60 6.6 60.599999999999994 7296
12/31/10 6 2010-12-31T12:06:13.650 2010 12
+true 8 8 8 80 8.8 80.8 7298 12/31/10
8 2010-12-31T12:08:13.780 2010 12
+
+-- !5 --
+2 24 15314771 999319712124142303 true 6.009337E8
4.8177228079770208E16 \N northern rural 2022-08-30T23:21:08
407186.2849 phones int_col 2019-01-01 [2.5954339078494106e+17,
5.88165568758352e+17, 4.7802599872265741e+17, 6.9266228812515571e+17,
9.86405645575228e+17] \N phones int_col
+5 59 317349992 998913039814974432 false 5.6584864E8
9.900861328269033E17 Handling man satisfy firework descent top. Racing
closed county set-up crown cave. Correctly front duration pure. \N
2022-09-02T19:52:57 372765.2493 desktops tinyint_col
2021-10-03 [9.9832612525719834e+17, 3.6120761530306432e+17,
9.9691314965094349e+17, 8.9912907179234752e+17, 1.1955893747098878e+17]
["CrySxz", "FMXGRcaGbahSVqhp", "oRKqPmhM", "VdODasEdDWFSRIQf"] desktops
tinyint_col
+6 62 915699741 999653836472045196 true 4.51937536E8
8.7961505445021914E17 Tale get speed platform august curved. Ease grass
neighbour landlord. Baby genetic youth. \N 2022-08-07T09:30:56
875620.2176 phones smallint_col \N [9.4235407151618547e+17,
4.8332499920295616e+17, 9.1670077477898342e+17] ["zNfbLeFx",
"GNTJOmWJyRmOK", "hwvfhSQGsaaMEqUrWCK", "cQrQsROKLARA", "nONj", "oepXBFB",
"IPtUql"] phones smallint_col
+
+-- !6 --
+00cwjIryUv EXHwpeK2Nl hv2PYEMYMM eo69nyw4Yv K6797tgjFg
LlFNd8Kyy5 wkpLCO3uo1 AIXCj1MfeD ni0HxZbiUO 6IjRdM8Gqi
qsTMK6A2eC 1wu7v9OPwW qavArd9tDc sU88hZADLj lyzWlwLOCx
2022-11-25
+
+-- !9 --
+"" "test"
+
+-- !10 --
+\\N\\N\\N\\N\\N\\N\\N\\N\\Ntesttestaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !11 --
+\N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !12 --
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !13 --
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+
+-- !14 --
+50000
+
+-- !15 --
+
+-- !16 --
+0
+
+-- !17 --
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+
+-- !18 --
+
+-- !19 --
+0
+
+-- !20 --
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+
+-- !21 --
+50000
+
+-- !22 --
+
+-- !23 --
+0
+
+-- !24 --
+\N
+\N
+\N
+\N
+
+-- !25 --
+\N
+\N
+\N
+\N
+
+-- !26 --
+
+-- !27 --
+test DATETIME(6) Yes true \N
+
+-- !28 --
+\N
+1970-01-01T07:43:20
+1970-01-01T08:00:00.001
+1970-01-01T08:00:00.003
+2023-04-20T15:51:49
+
+-- !29 --
+test DATETIME(6) Yes true \N
+
+-- !30 --
+\N
+1970-01-01T07:43:20
+1970-01-01T08:00:00.000001
+1970-01-01T08:00:00.000003
+2023-04-20T15:51:49
+
+-- !31 --
+test DATETIME(6) Yes true \N
+
+-- !32 --
+\N
+1970-01-01T07:59:59
+1970-01-01T08:00
+1970-01-01T08:00
+2023-04-20T15:51:49
+
+-- !7 --
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !38 --
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+
+-- !39 --
+50000
+
+-- !40 --
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+3 {"field0": 4, "field1": 5} 6
+
+-- !33 --
+\N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !36 --
+\N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !41 --
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+\N \N \N \N \N \N \N \N \N \N
test test
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
[...]
+
+-- !parquet --
+219
+
+-- !parquet1 --
+-7871.416 false
+-7871.416 false
+-7871.416 false
+-7871.416 true
+-7871.416 true
+-7871.416 true
+-7871.416 true
+
+-- !parquet2 --
+123.456
+123.456
+123.456
+123.456
+123.456
+123.456
+123.456
+
+-- !parquet3 --
+123.456
+123.456
+123.456
+123.456
+123.456
+123.456
+123.456
+
+-- !parquet4 --
+-2633.645
+-2633.645
+-2633.645
+-2633.645
+-2633.645
+-2633.645
+-2633.645
+
+-- !parquet5 --
+-7871.416
+-7871.416
+-7871.416
+-7871.416
+-7871.416
+-7871.416
+-7871.416
+
+-- !parquet7 --
+true
+true
+true
+true
+true
+true
+true
+
+-- !parquet8 --
+true
+true
+true
+true
+true
+true
+true
+
+-- !parquet9 --
+116
+
+-- !parquet10 --
+123
+
+-- !parquet11 --
+123.456
+123.456
+123.456
+123.456
+123.456
+123.456
+123.456
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy
index dfa9b433bc6..286588d15d4 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_basic_type.groovy
@@ -18,12 +18,17 @@
suite("test_hive_basic_type",
"p0,external,hive,external_docker,external_docker_hive") {
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ for (boolean enable_filter_by_min_max : [true, false]) {
String catalog_name = "test_hive_basic_type"
String ex_db_name = "`default`"
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
String hms_port = context.config.otherConfigs.get("hms_port")
String hdfs_port = context.config.otherConfigs.get("hdfs_port")
+ sql """set enable_parquet_filter_by_min_max =
${enable_filter_by_min_max};"""
+
+ sql """set enable_orc_filter_by_min_max =
${enable_filter_by_min_max};"""
+
sql """drop catalog if exists ${catalog_name} """
sql """CREATE CATALOG ${catalog_name} PROPERTIES (
@@ -144,6 +149,7 @@ suite("test_hive_basic_type",
"p0,external,hive,external_docker,external_docker_
}
}
//sql """drop catalog if exists ${catalog_name} """
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
