This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 788abf2d7c3 [fix](iceberg_orc)Fixed the bug that the iceberg reader
did not perform position delete when reading the orc file without a predicate.
(#34814)
788abf2d7c3 is described below
commit 788abf2d7c3c7c2d57487a9608e889e7662d5fb2
Author: daidai <[email protected]>
AuthorDate: Wed May 15 10:49:59 2024 +0800
[fix](iceberg_orc)Fixed the bug that the iceberg reader did not perform
position delete when reading the orc file without a predicate. (#34814)
fix this bug:
Scenario : iceberg table uses orc storage format
sql : `select * from iceberg_orc;`
When executing this sql, position delete filter is not performed.
---
be/src/vec/exec/format/orc/vorc_reader.cpp | 7 +++
.../iceberg/iceberg_position_delete.groovy | 56 ++++++++++++++++++++++
2 files changed, 63 insertions(+)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp
b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 6909e1b3798..942eba59d08 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -1747,9 +1747,16 @@ Status OrcReader::get_next_block(Block* block, size_t*
read_rows, bool* eof) {
}
} else {
if (_delete_rows_filter_ptr) {
+
_execute_filter_position_delete_rowids(*_delete_rows_filter_ptr);
RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block,
columns_to_filter,
(*_delete_rows_filter_ptr)));
+ } else {
+ std::unique_ptr<IColumn::Filter> filter(new
IColumn::Filter(block->rows(), 1));
+ _execute_filter_position_delete_rowids(*filter);
+ RETURN_IF_CATCH_EXCEPTION(
+ Block::filter_block_internal(block, columns_to_filter,
(*filter)));
}
+
Block::erase_useless_column(block, column_to_keep);
static_cast<void>(_convert_dict_cols_to_string_cols(block,
&batch_vec));
}
diff --git
a/regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
b/regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
index 7774b108da3..8f62aaba50d 100644
---
a/regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
+++
b/regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
@@ -101,6 +101,62 @@ suite("iceberg_position_delete",
"p2,external,iceberg,external_remote,external_r
qt_parquet_19 """ select count(*) from iceberg_position_parquet where
name != 'final entryxxxxxx' ;"""
qt_parquet_20 """ select count(*) from iceberg_position_parquet; """
+
+ List<List<Object>> iceberg_position_orc = sql """ select * from
iceberg_position_orc ;"""
+ List<List<Object>> iceberg_position_parquet = sql """ select * from
iceberg_position_parquet;"""
+ List<List<Object>> iceberg_position_gen = sql """ select * from
iceberg_position_gen_data;"""
+
+ assertTrue(iceberg_position_orc.size() == iceberg_position_gen.size())
+ assertTrue(iceberg_position_orc.size() ==
iceberg_position_parquet.size())
+ assertTrue(iceberg_position_orc.size() == 5632)
+
+
+ List<List<Object>> iceberg_position_orc_1 = sql """select * from
iceberg_position_orc where id != 1;"""
+ List<List<Object>> iceberg_position_orc_2 = sql """select * from
iceberg_position_orc where name != "hello word" ;"""
+ List<List<Object>> iceberg_position_orc_3 = sql """select id from
iceberg_position_orc where id != 1;"""
+ List<List<Object>> iceberg_position_orc_4 = sql """select name from
iceberg_position_orc where id != 1;"""
+ List<List<Object>> iceberg_position_orc_5 = sql """select name from
iceberg_position_orc where name != "hello word" ;"""
+ List<List<Object>> iceberg_position_orc_6 = sql """select id from
iceberg_position_orc where name != "hello word" ;"""
+ List<List<Object>> iceberg_position_orc_7 = sql """select * from
iceberg_position_orc where id != 1 and name != "33333";"""
+ assertTrue(iceberg_position_orc_1.size() == 5632)
+ assertTrue(iceberg_position_orc_2.size() == 5632)
+ assertTrue(iceberg_position_orc_3.size() == 5632)
+ assertTrue(iceberg_position_orc_4.size() == 5632)
+ assertTrue(iceberg_position_orc_5.size() == 5632)
+ assertTrue(iceberg_position_orc_6.size() == 5632)
+ assertTrue(iceberg_position_orc_7.size() == 5632)
+
+
+ List<List<Object>> iceberg_position_gen_1 = sql """select * from
iceberg_position_gen_data where id != 1 and name != "hello";"""
+ assertTrue(iceberg_position_gen_1.size() == 5632)
+
+ List<List<Object>> iceberg_position_gen_2 = sql """select * from
iceberg_position_gen_data where id != 2;"""
+ assertTrue(iceberg_position_gen_2.size() == 5120)
+
+ List<List<Object>> iceberg_position_gen_22 = sql """select * from
iceberg_position_gen_data where id != 5;"""
+ assertTrue(iceberg_position_gen_22.size() == 5632)
+
+ List<List<Object>> iceberg_position_gen_3 = sql """select * from
iceberg_position_gen_data where name != "hello word" ;"""
+ assertTrue(iceberg_position_gen_3.size() == 5632)
+
+ List<List<Object>> iceberg_position_gen_4 = sql """select id from
iceberg_position_gen_data where id != 2;"""
+ assertTrue(iceberg_position_gen_4.size() == 5120)
+
+ List<List<Object>> iceberg_position_gen_44 = sql """select id from
iceberg_position_gen_data where id != 5;"""
+ assertTrue(iceberg_position_gen_44.size() == 5632)
+
+ List<List<Object>> iceberg_position_gen_5 = sql """select name from
iceberg_position_gen_data where id != 2;"""
+ assertTrue(iceberg_position_gen_5.size() == 5120)
+
+ List<List<Object>> iceberg_position_gen_55 = sql """select name from
iceberg_position_gen_data where id != 5;"""
+ assertTrue(iceberg_position_gen_55.size() == 5632)
+
+ List<List<Object>> iceberg_position_gen_6 = sql """select name from
iceberg_position_gen_data where name != "hello wordxx" ;"""
+ assertTrue(iceberg_position_gen_6.size() == 5632)
+
+ List<List<Object>> iceberg_position_gen_7 = sql """select id from
iceberg_position_gen_data where name != "hello word" ;"""
+ assertTrue(iceberg_position_gen_7.size() == 5632)
+
sql """drop catalog ${catalog_name}"""
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
