This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 62b8b4b3b88 [fix](be) Preserve rows for untrusted Parquet null counts
(#67573)
62b8b4b3b88 is described below
commit 62b8b4b3b8816fda32faa335f433ebabbff2f779
Author: Gabriel <[email protected]>
AuthorDate: Tue Sep 8 11:04:20 2026 +0800
[fix](be) Preserve rows for untrusted Parquet null counts (#67573)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Legacy parquet-mr writers and Arrow versions through 5 can under-report
Parquet null counts. Format v2 treated a reported zero as proof that a
row group or page had no NULL values, so `IS NULL` pruning could drop
matching rows.
### What is changed?
- Derive null-count trust from Parquet `created_by` in format v2.
- Treat the legacy `parquet-cpp` identity and `parquet-cpp-arrow`
versions before 6.0.0 as untrusted for null-count pruning.
- Conservatively retain NULL candidates at both row-group and page-index
pruning layers while preserving min/max pruning.
- Add an affected PyArrow 3.0.0 dictionary-encoded fixture and Arrow
3/4/5/6 boundary coverage.
### Release note
Avoid incorrect `IS NULL` pruning for Parquet files produced by affected
legacy writers.
### Check List (For Author)
- Test: Unit Test
- `NativeParquetStatisticsTest.*:ParquetScanTest.*Null*` (24 tests
passed with ASAN)
- Behavior changed: Yes. Format v2 no longer trusts null counts from
affected legacy Parquet and Arrow writers for metadata pruning.
- Does this need documentation: No
---
be/src/format_v2/parquet/parquet_statistics.cpp | 67 ++++++++-----
be/src/format_v2/parquet/parquet_statistics.h | 3 +-
.../parquet/reader/native/column_chunk_reader.cpp | 10 +-
.../parquet/reader/native/column_chunk_reader.h | 2 +
be/test/format_v2/parquet/parquet_scan_test.cpp | 48 +++++++++
.../format_v2/parquet/parquet_statistics_test.cpp | 107 +++++++++++++++++++++
.../pyarrow-3.0.0-dictionary-null-count.parquet | Bin 0 -> 944 bytes
7 files changed, 211 insertions(+), 26 deletions(-)
diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp
b/be/src/format_v2/parquet/parquet_statistics.cpp
index 79cd23c5bb0..36acdcf8c23 100644
--- a/be/src/format_v2/parquet/parquet_statistics.cpp
+++ b/be/src/format_v2/parquet/parquet_statistics.cpp
@@ -45,6 +45,7 @@
#include "format_v2/parquet/parquet_column_schema.h"
#include "format_v2/parquet/parquet_file_context.h"
#include "format_v2/parquet/reader/native/block_split_bloom_filter.h"
+#include "format_v2/parquet/reader/native/column_chunk_reader.h"
#include "format_v2/parquet/reader/native_column_reader.h"
#include "format_v2/parquet/selection_vector.h" // count_range_rows
#include "format_v2/timestamp_statistics.h"
@@ -135,7 +136,7 @@ namespace {
bool build_native_page_statistics(const tparquet::ColumnIndex& column_index,
const ParquetColumnSchema& column_schema,
size_t page_idx,
int64_t page_rows, ParquetColumnStatistics*
page_statistics,
- const cctz::time_zone* timezone);
+ const cctz::time_zone* timezone, bool
null_count_trusted);
enum class ParquetRowGroupPruneReason {
NONE, // cannot prune; must read
@@ -976,13 +977,14 @@ std::shared_ptr<segment_v2::ZoneMap>
ParquetStatisticsUtils::MakeZoneMap(
ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics(
const ParquetColumnSchema& column_schema, const tparquet::Statistics*
statistics,
- int64_t column_value_count, const cctz::time_zone* timezone) {
+ int64_t column_value_count, const cctz::time_zone* timezone, bool
null_count_trusted) {
ParquetColumnStatistics result;
if (statistics == nullptr || column_value_count < 0) {
return result;
}
- if (statistics->__isset.null_count && statistics->null_count >
column_value_count) {
+ if (null_count_trusted && statistics->__isset.null_count &&
+ statistics->null_count > column_value_count) {
// An impossible null count makes all derived min/max and all-null
flags untrustworthy;
// disable pruning instead of turning corrupt footer metadata into
false negatives.
return result;
@@ -990,7 +992,8 @@ ParquetColumnStatistics
ParquetStatisticsUtils::TransformColumnStatistics(
const bool has_null_count = statistics->__isset.null_count &&
statistics->null_count >= 0;
const int64_t null_count = has_null_count ? statistics->null_count : 0;
- const bool has_not_null = has_null_count ? column_value_count > null_count
: true;
+ const bool has_not_null =
+ has_null_count && null_count_trusted ? column_value_count >
null_count : true;
const std::string* min_value = statistics->__isset.min_value
? &statistics->min_value
: (statistics->__isset.min ?
&statistics->min : nullptr);
@@ -1008,10 +1011,10 @@ ParquetColumnStatistics
ParquetStatisticsUtils::TransformColumnStatistics(
// Footer statistics and page indexes share the same little-endian
physical encoding. Reusing
// one decoder keeps native row-group and page pruning identical for
logical types and NaNs.
if (!build_native_page_statistics(index, column_schema, 0,
column_value_count, &result,
- timezone)) {
+ timezone, null_count_trusted)) {
return {};
}
- if (!has_null_count) {
+ if (!has_null_count || !null_count_trusted) {
result.has_null_count = false;
result.has_null = true;
}
@@ -1081,7 +1084,8 @@ bool check_native_statistics(const
tparquet::FileMetaData& metadata,
const tparquet::RowGroup& row_group,
const
std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
const format::FileScanRequest& request,
- ParquetPruningStats* pruning_stats, const
cctz::time_zone* timezone) {
+ ParquetPruningStats* pruning_stats, const
cctz::time_zone* timezone,
+ bool null_count_trusted) {
const auto conjuncts = metadata_pruning_conjuncts(request);
const auto add_column_zonemap = [&](ZoneMapEvalContext* ctx, int
slot_index,
const ParquetColumnSchema*
column_schema) {
@@ -1106,7 +1110,7 @@ bool check_native_statistics(const
tparquet::FileMetaData& metadata,
ParquetStatisticsUtils::TransformColumnStatistics(
*column_schema,
safe_statistics.has_value() ? &*safe_statistics :
nullptr,
- column_metadata.num_values, timezone));
+ column_metadata.num_values, timezone,
null_count_trusted));
}
add_slot_zonemap(ctx, slot_index, column_schema->type,
std::move(zone_map));
};
@@ -1163,7 +1167,8 @@ bool check_native_statistics(const
tparquet::FileMetaData& metadata,
bool check_shredded_variant_statistics(
const tparquet::FileMetaData& metadata, const tparquet::RowGroup&
row_group,
const std::vector<std::unique_ptr<ParquetColumnSchema>>& file_schema,
- const format::FileScanRequest& request, const cctz::time_zone*
timezone) {
+ const format::FileScanRequest& request, const cctz::time_zone*
timezone,
+ bool null_count_trusted) {
for (const auto& conjunct : metadata_pruning_conjuncts(request)) {
const auto predicate = extract_variant_shredded_predicate(conjunct);
if (!predicate.has_value()) {
@@ -1193,7 +1198,7 @@ bool check_shredded_variant_statistics(
}
const auto statistics =
ParquetStatisticsUtils::TransformColumnStatistics(
*shredding->typed_value, safe_statistics.has_value() ?
&*safe_statistics : nullptr,
- column_metadata.num_values, timezone);
+ column_metadata.num_values, timezone, null_count_trusted);
const auto normalized =
normalize_variant_statistics(*predicate,
*shredding->typed_value, statistics);
if (normalized.has_value() && variant_statistics_exclude(*predicate,
*normalized)) {
@@ -1501,6 +1506,9 @@ Status select_row_groups_by_metadata(
DORIS_CHECK(column != nullptr);
return column->contains_variant;
});
+ const bool null_count_trusted =
+ native::parquet_reader_compat(metadata.__isset.created_by ?
metadata.created_by : "")
+ .null_count_trusted;
selected_row_groups->reserve(candidate_size);
for (size_t candidate_idx = 0; candidate_idx < candidate_size;
++candidate_idx) {
const int row_group_idx = candidate_row_groups == nullptr
@@ -1526,9 +1534,10 @@ Status select_row_groups_by_metadata(
if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY &&
has_expr_zonemap_filter(request, runtime_state) &&
(check_native_statistics(metadata, row_group, file_schema,
request, pruning_stats,
- timezone) ||
- (contains_variant && check_shredded_variant_statistics(
- metadata, row_group, file_schema,
request, timezone)))) {
+ timezone, null_count_trusted) ||
+ (contains_variant &&
+ check_shredded_variant_statistics(metadata, row_group,
file_schema, request, timezone,
+ null_count_trusted)))) {
prune_reason = ParquetRowGroupPruneReason::STATISTICS;
}
if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY &&
@@ -1783,7 +1792,7 @@ bool set_native_page_boolean_min_max(const
tparquet::ColumnIndex& column_index,
bool build_native_page_statistics(const tparquet::ColumnIndex& column_index,
const ParquetColumnSchema& column_schema,
size_t page_idx,
int64_t page_rows, ParquetColumnStatistics*
page_statistics,
- const cctz::time_zone* timezone) {
+ const cctz::time_zone* timezone, bool
null_count_trusted) {
DORIS_CHECK(page_statistics != nullptr);
*page_statistics = {};
if (!column_index.__isset.null_counts || page_idx >=
column_index.null_pages.size() ||
@@ -1792,14 +1801,14 @@ bool build_native_page_statistics(const
tparquet::ColumnIndex& column_index,
}
const int64_t null_count = column_index.null_counts[page_idx];
const bool all_null = column_index.null_pages[page_idx];
- if (page_rows < 0 || null_count < 0 || null_count > page_rows ||
- all_null != (null_count == page_rows)) {
+ if (page_rows < 0 || (null_count_trusted && (null_count < 0 || null_count
> page_rows ||
+ all_null != (null_count ==
page_rows)))) {
// The caller supplies the exact flat page or row-group span.
Contradictory optional null
// metadata must disable pruning instead of turning a partial span
into an all-null proof.
return false;
}
- page_statistics->has_null_count = true;
- page_statistics->has_null = null_count > 0;
+ page_statistics->has_null_count = null_count_trusted;
+ page_statistics->has_null = !null_count_trusted || null_count > 0;
page_statistics->has_not_null = !all_null;
if (!page_statistics->has_not_null) {
return true;
@@ -1875,14 +1884,16 @@ public:
const std::unordered_map<int, NativeParquetPageIndex>&
page_indexes,
const std::vector<std::unique_ptr<ParquetColumnSchema>>&
file_schema,
const format::FileScanRequest& request, int64_t row_group_rows,
- ParquetPruningStats* pruning_stats, const cctz::time_zone*
timezone)
+ ParquetPruningStats* pruning_stats, const cctz::time_zone*
timezone,
+ bool null_count_trusted)
: _metadata(metadata),
_page_indexes(page_indexes),
_file_schema(file_schema),
_request(request),
_row_group_rows(row_group_rows),
_pruning_stats(pruning_stats),
- _timezone(timezone) {}
+ _timezone(timezone),
+ _null_count_trusted(null_count_trusted) {}
std::optional<std::vector<RowRange>> evaluate(const VExprSPtr& expr) const
{
if (expr == nullptr || !expr->can_evaluate_zonemap_filter()) {
@@ -1995,7 +2006,8 @@ private:
native_page_row_range(indexes.offset_index, page_idx,
_row_group_rows);
ParquetColumnStatistics statistics;
if (!build_native_page_statistics(indexes.column_index,
*column_schema, page_idx,
- page_range.length, &statistics,
_timezone)) {
+ page_range.length, &statistics,
_timezone,
+ _null_count_trusted)) {
_slot_page_zone_maps.emplace(slot_index, std::nullopt);
return nullptr;
}
@@ -2013,6 +2025,7 @@ private:
int64_t _row_group_rows;
ParquetPruningStats* _pruning_stats;
const cctz::time_zone* _timezone;
+ bool _null_count_trusted;
mutable std::unordered_map<int, std::optional<SlotPageZoneMaps>>
_slot_page_zone_maps;
};
@@ -2042,6 +2055,9 @@ Status select_row_group_ranges_by_native_page_index(
if (pruning_stats != nullptr) {
++pruning_stats->page_index_read_calls;
}
+ const bool null_count_trusted =
+ native::parquet_reader_compat(metadata.__isset.created_by ?
metadata.created_by : "")
+ .null_count_trusted;
std::map<int, VExprContextSPtrs> conjuncts_by_slot;
VExprContextSPtrs multi_slot_conjuncts;
@@ -2079,7 +2095,8 @@ Status select_row_group_ranges_by_native_page_index(
native_page_row_range(indexes.offset_index, page_idx,
row_group_rows);
ParquetColumnStatistics statistics;
if (!build_native_page_statistics(indexes.column_index,
*column_schema, page_idx,
- page_range.length, &statistics,
timezone)) {
+ page_range.length, &statistics,
timezone,
+ null_count_trusted)) {
usable = false;
break;
}
@@ -2106,7 +2123,8 @@ Status select_row_group_ranges_by_native_page_index(
}
NativePageIndexPredicateEvaluator evaluator(metadata, page_indexes,
file_schema, request,
- row_group_rows, pruning_stats,
timezone);
+ row_group_rows, pruning_stats,
timezone,
+ null_count_trusted);
for (const auto& conjunct : multi_slot_conjuncts) {
auto conjunct_ranges = evaluator.evaluate(conjunct->root());
if (!conjunct_ranges.has_value()) {
@@ -2148,7 +2166,8 @@ Status select_row_group_ranges_by_native_page_index(
native_page_row_range(indexes.offset_index, page_idx,
row_group_rows);
ParquetColumnStatistics statistics;
if (!build_native_page_statistics(indexes.column_index,
*shredding->typed_value,
- page_idx, page_range.length,
&statistics, timezone)) {
+ page_idx, page_range.length,
&statistics, timezone,
+ null_count_trusted)) {
usable = false;
break;
}
diff --git a/be/src/format_v2/parquet/parquet_statistics.h
b/be/src/format_v2/parquet/parquet_statistics.h
index cd758ebb909..20cae0b4060 100644
--- a/be/src/format_v2/parquet/parquet_statistics.h
+++ b/be/src/format_v2/parquet/parquet_statistics.h
@@ -131,7 +131,8 @@ struct ParquetStatisticsUtils {
static ParquetColumnStatistics TransformColumnStatistics(
const ParquetColumnSchema& column_schema, const
tparquet::Statistics* statistics,
- int64_t column_value_count, const cctz::time_zone* timezone =
nullptr);
+ int64_t column_value_count, const cctz::time_zone* timezone =
nullptr,
+ bool null_count_trusted = true);
static bool NativeBloomFilterExcludes(const ParquetColumnSchema&
column_schema, int slot_index,
const VExprContextSPtrs& conjuncts,
diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
index d1677c66676..89df1d75f86 100644
--- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
+++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp
@@ -189,10 +189,18 @@ ParquetReaderCompat parquet_reader_compat(const
std::string& created_by) {
return {};
}
const ::parquet::ApplicationVersion version(created_by);
+ const ::parquet::ApplicationVersion
arrow_fixed_null_count_version("parquet-cpp-arrow", 6, 0,
+ 0);
return {.parquet_816_padding =
version.VersionLt(::parquet::ApplicationVersion::PARQUET_816_FIXED_VERSION()),
.data_page_v2_always_compressed = version.VersionLt(
-
::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION())};
+
::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION()),
+ // Arrow <= 3 used parquet-cpp, then Arrow 4/5 used
parquet-cpp-arrow before the fix.
+ .null_count_trusted =
+ version.application_ != "parquet-cpp" &&
+ !version.VersionLt(arrow_fixed_null_count_version) &&
+ !version.VersionLt(
+
::parquet::ApplicationVersion::PARQUET_MR_FIXED_STATS_VERSION())};
}
Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata,
size_t file_size,
diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
index e866e8b1c49..a7701b7ec34 100644
--- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
+++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h
@@ -63,6 +63,8 @@ struct ColumnChunkRange {
struct ParquetReaderCompat {
bool parquet_816_padding = false;
bool data_page_v2_always_compressed = false;
+ // Legacy parquet-mr and Arrow writers can under-report null_count.
+ bool null_count_trusted = true;
};
ParquetReaderCompat parquet_reader_compat(const std::string& created_by);
diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp
b/be/test/format_v2/parquet/parquet_scan_test.cpp
index d12f78c9b10..17d80897b7a 100644
--- a/be/test/format_v2/parquet/parquet_scan_test.cpp
+++ b/be/test/format_v2/parquet/parquet_scan_test.cpp
@@ -26,6 +26,7 @@
#include <parquet/encoding.h>
#include <bit>
+#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <fstream>
@@ -3405,6 +3406,53 @@ TEST_F(ParquetScanTest,
PredicateOnlyPlainStringNullPredicatesUseDefinitionLevel
}
}
+TEST_F(ParquetScanTest, PyArrow3DictionaryNullCountFixturePreservesIsNullRows)
{
+ const char* source_root = std::getenv("ROOT");
+ ASSERT_NE(source_root, nullptr);
+ const auto fixture_path =
+ std::filesystem::path(source_root) /
+
"be/test/format_v2/parquet/test_data/pyarrow-3.0.0-dictionary-null-count.parquet";
+ ASSERT_TRUE(std::filesystem::copy_file(fixture_path, _file_path));
+
+ const auto fixture_reader =
::parquet::ParquetFileReader::OpenFile(_file_path, false);
+ const auto fixture_metadata = fixture_reader->metadata();
+ ASSERT_EQ(fixture_metadata->created_by(), "parquet-cpp version
1.5.1-SNAPSHOT");
+ const auto fixture_statistics =
fixture_metadata->RowGroup(0)->ColumnChunk(0)->statistics();
+ ASSERT_NE(fixture_statistics, nullptr);
+ ASSERT_EQ(fixture_statistics->null_count(), 0);
+
+ RuntimeProfile profile("profile");
+ auto reader = create_reader(0, -1, &profile);
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+ auto request = std::make_shared<format::FileScanRequest>();
+ format::FileScanRequestBuilder request_builder(request.get());
+
ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok());
+
ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok());
+ request->predicate_only_columns.push_back(format::LocalColumnId(0));
+ auto conjunct = create_string_null_conjunct(0, true);
+ ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok());
+ ASSERT_TRUE(conjunct->open(&state).ok());
+ request->conjuncts.push_back(conjunct);
+ ASSERT_TRUE(reader->open(request).ok());
+
+ ColumnInt32::Container actual;
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block(schema);
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ const auto& ids =
int32_data_column(*block.get_by_position(1).column).get_data();
+ actual.insert(actual.end(), ids.begin(), ids.end());
+ }
+ EXPECT_EQ(actual, (ColumnInt32::Container {0, 3, 6, 9, 12}));
+ conjunct->close();
+}
+
TEST_F(ParquetScanTest,
PredicateOnlyDictionaryIntNullPredicatesUseDefinitionLevels) {
for (const bool is_null : {true, false}) {
write_dictionary_int_pair_parquet_file(_file_path);
diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp
b/be/test/format_v2/parquet/parquet_statistics_test.cpp
index 4aa996ab00c..9f750faddda 100644
--- a/be/test/format_v2/parquet/parquet_statistics_test.cpp
+++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp
@@ -408,6 +408,29 @@ private:
const std::string _expr_name = "MetadataInt32GreaterThanExpr";
};
+class MetadataIsNullExpr final : public VExpr {
+public:
+ MetadataIsNullExpr() : VExpr(std::make_shared<DataTypeUInt8>(), false) {}
+
+ const std::string& expr_name() const override { return _expr_name; }
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
+ ColumnPtr&) const override {
+ return Status::InternalError("MetadataIsNullExpr is metadata-only");
+ }
+ bool can_evaluate_zonemap_filter() const override { return true; }
+ void collect_slot_column_ids(std::set<int>& column_ids) const override {
column_ids.insert(0); }
+ ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx)
const override {
+ const auto zone_map = ctx.zone_map(0);
+ if (zone_map == nullptr) {
+ return unsupported_zonemap_filter(ctx);
+ }
+ return zone_map->has_null ? ZoneMapFilterResult::kMayMatch :
ZoneMapFilterResult::kNoMatch;
+ }
+
+private:
+ const std::string _expr_name = "MetadataIsNullExpr";
+};
+
class MetadataSlotInt32GreaterThanExpr final : public VExpr {
public:
MetadataSlotInt32GreaterThanExpr(int slot_index, int32_t value)
@@ -1859,6 +1882,90 @@ TEST(NativeParquetStatisticsTest,
TypeDefinedBoundsRequireSupportedColumnOrder)
EXPECT_TRUE(selected_ranges.empty());
}
+TEST(NativeParquetStatisticsTest, WriterVersionControlsNullCountPruningTrust) {
+ const auto encode_int32 = [](int32_t value) {
+ std::string bytes(sizeof(value), '\0');
+ memcpy(bytes.data(), &value, sizeof(value));
+ return bytes;
+ };
+
+ auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ column_schema->local_id = 0;
+ column_schema->leaf_column_id = 0;
+ column_schema->type = make_nullable(std::make_shared<DataTypeInt32>());
+ column_schema->type_descriptor.doris_type = column_schema->type;
+ column_schema->type_descriptor.physical_type = tparquet::Type::INT32;
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
+ schema.push_back(std::move(column_schema));
+
+ tparquet::Statistics statistics;
+ statistics.__set_min_value(encode_int32(1));
+ statistics.__set_max_value(encode_int32(3));
+ statistics.__set_null_count(0);
+ tparquet::ColumnMetaData column_metadata;
+ column_metadata.__set_type(tparquet::Type::INT32);
+ column_metadata.__set_num_values(3);
+ column_metadata.__set_statistics(statistics);
+ tparquet::ColumnChunk chunk;
+ chunk.__set_meta_data(column_metadata);
+ tparquet::RowGroup row_group;
+ row_group.__set_columns({chunk});
+ row_group.__set_num_rows(3);
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order});
+ metadata.__set_row_groups({row_group});
+
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ request.predicate_columns =
{format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ request.conjuncts =
{VExprContext::create_shared(std::make_shared<MetadataIsNullExpr>())};
+
+ format::parquet::NativeParquetPageIndex page_index;
+ page_index.column_index.__set_min_values({encode_int32(1)});
+ page_index.column_index.__set_max_values({encode_int32(3)});
+ page_index.column_index.__set_null_pages({false});
+ page_index.column_index.__set_null_counts({0});
+ tparquet::PageLocation location;
+ location.__set_offset(0);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(0);
+ page_index.offset_index.__set_page_locations({location});
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex>
page_indexes;
+ page_indexes.emplace(0, std::move(page_index));
+
+ const auto check_writer = [&](const std::string& created_by, bool
expect_selected) {
+ SCOPED_TRACE(created_by);
+ metadata.__set_created_by(created_by);
+ std::vector<int> selected_row_groups;
+ ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata,
schema, request,
+ nullptr,
&selected_row_groups,
+ false,
nullptr)
+ .ok());
+ EXPECT_EQ(!selected_row_groups.empty(), expect_selected);
+
+ std::vector<format::parquet::RowRange> selected_ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
+
ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, metadata.row_groups[0], page_indexes,
schema, request, 3,
+ &selected_ranges, &skip_plans, nullptr)
+ .ok());
+ EXPECT_EQ(!selected_ranges.empty(), expect_selected);
+ };
+
+ check_writer("parquet-mr version 1.9.0-cdh6.3.2 (build test)", true);
+ check_writer("parquet-mr version 1.13.1 (build test)", false);
+ check_writer("parquet-cpp version 1.2.8 (build test)", true);
+ check_writer("parquet-cpp version 1.5.1-SNAPSHOT", true);
+ check_writer("parquet-cpp-arrow version 4.0.1", true);
+ check_writer("parquet-cpp-arrow version 5.0.0", true);
+ check_writer("parquet-cpp-arrow version 6.0.0", false);
+ check_writer("custom-writer version 1.0.0 (build test)", false);
+ check_writer("", false);
+}
+
TEST(NativeParquetStatisticsTest,
RuntimeFilterWrapperKeepsScalarPageIndexPruning) {
auto encode_int32 = [](int32_t value) {
std::string bytes(sizeof(value), '\0');
diff --git
a/be/test/format_v2/parquet/test_data/pyarrow-3.0.0-dictionary-null-count.parquet
b/be/test/format_v2/parquet/test_data/pyarrow-3.0.0-dictionary-null-count.parquet
new file mode 100644
index 00000000000..a5173a34309
Binary files /dev/null and
b/be/test/format_v2/parquet/test_data/pyarrow-3.0.0-dictionary-null-count.parquet
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
