This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5307f3ccd34 [fix](paimon) Read 64-bit deletion vectors in native scans
(#70401)
5307f3ccd34 is described below
commit 5307f3ccd34314152378cb18b83064cd6c1f73da
Author: Socrates <[email protected]>
AuthorDate: Wed Nov 18 10:02:22 2026 +0800
[fix](paimon) Read 64-bit deletion vectors in native scans (#70401)
### Background
Paimon stores deleted row positions as bitmap entries in index files.
During a native scan, Doris reads the entry for each split and applies
its bitmap as a row-position delete predicate.
### What problem does this PR solve?
Issue Number: None
Related PR: None
PR Type: Bugfix
Problem Summary: Native Doris scans fail on Paimon bitmap64 deletion
vectors. Doris treats every descriptor length as bitmap32 and adds four
bytes, so a bitmap64 entry ending at EOF fails the file-range check.
This change identifies the format from each entry header and applies the
corresponding bitmap, including when an index contains both versions.
### Details
- **Frequency:** A native scan fails whenever it must apply a bitmap64
entry.
- **Root cause:** Bitmap32 descriptor lengths count the magic and
bitmap. Bitmap64 descriptor lengths also count the prefix and checksum.
The previous reader assumed bitmap32 length semantics and decoded only
Roaring32.
- **Why existing tests missed it:** Native reader coverage used bitmap32
fixtures.
- **Fix:** Validate the descriptor and physical range using the entry
header, then use the shared Roaring64 and CRC32 decoder for bitmap64.
Keep the decoded bitmap in the query-local cache after successful
validation.
- **Other paths and branches:** This change covers the native
FileScannerV2 path on master. The JNI reader serves as the reference in
the Fedora smoke test.
- **Behavior change:** Native Paimon scans can apply bitmap64 deletion
vectors. The first read of a DV on a decoded-cache miss reads the header
and bitmap separately.
### Testing
- Fedora x86_64 ASAN BE build completed.
- Fedora SQL smoke test with Hive Metastore and Paimon 2.1.0. Both
bitmap32 and bitmap64 scans selected the native reader and returned
`(1,10)` and `(3,30)`, matching the JNI reader. The original bitmap64
query failed before the fix.
- Fedora `run-be-ut.sh --run
--filter='PaimonReaderTest.*:IcebergDeleteFileReaderHelperTest.*'` with
ASAN_UT passed all 40 tests (19 Paimon, 21 Iceberg). The local build
scoped the test target to these suites and their test support sources.
- Clang-format 16, build hygiene, and changed production lines checked
with clang-tidy.
- The existing `test_paimon_deletion_vector` suite passed on Fedora
against HDFS. Its bitmap64 case creates a physical deletion vector,
confirms native scan planning, and verifies the two surviving rows. The
baseline BE fails with the DV range error when querying the same HDFS
table.
### Release note
Native Paimon scans support 64-bit deletion vectors, including snapshots
that also reference 32-bit vectors.
### Check List (For Author)
- Behavior changed: Yes (see Details)
- Does this need documentation: No
- Pre-submission status: Ready
### AI Disclosure
- Tool: Codex (GPT-6)
- Generated: Implementation, BE unit tests, and this description
- Human review: Pending
---
be/src/format/table/deletion_vector.h | 4 +-
be/src/format/table/deletion_vector_reader.cpp | 42 +++---
be/src/format/table/deletion_vector_reader.h | 6 +-
be/src/format/table/paimon_deletion_vector.cpp | 82 +++++++++--
be/src/format/table/paimon_deletion_vector.h | 10 +-
be/src/format_v2/table/paimon_reader.cpp | 5 +-
be/src/format_v2/table_reader.cpp | 15 +-
be/test/format_v2/table/paimon_reader_test.cpp | 157 ++++++++++++++++++---
.../paimon/test_paimon_deletion_vector.out | 4 +
.../paimon/test_paimon_deletion_vector.groovy | 35 +++++
10 files changed, 284 insertions(+), 76 deletions(-)
diff --git a/be/src/format/table/deletion_vector.h
b/be/src/format/table/deletion_vector.h
index 12ec7dbc6eb..e1e10655a73 100644
--- a/be/src/format/table/deletion_vector.h
+++ b/be/src/format/table/deletion_vector.h
@@ -29,8 +29,8 @@ namespace doris {
inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30;
inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12;
-// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum
serialized size is below this
-// limit. Keep its guard independent from the Iceberg capability limit.
+// Paimon bitmap32 fits below this limit. Bitmap64 may exceed it, so check the
serialized size
+// before allocating a read buffer. Keep the Paimon limit independent from the
Iceberg limit.
inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30;
// A deletion vector is already a bitmap on the wire. Keep decoded DVs
compressed in the
diff --git a/be/src/format/table/deletion_vector_reader.cpp
b/be/src/format/table/deletion_vector_reader.cpp
index 946e81fa2e6..64d1d8b5020 100644
--- a/be/src/format/table/deletion_vector_reader.cpp
+++ b/be/src/format/table/deletion_vector_reader.cpp
@@ -30,7 +30,6 @@ namespace {
constexpr int64_t ICEBERG_DELETION_VECTOR_MIN_BYTES =
static_cast<int64_t>(ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES);
constexpr int64_t PAIMON_DELETION_VECTOR_MIN_BYTES = 8;
-constexpr int64_t PAIMON_LENGTH_PREFIX_BYTES = 4;
enum class DeletionVectorSizeLimitStatus {
DATA_QUALITY,
@@ -39,8 +38,7 @@ enum class DeletionVectorSizeLimitStatus {
Status validate_deletion_vector_read_range(const char* description, int64_t
offset, int64_t size,
int64_t min_size, int64_t max_size,
- DeletionVectorSizeLimitStatus
size_limit_status,
- size_t& bytes_read) {
+ DeletionVectorSizeLimitStatus
size_limit_status) {
if (offset < 0) {
return Status::DataQualityError("{} offset must be non-negative: {}",
description, offset);
}
@@ -60,7 +58,6 @@ Status validate_deletion_vector_read_range(const char*
description, int64_t offs
return Status::DataQualityError("{} offset plus size overflows: offset
{}, size {}",
description, offset, size);
}
- bytes_read = static_cast<size_t>(size);
return Status::OK();
}
@@ -82,25 +79,17 @@ Status missing_file_as_error(Status status, const
std::string& path) {
Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t
size,
size_t& bytes_read) {
- return validate_deletion_vector_read_range(
+ RETURN_IF_ERROR(validate_deletion_vector_read_range(
"Iceberg deletion vector", offset, size,
ICEBERG_DELETION_VECTOR_MIN_BYTES,
- MAX_ICEBERG_DELETION_VECTOR_BYTES,
DeletionVectorSizeLimitStatus::NOT_SUPPORTED,
- bytes_read);
+ MAX_ICEBERG_DELETION_VECTOR_BYTES,
DeletionVectorSizeLimitStatus::NOT_SUPPORTED));
+ bytes_read = static_cast<size_t>(size);
+ return Status::OK();
}
-Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t
length,
- size_t& bytes_read) {
- if (length < 0) {
- return Status::DataQualityError("Paimon deletion vector length must be
non-negative: {}",
- length);
- }
- if (length > std::numeric_limits<int64_t>::max() -
PAIMON_LENGTH_PREFIX_BYTES) {
- return Status::DataQualityError("Paimon deletion vector length
overflows: {}", length);
- }
+Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t
size) {
return validate_deletion_vector_read_range(
- "Paimon deletion vector", offset, length +
PAIMON_LENGTH_PREFIX_BYTES,
- PAIMON_DELETION_VECTOR_MIN_BYTES, MAX_PAIMON_DELETION_VECTOR_BYTES,
- DeletionVectorSizeLimitStatus::DATA_QUALITY, bytes_read);
+ "Paimon deletion vector", offset, size,
PAIMON_DELETION_VECTOR_MIN_BYTES,
+ MAX_PAIMON_DELETION_VECTOR_BYTES,
DeletionVectorSizeLimitStatus::DATA_QUALITY);
}
DeletionVectorReader::~DeletionVectorReader() {
@@ -151,16 +140,19 @@ Status DeletionVectorReader::open() {
_init_file_description();
RETURN_IF_ERROR(missing_file_as_error(_create_file_reader(), _desc.path));
- const size_t file_size = _file_reader->size();
- const size_t start_offset = static_cast<size_t>(_desc.start_offset);
- const size_t range_size = static_cast<size_t>(_desc.size);
- if (start_offset > file_size || range_size > file_size - start_offset) {
+ RETURN_IF_ERROR(validate_file_range(_desc.start_offset, _desc.size));
+ _is_opened = true;
+ return Status::OK();
+}
+
+Status DeletionVectorReader::validate_file_range(size_t offset, size_t size)
const {
+ const size_t length = file_size();
+ if (offset > length || size > length - offset) {
return Status::DataQualityError(
"Deletion vector range exceeds file size: path {}, offset {},
size {}, file size "
"{}",
- _desc.path, _desc.start_offset, _desc.size, file_size);
+ _desc.path, offset, size, length);
}
- _is_opened = true;
return Status::OK();
}
diff --git a/be/src/format/table/deletion_vector_reader.h
b/be/src/format/table/deletion_vector_reader.h
index c2ed07a242a..2c75c7e8a58 100644
--- a/be/src/format/table/deletion_vector_reader.h
+++ b/be/src/format/table/deletion_vector_reader.h
@@ -41,8 +41,7 @@ namespace doris {
Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t
size,
size_t& bytes_read);
-Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t
length,
- size_t& bytes_read);
+Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t
size);
struct DeleteFileDesc {
enum class Format {
@@ -54,6 +53,7 @@ struct DeleteFileDesc {
std::string path = "";
std::string fs_name = "";
int64_t start_offset = 0;
+ // Paimon bitmap32 descriptor length excludes the length prefix and
checksum.
int64_t size = 0;
int64_t file_size = -1;
int64_t modification_time = 0;
@@ -89,6 +89,8 @@ public:
DeletionVectorReader(const DeletionVectorReader&) = delete;
DeletionVectorReader& operator=(const DeletionVectorReader&) = delete;
Status open();
+ size_t file_size() const { return _file_reader->size(); }
+ Status validate_file_range(size_t offset, size_t size) const;
Status read_at(size_t offset, Slice result);
// These are deliberately exposed separately from the decoded-DV cache
counters. Local I/O is
diff --git a/be/src/format/table/paimon_deletion_vector.cpp
b/be/src/format/table/paimon_deletion_vector.cpp
index a9adb8ae28e..f2a2f48ae56 100644
--- a/be/src/format/table/paimon_deletion_vector.cpp
+++ b/be/src/format/table/paimon_deletion_vector.cpp
@@ -19,18 +19,23 @@
#include <fmt/format.h>
-#include <cstring>
+#include <algorithm>
#include <stdexcept>
#include "common/status.h"
#include "exec/common/endian.h"
#include "format/table/deletion_vector_reader.h"
+#include "format/table/iceberg_delete_file_reader_helper.h"
namespace doris {
namespace {
-constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'};
+constexpr uint32_t PAIMON_BITMAP32_MAGIC = 0x5e43f2d0;
+constexpr uint32_t PAIMON_BITMAP64_MAGIC = 0xd1d33964;
+constexpr size_t PAIMON_DV_HEADER_BYTES = 8;
+constexpr int64_t PAIMON_DV_LENGTH_PREFIX_BYTES = 4;
+constexpr int64_t PAIMON_DV_CRC_BYTES = 4;
} // namespace
@@ -39,15 +44,56 @@ std::string build_paimon_deletion_vector_cache_key(const
TPaimonDeletionFileDesc
deletion_file.length);
}
-Status validate_paimon_deletion_vector_descriptor(const
TPaimonDeletionFileDesc& deletion_file,
- size_t& bytes_read) {
+Status validate_paimon_deletion_vector_descriptor(const
TPaimonDeletionFileDesc& deletion_file) {
if (!deletion_file.__isset.path || !deletion_file.__isset.offset ||
!deletion_file.__isset.length) {
return Status::DataQualityError(
"Paimon deletion file descriptor misses path/offset/length");
}
- return validate_paimon_deletion_vector_read_range(deletion_file.offset,
deletion_file.length,
- bytes_read);
+ return validate_paimon_deletion_vector_read_range(deletion_file.offset,
deletion_file.length);
+}
+
+Status read_paimon_deletion_vector_buffer(DeletionVectorReader& reader, const
DeleteFileDesc& desc,
+ std::vector<char>& buffer) {
+ const auto offset = static_cast<size_t>(desc.start_offset);
+ // An 8-byte header is within the validated descriptor range. Read enough
for bitmap32's
+ // extra prefix, capped at EOF so a bitmap64 entry at the end needs no
trailing bytes.
+ const size_t bytes_to_read = std::min(
+ {static_cast<size_t>(desc.size + PAIMON_DV_LENGTH_PREFIX_BYTES),
+ static_cast<size_t>(MAX_PAIMON_DELETION_VECTOR_BYTES),
reader.file_size() - offset});
+ buffer.resize(bytes_to_read);
+ RETURN_IF_ERROR(reader.read_at(offset, {buffer.data(), buffer.size()}));
+ const uint32_t payload_size = BigEndian::Load32(buffer.data());
+ const uint32_t magic = BigEndian::Load32(buffer.data() +
PAIMON_DV_LENGTH_PREFIX_BYTES);
+ // The length prefix counts the magic itself, even for an empty bitmap.
+ if (payload_size < sizeof(magic)) {
+ return Status::DataQualityError(
+ "Paimon deletion vector payload too small: path {}, offset {},
size {}", desc.path,
+ desc.start_offset, payload_size);
+ }
+ if (magic != PAIMON_BITMAP32_MAGIC && magic != PAIMON_BITMAP64_MAGIC) {
+ return Status::DataQualityError(
+ "Paimon deletion vector magic number mismatch: path {}, offset
{}, magic {}",
+ desc.path, desc.start_offset, magic);
+ }
+
+ // Paimon may mix bitmap32 and bitmap64 entries in one index file. The
bitmap32 descriptor
+ // counts only magic and bitmap, while bitmap64 counts the prefix and CRC
as well.
+ const bool bitmap64 = magic == PAIMON_BITMAP64_MAGIC;
+ const int64_t entry_size =
+ payload_size + PAIMON_DV_LENGTH_PREFIX_BYTES + (bitmap64 ?
PAIMON_DV_CRC_BYTES : 0);
+ const int64_t expected_length = bitmap64 ? entry_size : payload_size;
+ if (desc.size != expected_length) {
+ return Status::DataQualityError(
+ "Paimon deletion vector descriptor length mismatch: path {},
offset {}, expected "
+ "{}, actual {}",
+ desc.path, desc.start_offset, expected_length, desc.size);
+ }
+
RETURN_IF_ERROR(validate_paimon_deletion_vector_read_range(desc.start_offset,
entry_size));
+ RETURN_IF_ERROR(reader.validate_file_range(offset, entry_size));
+ // A bitmap64 entry followed by another entry may have up to four extra
bytes in the buffer.
+ buffer.resize(entry_size);
+ return Status::OK();
}
Status decode_paimon_deletion_vector_buffer(const char* buf, size_t
buffer_size,
@@ -58,27 +104,35 @@ Status decode_paimon_deletion_vector_buffer(const char*
buf, size_t buffer_size,
if (buf == nullptr) {
return Status::DataQualityError("Paimon deletion vector blob is null");
}
- if (buffer_size < 8) [[unlikely]] {
+ if (buffer_size < PAIMON_DV_HEADER_BYTES) [[unlikely]] {
return Status::DataQualityError("Deletion vector file size too small:
{}", buffer_size);
}
+ const uint32_t magic = BigEndian::Load32(buf +
PAIMON_DV_LENGTH_PREFIX_BYTES);
+ if (magic == PAIMON_BITMAP64_MAGIC) {
+ // Bitmap64 uses the same on-disk framing and portable Roaring64
layout as Iceberg DVs.
+ return decode_iceberg_deletion_vector_buffer(buf, buffer_size,
deletion_vector);
+ }
+
const uint32_t actual_length = BigEndian::Load32(buf);
- if (static_cast<uint64_t>(actual_length) + 4 != buffer_size) [[unlikely]] {
+ const uint64_t expected_size =
+ static_cast<uint64_t>(actual_length) +
PAIMON_DV_LENGTH_PREFIX_BYTES;
+ if (expected_size != buffer_size) [[unlikely]] {
return Status::DataQualityError(
- "Paimon deletion vector length mismatch, expected: {}, actual:
{}",
- static_cast<uint64_t>(actual_length) + 4, buffer_size);
+ "Paimon deletion vector length mismatch, expected: {}, actual:
{}", expected_size,
+ buffer_size);
}
- if (memcmp(buf + sizeof(actual_length), PAIMON_BITMAP_MAGIC, 4) != 0)
[[unlikely]] {
+ if (magic != PAIMON_BITMAP32_MAGIC) [[unlikely]] {
return Status::DataQualityError(
"Paimon deletion vector magic number mismatch, expected: {},
actual: {}",
- BigEndian::Load32(PAIMON_BITMAP_MAGIC),
- BigEndian::Load32(buf + sizeof(actual_length)));
+ PAIMON_BITMAP32_MAGIC, magic);
}
roaring::Roaring roaring_bitmap;
try {
- roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8);
+ roaring_bitmap = roaring::Roaring::readSafe(buf +
PAIMON_DV_HEADER_BYTES,
+ buffer_size -
PAIMON_DV_HEADER_BYTES);
} catch (const std::runtime_error& e) {
return Status::RuntimeError(
"DeletionVector deserialize error: failed to deserialize
roaring bitmap, {}",
diff --git a/be/src/format/table/paimon_deletion_vector.h
b/be/src/format/table/paimon_deletion_vector.h
index 4f809c2de01..afca2ca5a6f 100644
--- a/be/src/format/table/paimon_deletion_vector.h
+++ b/be/src/format/table/paimon_deletion_vector.h
@@ -19,6 +19,7 @@
#include <cstddef>
#include <string>
+#include <vector>
#include "common/status.h"
#include "format/table/deletion_vector.h"
@@ -26,9 +27,14 @@
namespace doris {
+class DeletionVectorReader;
+struct DeleteFileDesc;
+
std::string build_paimon_deletion_vector_cache_key(const
TPaimonDeletionFileDesc& deletion_file);
-Status validate_paimon_deletion_vector_descriptor(const
TPaimonDeletionFileDesc& deletion_file,
- size_t& bytes_read);
+Status validate_paimon_deletion_vector_descriptor(const
TPaimonDeletionFileDesc& deletion_file);
+// The reader is open and the Paimon descriptor range has been validated.
+Status read_paimon_deletion_vector_buffer(DeletionVectorReader& reader, const
DeleteFileDesc& desc,
+ std::vector<char>& buffer);
Status decode_paimon_deletion_vector_buffer(const char* buf, size_t
buffer_size,
DeletionVector* deletion_vector);
diff --git a/be/src/format_v2/table/paimon_reader.cpp
b/be/src/format_v2/table/paimon_reader.cpp
index 1ac1a3aa4e6..408da128669 100644
--- a/be/src/format_v2/table/paimon_reader.cpp
+++ b/be/src/format_v2/table/paimon_reader.cpp
@@ -365,13 +365,12 @@ Status PaimonReader::_parse_deletion_vector_file(const
TTableFormatFileDesc& t_d
return Status::OK();
}
const auto& deletion_file = table_desc.deletion_file;
- size_t bytes_read = 0;
- RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file,
bytes_read));
+ RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file));
desc->key = build_paimon_deletion_vector_cache_key(deletion_file);
desc->path = deletion_file.path;
desc->start_offset = deletion_file.offset;
- desc->size = static_cast<int64_t>(bytes_read);
+ desc->size = deletion_file.length;
desc->file_size = -1;
desc->format = DeleteFileDesc::Format::PAIMON;
*has_delete_file = true;
diff --git a/be/src/format_v2/table_reader.cpp
b/be/src/format_v2/table_reader.cpp
index f8a1693f972..49921d7b702 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -1580,8 +1580,6 @@ Status TableReader::_parse_delete_predicates(const
SplitReadOptions& options) {
return nullptr;
}
- size_t bytes_read = desc.size;
- std::vector<char> buffer(bytes_read);
DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.io_error", {
create_status =
Status::IOError("injected format v2 deletion
vector read failure");
@@ -1591,8 +1589,14 @@ Status TableReader::_parse_delete_predicates(const
SplitReadOptions& options) {
create_status = Status::EndOfFile("stop read.");
return nullptr;
});
- create_status =
- dv_reader.read_at(desc.start_offset,
{buffer.data(), bytes_read});
+ std::vector<char> buffer;
+ if (desc.format == DeleteFileDesc::Format::PAIMON) {
+ create_status =
read_paimon_deletion_vector_buffer(dv_reader, desc, buffer);
+ } else {
+ buffer.resize(desc.size);
+ create_status = dv_reader.read_at(desc.start_offset,
+ {buffer.data(),
buffer.size()});
+ }
const auto& file_cache_stats =
dv_reader.file_cache_statistics();
COUNTER_UPDATE(_profile.dv_file_cache_hit_count,
file_cache_stats.num_local_io_total);
@@ -1604,9 +1608,8 @@ Status TableReader::_parse_delete_predicates(const
SplitReadOptions& options) {
return nullptr;
}
- const char* buf = buffer.data();
SCOPED_TIMER(_profile.parse_delete_file_time);
- create_status = parse_deletion_vector(buf, bytes_read,
desc.format,
+ create_status = parse_deletion_vector(buffer.data(),
buffer.size(), desc.format,
deletion_vector.get());
if (!create_status.ok()) [[unlikely]] {
return nullptr;
diff --git a/be/test/format_v2/table/paimon_reader_test.cpp
b/be/test/format_v2/table/paimon_reader_test.cpp
index 33e2f50c9a5..354e608e48f 100644
--- a/be/test/format_v2/table/paimon_reader_test.cpp
+++ b/be/test/format_v2/table/paimon_reader_test.cpp
@@ -29,6 +29,7 @@
#include <fstream>
#include <memory>
#include <string>
+#include <string_view>
#include <thread>
#include <vector>
@@ -48,6 +49,7 @@
#include "core/data_type/data_type_struct.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/field.h"
+#include "cpp/sync_point.h"
#include "exec/common/endian.h"
#include "exprs/vectorized_fn_call.h"
#include "exprs/vexpr_context.h"
@@ -66,6 +68,8 @@
#include "runtime/runtime_profile.h"
#include "runtime/runtime_state.h"
#include "storage/options.h"
+#include "util/defer_op.h"
+#include "util/hash_util.hpp"
#include "util/timezone_utils.h"
namespace doris::format {
@@ -265,6 +269,13 @@ std::vector<char> build_paimon_deletion_vector_buffer(
return blob;
}
+// Paimon 2.1.0 Bitmap64DeletionVector from the HMS reproduction, with row
position 1 deleted.
+constexpr char PAIMON_BITMAP64_DV[] =
+ "\x00\x00\x00\x22\xd1\xd3\x39\x64\x01\x00\x00\x00\x00\x00\x00\x00"
+ "\x00\x00\x00\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00"
+ "\x10\x00\x00\x00\x01\x00\xee\xbd\x85\xf4";
+constexpr int64_t PAIMON_BITMAP64_DV_SIZE = sizeof(PAIMON_BITMAP64_DV) - 1;
+
int64_t write_paimon_deletion_vector_file(const std::string& file_path,
const std::vector<uint32_t>&
deleted_positions) {
const auto blob = build_paimon_deletion_vector_buffer(deleted_positions);
@@ -705,6 +716,64 @@ TEST(PaimonReaderTest,
DecodeDeletionVectorBufferRejectsCorruptRoaringBitmap) {
.ok());
}
+TEST(PaimonReaderTest, DecodeBitmap64DeletionVectorRejectsInvalidChecksum) {
+ std::string buffer(PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE);
+ buffer.back() ^= 1;
+ DeletionVector deletion_vector;
+
+ auto status =
+ decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(),
&deletion_vector);
+ EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>());
+ EXPECT_NE(status.to_string().find("CRC32 mismatch"), std::string::npos);
+ EXPECT_TRUE(deletion_vector.isEmpty());
+}
+
+TEST(PaimonReaderTest, ReadsEachDeletionVectorFormatAtEofInOneRead) {
+ const auto path = std::filesystem::temp_directory_path() /
"doris_paimon_dv_single_read_test";
+ Defer cleanup([&] { std::filesystem::remove(path); });
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ const auto scan_params = make_local_parquet_scan_params();
+
+ size_t reads = 0;
+ auto* sync_point = SyncPoint::get_instance();
+ SyncPoint::CallbackGuard guard;
+ sync_point->set_call_back(
+ "LocalFileReader::pread", [&](auto&&) { ++reads; }, &guard);
+ sync_point->enable_processing();
+ Defer stop_sync([&] { sync_point->disable_processing(); });
+
+ const auto bitmap32 = build_paimon_deletion_vector_buffer({1});
+ const struct {
+ std::string_view bytes;
+ int64_t descriptor_size;
+ } cases[] = {
+ {.bytes = {bitmap32.data(), bitmap32.size()},
+ .descriptor_size = static_cast<int64_t>(bitmap32.size()) - 4},
+ {.bytes = {PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE},
+ .descriptor_size = PAIMON_BITMAP64_DV_SIZE},
+ };
+ for (const auto& test_case : cases) {
+ SCOPED_TRACE(test_case.descriptor_size);
+ {
+ std::ofstream output(path, std::ios::binary);
+ output.put(1); // Paimon index file version.
+ output.write(test_case.bytes.data(), test_case.bytes.size());
+ ASSERT_TRUE(output.good());
+ }
+ const DeleteFileDesc desc {
+ .path = path.string(), .start_offset = 1, .size =
test_case.descriptor_size};
+ DeletionVectorReader reader(&state, nullptr, scan_params, desc,
nullptr);
+ ASSERT_TRUE(reader.open().ok());
+
+ reads = 0;
+ std::vector<char> buffer;
+ const auto status = read_paimon_deletion_vector_buffer(reader, desc,
buffer);
+ ASSERT_TRUE(status.ok()) << status;
+ EXPECT_EQ(reads, 1);
+ EXPECT_EQ(std::string_view(buffer.data(), buffer.size()),
test_case.bytes);
+ }
+}
+
// Scenario: PaimonReader must clear the previous split schema id before
reading a new split. A
// schema-evolved split must not force the following split without schema id
to keep BY_FIELD_ID.
TEST(PaimonReaderTest, ResetsSplitSchemaIdBeforePreparingNextSplit) {
@@ -935,9 +1004,30 @@ TEST(PaimonReaderTest,
ReadsInt96HistorySemanticsThroughTableMapper) {
std::filesystem::remove(path);
}
-// Scenario: Paimon reader should parse its bitmap deletion vector and let
TableReader apply the
-// generated row-position delete predicate before returning table rows.
-TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
+void check_paimon_split_ids(paimon::PaimonReader& reader, const
SplitReadOptions& split_options,
+ const std::vector<ColumnDefinition>&
projected_columns,
+ const std::vector<int32_t>& expected) {
+ ASSERT_TRUE(reader.prepare_split(split_options).ok());
+
+ std::vector<int32_t> ids;
+ bool eos = false;
+ while (!eos) {
+ Block block = build_table_block(projected_columns);
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ if (block.rows() == 0) {
+ continue;
+ }
+ const auto& id_column =
+ assert_cast<const
ColumnInt32&>(expect_not_null_table_column(block, 0));
+ for (size_t row = 0; row < block.rows(); ++row) {
+ ids.push_back(id_column.get_element(row));
+ }
+ }
+ EXPECT_EQ(ids, expected);
+}
+
+// Paimon can write both bitmap versions into the same index file. The last
entry ends at EOF.
+TEST(PaimonReaderTest, AppliesMixedBitmapDeletionVectorFile) {
const auto test_dir =
std::filesystem::temp_directory_path() /
"doris_paimon_deletion_vector_file_test";
std::filesystem::remove_all(test_dir);
@@ -947,7 +1037,22 @@ TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
const auto dv_path = (test_dir / "delete-vector.bin").string();
write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40,
50},
{"one", "two", "three", "four", "five"});
- const auto dv_length = write_paimon_deletion_vector_file(dv_path, {0, 4});
+ const auto bitmap32 = build_paimon_deletion_vector_buffer({0, 4});
+ const auto bitmap32_length = static_cast<uint32_t>(bitmap32.size() - 4);
+ const uint32_t bitmap32_crc = HashUtil::zlib_crc_hash(bitmap32.data() + 4,
bitmap32_length, 0);
+ char crc_bytes[4];
+ BigEndian::Store32(crc_bytes, bitmap32_crc);
+ {
+ std::ofstream output(dv_path, std::ios::binary);
+ output.put(1); // Paimon index file version.
+ output.write(bitmap32.data(),
static_cast<std::streamsize>(bitmap32.size()));
+ output.write(crc_bytes, sizeof(crc_bytes));
+ output.write(PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE);
+ output.write(PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE);
+ ASSERT_TRUE(output.good());
+ }
+ const int64_t bitmap64_offset = 1 + bitmap32.size() + sizeof(crc_bytes);
+ const int64_t bitmap64_eof_offset = bitmap64_offset +
PAIMON_BITMAP64_DV_SIZE;
std::vector<ColumnDefinition> projected_columns;
projected_columns.push_back(make_table_column(0, "id",
std::make_shared<DataTypeInt32>()));
@@ -973,25 +1078,33 @@ TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
auto split_options = build_split_options(file_path);
split_options.cache = &cache;
- split_options.current_range.__set_table_format_params(
- make_paimon_table_format_desc(dv_path, 0, dv_length));
- ASSERT_TRUE(reader.prepare_split(split_options).ok());
-
- std::vector<int32_t> ids;
- bool eos = false;
- while (!eos) {
- Block block = build_table_block(projected_columns);
- ASSERT_TRUE(reader.get_block(&block, &eos).ok());
- if (block.rows() == 0) {
- continue;
- }
- const auto& id_column =
- assert_cast<const
ColumnInt32&>(expect_not_null_table_column(block, 0));
- for (size_t row = 0; row < block.rows(); ++row) {
- ids.push_back(id_column.get_element(row));
- }
+ const struct {
+ int64_t offset;
+ int64_t length;
+ std::vector<int32_t> expected;
+ } cases[] = {
+ {.offset = 1, .length = bitmap32_length, .expected = {2, 3, 4}},
+ {.offset = bitmap64_offset,
+ .length = PAIMON_BITMAP64_DV_SIZE,
+ .expected = {1, 3, 4, 5}},
+ {.offset = bitmap64_eof_offset,
+ .length = PAIMON_BITMAP64_DV_SIZE,
+ .expected = {1, 3, 4, 5}},
+ // Repeat the first split through the decoded cache.
+ {.offset = 1, .length = bitmap32_length, .expected = {2, 3, 4}},
+ };
+ for (const auto& test_case : cases) {
+ split_options.current_range.__set_table_format_params(
+ make_paimon_table_format_desc(dv_path, test_case.offset,
test_case.length));
+ ASSERT_NO_FATAL_FAILURE(check_paimon_split_ids(reader, split_options,
projected_columns,
+ test_case.expected));
}
- EXPECT_EQ(ids, std::vector<int32_t>({2, 3, 4}));
+
+
split_options.current_range.__set_table_format_params(make_paimon_table_format_desc(
+ dv_path, bitmap64_eof_offset, PAIMON_BITMAP64_DV_SIZE - 1));
+ const auto status = reader.prepare_split(split_options);
+ EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>());
+ EXPECT_NE(status.to_string().find("descriptor length mismatch"),
std::string::npos);
ASSERT_TRUE(reader.close().ok());
std::filesystem::remove_all(test_dir);
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
index d8b9e547101..d6fca52f467 100644
---
a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
+++
b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
@@ -95,3 +95,7 @@
-- !13 --
4
+-- !bitmap64_native --
+1 10
+3 30
+
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
index 172df875927..33bd56210a9 100644
---
a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
@@ -105,4 +105,39 @@ suite("test_paimon_deletion_vector", "p0,external") {
sql """set force_jni_scanner=false"""
}
+ sql "CREATE DATABASE IF NOT EXISTS
test_paimon_deletion_vector.paimon_bitmap64_dv_db"
+ sql "DROP TABLE IF EXISTS
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv"
+ sql """
+ CREATE TABLE
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv (
+ id INT NOT NULL, payload INT
+ ) ENGINE=paimon PROPERTIES (
+ 'primary-key' = 'id',
+ 'bucket' = '1',
+ 'file.format' = 'parquet',
+ 'deletion-vectors.enabled' = 'true',
+ 'deletion-vectors.bitmap64' = 'true'
+ )
+ """
+ sql "INSERT INTO
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv VALUES (1, 10),
(2, 20), (3, 30)"
+ sql "DELETE FROM
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv WHERE id = 2"
+
+ // The native query must read a physical DV rather than an already
rewritten data file.
+ def indexes = sql """
+ SELECT row_count
+ FROM
test_paimon_deletion_vector.paimon_bitmap64_dv_db.`bitmap64_dv\$table_indexes`
+ WHERE index_type = 'DELETION_VECTORS' AND row_count > 0
+ """
+ assertFalse(indexes.isEmpty(), "DELETE must leave a deletion-vector entry")
+
+ def bitmap64Query = """
+ SELECT /*+ SET_VAR(enable_file_scanner_v2=true,
enable_paimon_rust_reader=false, force_jni_scanner=false) */
+ id, payload
+ FROM test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv
+ ORDER BY id
+ """
+ explain {
+ sql bitmap64Query
+ contains "paimonNativeReadSplits=1/1"
+ }
+ order_qt_bitmap64_native bitmap64Query
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]