This is an automated email from the ASF dual-hosted git repository.

Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 5307f3ccd34 [fix](paimon) Read 64-bit deletion vectors in native scans 
(#70401)
5307f3ccd34 is described below

commit 5307f3ccd34314152378cb18b83064cd6c1f73da
Author: Socrates <[email protected]>
AuthorDate: Wed Nov 18 10:02:22 2026 +0800

    [fix](paimon) Read 64-bit deletion vectors in native scans (#70401)
    
    ### Background
    
    Paimon stores deleted row positions as bitmap entries in index files.
    During a native scan, Doris reads the entry for each split and applies
    its bitmap as a row-position delete predicate.
    
    ### What problem does this PR solve?
    
    Issue Number: None
    
    Related PR: None
    
    PR Type: Bugfix
    
    Problem Summary: Native Doris scans fail on Paimon bitmap64 deletion
    vectors. Doris treats every descriptor length as bitmap32 and adds four
    bytes, so a bitmap64 entry ending at EOF fails the file-range check.
    This change identifies the format from each entry header and applies the
    corresponding bitmap, including when an index contains both versions.
    
    ### Details
    
    - **Frequency:** A native scan fails whenever it must apply a bitmap64
    entry.
    - **Root cause:** Bitmap32 descriptor lengths count the magic and
    bitmap. Bitmap64 descriptor lengths also count the prefix and checksum.
    The previous reader assumed bitmap32 length semantics and decoded only
    Roaring32.
    - **Why existing tests missed it:** Native reader coverage used bitmap32
    fixtures.
    - **Fix:** Validate the descriptor and physical range using the entry
    header, then use the shared Roaring64 and CRC32 decoder for bitmap64.
    Keep the decoded bitmap in the query-local cache after successful
    validation.
    - **Other paths and branches:** This change covers the native
    FileScannerV2 path on master. The JNI reader serves as the reference in
    the Fedora smoke test.
    - **Behavior change:** Native Paimon scans can apply bitmap64 deletion
    vectors. The first read of a DV on a decoded-cache miss reads the header
    and bitmap separately.
    
    ### Testing
    
    - Fedora x86_64 ASAN BE build completed.
    - Fedora SQL smoke test with Hive Metastore and Paimon 2.1.0. Both
    bitmap32 and bitmap64 scans selected the native reader and returned
    `(1,10)` and `(3,30)`, matching the JNI reader. The original bitmap64
    query failed before the fix.
    - Fedora `run-be-ut.sh --run
    --filter='PaimonReaderTest.*:IcebergDeleteFileReaderHelperTest.*'` with
    ASAN_UT passed all 40 tests (19 Paimon, 21 Iceberg). The local build
    scoped the test target to these suites and their test support sources.
    - Clang-format 16, build hygiene, and changed production lines checked
    with clang-tidy.
    
    - The existing `test_paimon_deletion_vector` suite passed on Fedora
    against HDFS. Its bitmap64 case creates a physical deletion vector,
    confirms native scan planning, and verifies the two surviving rows. The
    baseline BE fails with the DV range error when querying the same HDFS
    table.
    
    ### Release note
    
    Native Paimon scans support 64-bit deletion vectors, including snapshots
    that also reference 32-bit vectors.
    
    ### Check List (For Author)
    
    - Behavior changed: Yes (see Details)
    - Does this need documentation: No
    - Pre-submission status: Ready
    
    ### AI Disclosure
    
    - Tool: Codex (GPT-6)
    - Generated: Implementation, BE unit tests, and this description
    - Human review: Pending
---
 be/src/format/table/deletion_vector.h              |   4 +-
 be/src/format/table/deletion_vector_reader.cpp     |  42 +++---
 be/src/format/table/deletion_vector_reader.h       |   6 +-
 be/src/format/table/paimon_deletion_vector.cpp     |  82 +++++++++--
 be/src/format/table/paimon_deletion_vector.h       |  10 +-
 be/src/format_v2/table/paimon_reader.cpp           |   5 +-
 be/src/format_v2/table_reader.cpp                  |  15 +-
 be/test/format_v2/table/paimon_reader_test.cpp     | 157 ++++++++++++++++++---
 .../paimon/test_paimon_deletion_vector.out         |   4 +
 .../paimon/test_paimon_deletion_vector.groovy      |  35 +++++
 10 files changed, 284 insertions(+), 76 deletions(-)

diff --git a/be/src/format/table/deletion_vector.h 
b/be/src/format/table/deletion_vector.h
index 12ec7dbc6eb..e1e10655a73 100644
--- a/be/src/format/table/deletion_vector.h
+++ b/be/src/format/table/deletion_vector.h
@@ -29,8 +29,8 @@ namespace doris {
 inline constexpr int64_t MAX_ICEBERG_DELETION_VECTOR_BYTES = 1L << 30;
 inline constexpr size_t ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES = 12;
 
-// Paimon v1 uses a run-optimized 32-bit Roaring bitmap whose maximum 
serialized size is below this
-// limit. Keep its guard independent from the Iceberg capability limit.
+// Paimon bitmap32 fits below this limit. Bitmap64 may exceed it, so check the 
serialized size
+// before allocating a read buffer. Keep the Paimon limit independent from the 
Iceberg limit.
 inline constexpr int64_t MAX_PAIMON_DELETION_VECTOR_BYTES = 1L << 30;
 
 // A deletion vector is already a bitmap on the wire. Keep decoded DVs 
compressed in the
diff --git a/be/src/format/table/deletion_vector_reader.cpp 
b/be/src/format/table/deletion_vector_reader.cpp
index 946e81fa2e6..64d1d8b5020 100644
--- a/be/src/format/table/deletion_vector_reader.cpp
+++ b/be/src/format/table/deletion_vector_reader.cpp
@@ -30,7 +30,6 @@ namespace {
 constexpr int64_t ICEBERG_DELETION_VECTOR_MIN_BYTES =
         static_cast<int64_t>(ICEBERG_DELETION_VECTOR_BLOB_OVERHEAD_BYTES);
 constexpr int64_t PAIMON_DELETION_VECTOR_MIN_BYTES = 8;
-constexpr int64_t PAIMON_LENGTH_PREFIX_BYTES = 4;
 
 enum class DeletionVectorSizeLimitStatus {
     DATA_QUALITY,
@@ -39,8 +38,7 @@ enum class DeletionVectorSizeLimitStatus {
 
 Status validate_deletion_vector_read_range(const char* description, int64_t 
offset, int64_t size,
                                            int64_t min_size, int64_t max_size,
-                                           DeletionVectorSizeLimitStatus 
size_limit_status,
-                                           size_t& bytes_read) {
+                                           DeletionVectorSizeLimitStatus 
size_limit_status) {
     if (offset < 0) {
         return Status::DataQualityError("{} offset must be non-negative: {}", 
description, offset);
     }
@@ -60,7 +58,6 @@ Status validate_deletion_vector_read_range(const char* 
description, int64_t offs
         return Status::DataQualityError("{} offset plus size overflows: offset 
{}, size {}",
                                         description, offset, size);
     }
-    bytes_read = static_cast<size_t>(size);
     return Status::OK();
 }
 
@@ -82,25 +79,17 @@ Status missing_file_as_error(Status status, const 
std::string& path) {
 
 Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t 
size,
                                                    size_t& bytes_read) {
-    return validate_deletion_vector_read_range(
+    RETURN_IF_ERROR(validate_deletion_vector_read_range(
             "Iceberg deletion vector", offset, size, 
ICEBERG_DELETION_VECTOR_MIN_BYTES,
-            MAX_ICEBERG_DELETION_VECTOR_BYTES, 
DeletionVectorSizeLimitStatus::NOT_SUPPORTED,
-            bytes_read);
+            MAX_ICEBERG_DELETION_VECTOR_BYTES, 
DeletionVectorSizeLimitStatus::NOT_SUPPORTED));
+    bytes_read = static_cast<size_t>(size);
+    return Status::OK();
 }
 
-Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t 
length,
-                                                  size_t& bytes_read) {
-    if (length < 0) {
-        return Status::DataQualityError("Paimon deletion vector length must be 
non-negative: {}",
-                                        length);
-    }
-    if (length > std::numeric_limits<int64_t>::max() - 
PAIMON_LENGTH_PREFIX_BYTES) {
-        return Status::DataQualityError("Paimon deletion vector length 
overflows: {}", length);
-    }
+Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t 
size) {
     return validate_deletion_vector_read_range(
-            "Paimon deletion vector", offset, length + 
PAIMON_LENGTH_PREFIX_BYTES,
-            PAIMON_DELETION_VECTOR_MIN_BYTES, MAX_PAIMON_DELETION_VECTOR_BYTES,
-            DeletionVectorSizeLimitStatus::DATA_QUALITY, bytes_read);
+            "Paimon deletion vector", offset, size, 
PAIMON_DELETION_VECTOR_MIN_BYTES,
+            MAX_PAIMON_DELETION_VECTOR_BYTES, 
DeletionVectorSizeLimitStatus::DATA_QUALITY);
 }
 
 DeletionVectorReader::~DeletionVectorReader() {
@@ -151,16 +140,19 @@ Status DeletionVectorReader::open() {
     _init_file_description();
     RETURN_IF_ERROR(missing_file_as_error(_create_file_reader(), _desc.path));
 
-    const size_t file_size = _file_reader->size();
-    const size_t start_offset = static_cast<size_t>(_desc.start_offset);
-    const size_t range_size = static_cast<size_t>(_desc.size);
-    if (start_offset > file_size || range_size > file_size - start_offset) {
+    RETURN_IF_ERROR(validate_file_range(_desc.start_offset, _desc.size));
+    _is_opened = true;
+    return Status::OK();
+}
+
+Status DeletionVectorReader::validate_file_range(size_t offset, size_t size) 
const {
+    const size_t length = file_size();
+    if (offset > length || size > length - offset) {
         return Status::DataQualityError(
                 "Deletion vector range exceeds file size: path {}, offset {}, 
size {}, file size "
                 "{}",
-                _desc.path, _desc.start_offset, _desc.size, file_size);
+                _desc.path, offset, size, length);
     }
-    _is_opened = true;
     return Status::OK();
 }
 
diff --git a/be/src/format/table/deletion_vector_reader.h 
b/be/src/format/table/deletion_vector_reader.h
index c2ed07a242a..2c75c7e8a58 100644
--- a/be/src/format/table/deletion_vector_reader.h
+++ b/be/src/format/table/deletion_vector_reader.h
@@ -41,8 +41,7 @@ namespace doris {
 Status validate_iceberg_deletion_vector_read_range(int64_t offset, int64_t 
size,
                                                    size_t& bytes_read);
 
-Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t 
length,
-                                                  size_t& bytes_read);
+Status validate_paimon_deletion_vector_read_range(int64_t offset, int64_t 
size);
 
 struct DeleteFileDesc {
     enum class Format {
@@ -54,6 +53,7 @@ struct DeleteFileDesc {
     std::string path = "";
     std::string fs_name = "";
     int64_t start_offset = 0;
+    // Paimon bitmap32 descriptor length excludes the length prefix and 
checksum.
     int64_t size = 0;
     int64_t file_size = -1;
     int64_t modification_time = 0;
@@ -89,6 +89,8 @@ public:
     DeletionVectorReader(const DeletionVectorReader&) = delete;
     DeletionVectorReader& operator=(const DeletionVectorReader&) = delete;
     Status open();
+    size_t file_size() const { return _file_reader->size(); }
+    Status validate_file_range(size_t offset, size_t size) const;
     Status read_at(size_t offset, Slice result);
 
     // These are deliberately exposed separately from the decoded-DV cache 
counters. Local I/O is
diff --git a/be/src/format/table/paimon_deletion_vector.cpp 
b/be/src/format/table/paimon_deletion_vector.cpp
index a9adb8ae28e..f2a2f48ae56 100644
--- a/be/src/format/table/paimon_deletion_vector.cpp
+++ b/be/src/format/table/paimon_deletion_vector.cpp
@@ -19,18 +19,23 @@
 
 #include <fmt/format.h>
 
-#include <cstring>
+#include <algorithm>
 #include <stdexcept>
 
 #include "common/status.h"
 #include "exec/common/endian.h"
 #include "format/table/deletion_vector_reader.h"
+#include "format/table/iceberg_delete_file_reader_helper.h"
 
 namespace doris {
 
 namespace {
 
-constexpr char PAIMON_BITMAP_MAGIC[] = {'\x5E', '\x43', '\xF2', '\xD0'};
+constexpr uint32_t PAIMON_BITMAP32_MAGIC = 0x5e43f2d0;
+constexpr uint32_t PAIMON_BITMAP64_MAGIC = 0xd1d33964;
+constexpr size_t PAIMON_DV_HEADER_BYTES = 8;
+constexpr int64_t PAIMON_DV_LENGTH_PREFIX_BYTES = 4;
+constexpr int64_t PAIMON_DV_CRC_BYTES = 4;
 
 } // namespace
 
@@ -39,15 +44,56 @@ std::string build_paimon_deletion_vector_cache_key(const 
TPaimonDeletionFileDesc
                        deletion_file.length);
 }
 
-Status validate_paimon_deletion_vector_descriptor(const 
TPaimonDeletionFileDesc& deletion_file,
-                                                  size_t& bytes_read) {
+Status validate_paimon_deletion_vector_descriptor(const 
TPaimonDeletionFileDesc& deletion_file) {
     if (!deletion_file.__isset.path || !deletion_file.__isset.offset ||
         !deletion_file.__isset.length) {
         return Status::DataQualityError(
                 "Paimon deletion file descriptor misses path/offset/length");
     }
-    return validate_paimon_deletion_vector_read_range(deletion_file.offset, 
deletion_file.length,
-                                                      bytes_read);
+    return validate_paimon_deletion_vector_read_range(deletion_file.offset, 
deletion_file.length);
+}
+
+Status read_paimon_deletion_vector_buffer(DeletionVectorReader& reader, const 
DeleteFileDesc& desc,
+                                          std::vector<char>& buffer) {
+    const auto offset = static_cast<size_t>(desc.start_offset);
+    // An 8-byte header is within the validated descriptor range. Read enough 
for bitmap32's
+    // extra prefix, capped at EOF so a bitmap64 entry at the end needs no 
trailing bytes.
+    const size_t bytes_to_read = std::min(
+            {static_cast<size_t>(desc.size + PAIMON_DV_LENGTH_PREFIX_BYTES),
+             static_cast<size_t>(MAX_PAIMON_DELETION_VECTOR_BYTES), 
reader.file_size() - offset});
+    buffer.resize(bytes_to_read);
+    RETURN_IF_ERROR(reader.read_at(offset, {buffer.data(), buffer.size()}));
+    const uint32_t payload_size = BigEndian::Load32(buffer.data());
+    const uint32_t magic = BigEndian::Load32(buffer.data() + 
PAIMON_DV_LENGTH_PREFIX_BYTES);
+    // The length prefix counts the magic itself, even for an empty bitmap.
+    if (payload_size < sizeof(magic)) {
+        return Status::DataQualityError(
+                "Paimon deletion vector payload too small: path {}, offset {}, 
size {}", desc.path,
+                desc.start_offset, payload_size);
+    }
+    if (magic != PAIMON_BITMAP32_MAGIC && magic != PAIMON_BITMAP64_MAGIC) {
+        return Status::DataQualityError(
+                "Paimon deletion vector magic number mismatch: path {}, offset 
{}, magic {}",
+                desc.path, desc.start_offset, magic);
+    }
+
+    // Paimon may mix bitmap32 and bitmap64 entries in one index file. The 
bitmap32 descriptor
+    // counts only magic and bitmap, while bitmap64 counts the prefix and CRC 
as well.
+    const bool bitmap64 = magic == PAIMON_BITMAP64_MAGIC;
+    const int64_t entry_size =
+            payload_size + PAIMON_DV_LENGTH_PREFIX_BYTES + (bitmap64 ? 
PAIMON_DV_CRC_BYTES : 0);
+    const int64_t expected_length = bitmap64 ? entry_size : payload_size;
+    if (desc.size != expected_length) {
+        return Status::DataQualityError(
+                "Paimon deletion vector descriptor length mismatch: path {}, 
offset {}, expected "
+                "{}, actual {}",
+                desc.path, desc.start_offset, expected_length, desc.size);
+    }
+    
RETURN_IF_ERROR(validate_paimon_deletion_vector_read_range(desc.start_offset, 
entry_size));
+    RETURN_IF_ERROR(reader.validate_file_range(offset, entry_size));
+    // A bitmap64 entry followed by another entry may have up to four extra 
bytes in the buffer.
+    buffer.resize(entry_size);
+    return Status::OK();
 }
 
 Status decode_paimon_deletion_vector_buffer(const char* buf, size_t 
buffer_size,
@@ -58,27 +104,35 @@ Status decode_paimon_deletion_vector_buffer(const char* 
buf, size_t buffer_size,
     if (buf == nullptr) {
         return Status::DataQualityError("Paimon deletion vector blob is null");
     }
-    if (buffer_size < 8) [[unlikely]] {
+    if (buffer_size < PAIMON_DV_HEADER_BYTES) [[unlikely]] {
         return Status::DataQualityError("Deletion vector file size too small: 
{}", buffer_size);
     }
 
+    const uint32_t magic = BigEndian::Load32(buf + 
PAIMON_DV_LENGTH_PREFIX_BYTES);
+    if (magic == PAIMON_BITMAP64_MAGIC) {
+        // Bitmap64 uses the same on-disk framing and portable Roaring64 
layout as Iceberg DVs.
+        return decode_iceberg_deletion_vector_buffer(buf, buffer_size, 
deletion_vector);
+    }
+
     const uint32_t actual_length = BigEndian::Load32(buf);
-    if (static_cast<uint64_t>(actual_length) + 4 != buffer_size) [[unlikely]] {
+    const uint64_t expected_size =
+            static_cast<uint64_t>(actual_length) + 
PAIMON_DV_LENGTH_PREFIX_BYTES;
+    if (expected_size != buffer_size) [[unlikely]] {
         return Status::DataQualityError(
-                "Paimon deletion vector length mismatch, expected: {}, actual: 
{}",
-                static_cast<uint64_t>(actual_length) + 4, buffer_size);
+                "Paimon deletion vector length mismatch, expected: {}, actual: 
{}", expected_size,
+                buffer_size);
     }
 
-    if (memcmp(buf + sizeof(actual_length), PAIMON_BITMAP_MAGIC, 4) != 0) 
[[unlikely]] {
+    if (magic != PAIMON_BITMAP32_MAGIC) [[unlikely]] {
         return Status::DataQualityError(
                 "Paimon deletion vector magic number mismatch, expected: {}, 
actual: {}",
-                BigEndian::Load32(PAIMON_BITMAP_MAGIC),
-                BigEndian::Load32(buf + sizeof(actual_length)));
+                PAIMON_BITMAP32_MAGIC, magic);
     }
 
     roaring::Roaring roaring_bitmap;
     try {
-        roaring_bitmap = roaring::Roaring::readSafe(buf + 8, buffer_size - 8);
+        roaring_bitmap = roaring::Roaring::readSafe(buf + 
PAIMON_DV_HEADER_BYTES,
+                                                    buffer_size - 
PAIMON_DV_HEADER_BYTES);
     } catch (const std::runtime_error& e) {
         return Status::RuntimeError(
                 "DeletionVector deserialize error: failed to deserialize 
roaring bitmap, {}",
diff --git a/be/src/format/table/paimon_deletion_vector.h 
b/be/src/format/table/paimon_deletion_vector.h
index 4f809c2de01..afca2ca5a6f 100644
--- a/be/src/format/table/paimon_deletion_vector.h
+++ b/be/src/format/table/paimon_deletion_vector.h
@@ -19,6 +19,7 @@
 
 #include <cstddef>
 #include <string>
+#include <vector>
 
 #include "common/status.h"
 #include "format/table/deletion_vector.h"
@@ -26,9 +27,14 @@
 
 namespace doris {
 
+class DeletionVectorReader;
+struct DeleteFileDesc;
+
 std::string build_paimon_deletion_vector_cache_key(const 
TPaimonDeletionFileDesc& deletion_file);
-Status validate_paimon_deletion_vector_descriptor(const 
TPaimonDeletionFileDesc& deletion_file,
-                                                  size_t& bytes_read);
+Status validate_paimon_deletion_vector_descriptor(const 
TPaimonDeletionFileDesc& deletion_file);
+// The reader is open and the Paimon descriptor range has been validated.
+Status read_paimon_deletion_vector_buffer(DeletionVectorReader& reader, const 
DeleteFileDesc& desc,
+                                          std::vector<char>& buffer);
 Status decode_paimon_deletion_vector_buffer(const char* buf, size_t 
buffer_size,
                                             DeletionVector* deletion_vector);
 
diff --git a/be/src/format_v2/table/paimon_reader.cpp 
b/be/src/format_v2/table/paimon_reader.cpp
index 1ac1a3aa4e6..408da128669 100644
--- a/be/src/format_v2/table/paimon_reader.cpp
+++ b/be/src/format_v2/table/paimon_reader.cpp
@@ -365,13 +365,12 @@ Status PaimonReader::_parse_deletion_vector_file(const 
TTableFormatFileDesc& t_d
         return Status::OK();
     }
     const auto& deletion_file = table_desc.deletion_file;
-    size_t bytes_read = 0;
-    RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file, 
bytes_read));
+    RETURN_IF_ERROR(validate_paimon_deletion_vector_descriptor(deletion_file));
 
     desc->key = build_paimon_deletion_vector_cache_key(deletion_file);
     desc->path = deletion_file.path;
     desc->start_offset = deletion_file.offset;
-    desc->size = static_cast<int64_t>(bytes_read);
+    desc->size = deletion_file.length;
     desc->file_size = -1;
     desc->format = DeleteFileDesc::Format::PAIMON;
     *has_delete_file = true;
diff --git a/be/src/format_v2/table_reader.cpp 
b/be/src/format_v2/table_reader.cpp
index f8a1693f972..49921d7b702 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -1580,8 +1580,6 @@ Status TableReader::_parse_delete_predicates(const 
SplitReadOptions& options) {
                         return nullptr;
                     }
 
-                    size_t bytes_read = desc.size;
-                    std::vector<char> buffer(bytes_read);
                     
DBUG_EXECUTE_IF("TableReader.parse_deletion_vector.io_error", {
                         create_status =
                                 Status::IOError("injected format v2 deletion 
vector read failure");
@@ -1591,8 +1589,14 @@ Status TableReader::_parse_delete_predicates(const 
SplitReadOptions& options) {
                         create_status = Status::EndOfFile("stop read.");
                         return nullptr;
                     });
-                    create_status =
-                            dv_reader.read_at(desc.start_offset, 
{buffer.data(), bytes_read});
+                    std::vector<char> buffer;
+                    if (desc.format == DeleteFileDesc::Format::PAIMON) {
+                        create_status = 
read_paimon_deletion_vector_buffer(dv_reader, desc, buffer);
+                    } else {
+                        buffer.resize(desc.size);
+                        create_status = dv_reader.read_at(desc.start_offset,
+                                                          {buffer.data(), 
buffer.size()});
+                    }
                     const auto& file_cache_stats = 
dv_reader.file_cache_statistics();
                     COUNTER_UPDATE(_profile.dv_file_cache_hit_count,
                                    file_cache_stats.num_local_io_total);
@@ -1604,9 +1608,8 @@ Status TableReader::_parse_delete_predicates(const 
SplitReadOptions& options) {
                         return nullptr;
                     }
 
-                    const char* buf = buffer.data();
                     SCOPED_TIMER(_profile.parse_delete_file_time);
-                    create_status = parse_deletion_vector(buf, bytes_read, 
desc.format,
+                    create_status = parse_deletion_vector(buffer.data(), 
buffer.size(), desc.format,
                                                           
deletion_vector.get());
                     if (!create_status.ok()) [[unlikely]] {
                         return nullptr;
diff --git a/be/test/format_v2/table/paimon_reader_test.cpp 
b/be/test/format_v2/table/paimon_reader_test.cpp
index 33e2f50c9a5..354e608e48f 100644
--- a/be/test/format_v2/table/paimon_reader_test.cpp
+++ b/be/test/format_v2/table/paimon_reader_test.cpp
@@ -29,6 +29,7 @@
 #include <fstream>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <thread>
 #include <vector>
 
@@ -48,6 +49,7 @@
 #include "core/data_type/data_type_struct.h"
 #include "core/data_type/data_type_timestamptz.h"
 #include "core/field.h"
+#include "cpp/sync_point.h"
 #include "exec/common/endian.h"
 #include "exprs/vectorized_fn_call.h"
 #include "exprs/vexpr_context.h"
@@ -66,6 +68,8 @@
 #include "runtime/runtime_profile.h"
 #include "runtime/runtime_state.h"
 #include "storage/options.h"
+#include "util/defer_op.h"
+#include "util/hash_util.hpp"
 #include "util/timezone_utils.h"
 
 namespace doris::format {
@@ -265,6 +269,13 @@ std::vector<char> build_paimon_deletion_vector_buffer(
     return blob;
 }
 
+// Paimon 2.1.0 Bitmap64DeletionVector from the HMS reproduction, with row 
position 1 deleted.
+constexpr char PAIMON_BITMAP64_DV[] =
+        "\x00\x00\x00\x22\xd1\xd3\x39\x64\x01\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00"
+        "\x10\x00\x00\x00\x01\x00\xee\xbd\x85\xf4";
+constexpr int64_t PAIMON_BITMAP64_DV_SIZE = sizeof(PAIMON_BITMAP64_DV) - 1;
+
 int64_t write_paimon_deletion_vector_file(const std::string& file_path,
                                           const std::vector<uint32_t>& 
deleted_positions) {
     const auto blob = build_paimon_deletion_vector_buffer(deleted_positions);
@@ -705,6 +716,64 @@ TEST(PaimonReaderTest, 
DecodeDeletionVectorBufferRejectsCorruptRoaringBitmap) {
                     .ok());
 }
 
+TEST(PaimonReaderTest, DecodeBitmap64DeletionVectorRejectsInvalidChecksum) {
+    std::string buffer(PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE);
+    buffer.back() ^= 1;
+    DeletionVector deletion_vector;
+
+    auto status =
+            decode_paimon_deletion_vector_buffer(buffer.data(), buffer.size(), 
&deletion_vector);
+    EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>());
+    EXPECT_NE(status.to_string().find("CRC32 mismatch"), std::string::npos);
+    EXPECT_TRUE(deletion_vector.isEmpty());
+}
+
+TEST(PaimonReaderTest, ReadsEachDeletionVectorFormatAtEofInOneRead) {
+    const auto path = std::filesystem::temp_directory_path() / 
"doris_paimon_dv_single_read_test";
+    Defer cleanup([&] { std::filesystem::remove(path); });
+    RuntimeState state {TQueryOptions(), TQueryGlobals()};
+    const auto scan_params = make_local_parquet_scan_params();
+
+    size_t reads = 0;
+    auto* sync_point = SyncPoint::get_instance();
+    SyncPoint::CallbackGuard guard;
+    sync_point->set_call_back(
+            "LocalFileReader::pread", [&](auto&&) { ++reads; }, &guard);
+    sync_point->enable_processing();
+    Defer stop_sync([&] { sync_point->disable_processing(); });
+
+    const auto bitmap32 = build_paimon_deletion_vector_buffer({1});
+    const struct {
+        std::string_view bytes;
+        int64_t descriptor_size;
+    } cases[] = {
+            {.bytes = {bitmap32.data(), bitmap32.size()},
+             .descriptor_size = static_cast<int64_t>(bitmap32.size()) - 4},
+            {.bytes = {PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE},
+             .descriptor_size = PAIMON_BITMAP64_DV_SIZE},
+    };
+    for (const auto& test_case : cases) {
+        SCOPED_TRACE(test_case.descriptor_size);
+        {
+            std::ofstream output(path, std::ios::binary);
+            output.put(1); // Paimon index file version.
+            output.write(test_case.bytes.data(), test_case.bytes.size());
+            ASSERT_TRUE(output.good());
+        }
+        const DeleteFileDesc desc {
+                .path = path.string(), .start_offset = 1, .size = 
test_case.descriptor_size};
+        DeletionVectorReader reader(&state, nullptr, scan_params, desc, 
nullptr);
+        ASSERT_TRUE(reader.open().ok());
+
+        reads = 0;
+        std::vector<char> buffer;
+        const auto status = read_paimon_deletion_vector_buffer(reader, desc, 
buffer);
+        ASSERT_TRUE(status.ok()) << status;
+        EXPECT_EQ(reads, 1);
+        EXPECT_EQ(std::string_view(buffer.data(), buffer.size()), 
test_case.bytes);
+    }
+}
+
 // Scenario: PaimonReader must clear the previous split schema id before 
reading a new split. A
 // schema-evolved split must not force the following split without schema id 
to keep BY_FIELD_ID.
 TEST(PaimonReaderTest, ResetsSplitSchemaIdBeforePreparingNextSplit) {
@@ -935,9 +1004,30 @@ TEST(PaimonReaderTest, 
ReadsInt96HistorySemanticsThroughTableMapper) {
     std::filesystem::remove(path);
 }
 
-// Scenario: Paimon reader should parse its bitmap deletion vector and let 
TableReader apply the
-// generated row-position delete predicate before returning table rows.
-TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
+void check_paimon_split_ids(paimon::PaimonReader& reader, const 
SplitReadOptions& split_options,
+                            const std::vector<ColumnDefinition>& 
projected_columns,
+                            const std::vector<int32_t>& expected) {
+    ASSERT_TRUE(reader.prepare_split(split_options).ok());
+
+    std::vector<int32_t> ids;
+    bool eos = false;
+    while (!eos) {
+        Block block = build_table_block(projected_columns);
+        ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+        if (block.rows() == 0) {
+            continue;
+        }
+        const auto& id_column =
+                assert_cast<const 
ColumnInt32&>(expect_not_null_table_column(block, 0));
+        for (size_t row = 0; row < block.rows(); ++row) {
+            ids.push_back(id_column.get_element(row));
+        }
+    }
+    EXPECT_EQ(ids, expected);
+}
+
+// Paimon can write both bitmap versions into the same index file. The last 
entry ends at EOF.
+TEST(PaimonReaderTest, AppliesMixedBitmapDeletionVectorFile) {
     const auto test_dir =
             std::filesystem::temp_directory_path() / 
"doris_paimon_deletion_vector_file_test";
     std::filesystem::remove_all(test_dir);
@@ -947,7 +1037,22 @@ TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
     const auto dv_path = (test_dir / "delete-vector.bin").string();
     write_int_pair_parquet_file(file_path, {1, 2, 3, 4, 5}, {10, 20, 30, 40, 
50},
                                 {"one", "two", "three", "four", "five"});
-    const auto dv_length = write_paimon_deletion_vector_file(dv_path, {0, 4});
+    const auto bitmap32 = build_paimon_deletion_vector_buffer({0, 4});
+    const auto bitmap32_length = static_cast<uint32_t>(bitmap32.size() - 4);
+    const uint32_t bitmap32_crc = HashUtil::zlib_crc_hash(bitmap32.data() + 4, 
bitmap32_length, 0);
+    char crc_bytes[4];
+    BigEndian::Store32(crc_bytes, bitmap32_crc);
+    {
+        std::ofstream output(dv_path, std::ios::binary);
+        output.put(1); // Paimon index file version.
+        output.write(bitmap32.data(), 
static_cast<std::streamsize>(bitmap32.size()));
+        output.write(crc_bytes, sizeof(crc_bytes));
+        output.write(PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE);
+        output.write(PAIMON_BITMAP64_DV, PAIMON_BITMAP64_DV_SIZE);
+        ASSERT_TRUE(output.good());
+    }
+    const int64_t bitmap64_offset = 1 + bitmap32.size() + sizeof(crc_bytes);
+    const int64_t bitmap64_eof_offset = bitmap64_offset + 
PAIMON_BITMAP64_DV_SIZE;
 
     std::vector<ColumnDefinition> projected_columns;
     projected_columns.push_back(make_table_column(0, "id", 
std::make_shared<DataTypeInt32>()));
@@ -973,25 +1078,33 @@ TEST(PaimonReaderTest, AppliesBitmapDeletionVectorFile) {
 
     auto split_options = build_split_options(file_path);
     split_options.cache = &cache;
-    split_options.current_range.__set_table_format_params(
-            make_paimon_table_format_desc(dv_path, 0, dv_length));
-    ASSERT_TRUE(reader.prepare_split(split_options).ok());
-
-    std::vector<int32_t> ids;
-    bool eos = false;
-    while (!eos) {
-        Block block = build_table_block(projected_columns);
-        ASSERT_TRUE(reader.get_block(&block, &eos).ok());
-        if (block.rows() == 0) {
-            continue;
-        }
-        const auto& id_column =
-                assert_cast<const 
ColumnInt32&>(expect_not_null_table_column(block, 0));
-        for (size_t row = 0; row < block.rows(); ++row) {
-            ids.push_back(id_column.get_element(row));
-        }
+    const struct {
+        int64_t offset;
+        int64_t length;
+        std::vector<int32_t> expected;
+    } cases[] = {
+            {.offset = 1, .length = bitmap32_length, .expected = {2, 3, 4}},
+            {.offset = bitmap64_offset,
+             .length = PAIMON_BITMAP64_DV_SIZE,
+             .expected = {1, 3, 4, 5}},
+            {.offset = bitmap64_eof_offset,
+             .length = PAIMON_BITMAP64_DV_SIZE,
+             .expected = {1, 3, 4, 5}},
+            // Repeat the first split through the decoded cache.
+            {.offset = 1, .length = bitmap32_length, .expected = {2, 3, 4}},
+    };
+    for (const auto& test_case : cases) {
+        split_options.current_range.__set_table_format_params(
+                make_paimon_table_format_desc(dv_path, test_case.offset, 
test_case.length));
+        ASSERT_NO_FATAL_FAILURE(check_paimon_split_ids(reader, split_options, 
projected_columns,
+                                                       test_case.expected));
     }
-    EXPECT_EQ(ids, std::vector<int32_t>({2, 3, 4}));
+
+    
split_options.current_range.__set_table_format_params(make_paimon_table_format_desc(
+            dv_path, bitmap64_eof_offset, PAIMON_BITMAP64_DV_SIZE - 1));
+    const auto status = reader.prepare_split(split_options);
+    EXPECT_TRUE(status.is<ErrorCode::DATA_QUALITY_ERROR>());
+    EXPECT_NE(status.to_string().find("descriptor length mismatch"), 
std::string::npos);
 
     ASSERT_TRUE(reader.close().ok());
     std::filesystem::remove_all(test_dir);
diff --git 
a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out 
b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
index d8b9e547101..d6fca52f467 100644
--- 
a/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
+++ 
b/regression-test/data/external_table_p0/paimon/test_paimon_deletion_vector.out
@@ -95,3 +95,7 @@
 -- !13 --
 4
 
+-- !bitmap64_native --
+1      10
+3      30
+
diff --git 
a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
 
b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
index 172df875927..33bd56210a9 100644
--- 
a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
+++ 
b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector.groovy
@@ -105,4 +105,39 @@ suite("test_paimon_deletion_vector", "p0,external") {
         sql """set force_jni_scanner=false"""
     }
 
+    sql "CREATE DATABASE IF NOT EXISTS 
test_paimon_deletion_vector.paimon_bitmap64_dv_db"
+    sql "DROP TABLE IF EXISTS 
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv"
+    sql """
+        CREATE TABLE 
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv (
+            id INT NOT NULL, payload INT
+        ) ENGINE=paimon PROPERTIES (
+            'primary-key' = 'id',
+            'bucket' = '1',
+            'file.format' = 'parquet',
+            'deletion-vectors.enabled' = 'true',
+            'deletion-vectors.bitmap64' = 'true'
+        )
+    """
+    sql "INSERT INTO 
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv VALUES (1, 10), 
(2, 20), (3, 30)"
+    sql "DELETE FROM 
test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv WHERE id = 2"
+
+    // The native query must read a physical DV rather than an already 
rewritten data file.
+    def indexes = sql """
+        SELECT row_count
+        FROM 
test_paimon_deletion_vector.paimon_bitmap64_dv_db.`bitmap64_dv\$table_indexes`
+        WHERE index_type = 'DELETION_VECTORS' AND row_count > 0
+    """
+    assertFalse(indexes.isEmpty(), "DELETE must leave a deletion-vector entry")
+
+    def bitmap64Query = """
+        SELECT /*+ SET_VAR(enable_file_scanner_v2=true, 
enable_paimon_rust_reader=false, force_jni_scanner=false) */
+               id, payload
+        FROM test_paimon_deletion_vector.paimon_bitmap64_dv_db.bitmap64_dv
+        ORDER BY id
+    """
+    explain {
+        sql bitmap64Query
+        contains "paimonNativeReadSplits=1/1"
+    }
+    order_qt_bitmap64_native bitmap64Query
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to