zjw1111 commented on code in PR #449:
URL: https://github.com/apache/paimon-cpp/pull/449#discussion_r4409922733


##########
src/paimon/core/append/bucketed_append_compact_manager.h:
##########
@@ -208,6 +214,8 @@ class BucketedAppendCompactManager : public 
CompactFutureManager {
     std::optional<std::vector<std::shared_ptr<DataFileMeta>>> compacting_;
     DataFileMetaPriorityQueue to_compact_;
     std::shared_ptr<CancellationController> cancellation_controller_;
+    // Realtime deduplicate commits rebuild the bucket DV against their 
publication snapshot.

Review Comment:
   This comment doesn’t seem to explain the meaning of the bool variable below.



##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and 
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+    using DeletionVectors = std::map<std::string, 
std::shared_ptr<DeletionVector>>;
+    using FilePositions = std::map<std::string, RoaringBitmap32>;
+
+    struct BucketSnapshot {
+        std::vector<ManifestEntry> files;
+        std::vector<std::shared_ptr<IndexFileMeta>> dv_index_files;
+    };
+
+    using SnapshotBuckets = std::unordered_map<BinaryRow, std::map<int32_t, 
BucketSnapshot>>;
+
+    // Compaction F1/F2 -> F3 with F5 retained: lookup_files={F3}, 
final_file_names={F3, F5},
+    // and deleted_offsets contains offsets recovered from rows deleted by 
F1/F2's latest DVs.
+    struct BucketUpdate {
+        std::vector<std::shared_ptr<DataFileMeta>> lookup_files;
+        // Unlike lookup_files, this includes files not participating in 
compaction.
+        std::unordered_set<std::string> final_file_names;
+        RoaringBitmap64 deleted_offsets;
+    };
+
+    RealtimeDeduplicateDvMaterializer(std::shared_ptr<TableSchema> 
table_schema,
+                                      std::shared_ptr<FileStorePathFactory> 
path_factory,
+                                      std::shared_ptr<CommitScanner> scanner,
+                                      const CoreOptions& options, 
std::shared_ptr<MemoryPool> pool);
+
+    Result<SnapshotBuckets> LoadSnapshot(const std::optional<Snapshot>& 
snapshot,
+                                         const std::vector<BinaryRow>& 
partitions) const;
+
+    Result<DeletionVectors> ReadDeletionVectors(const BinaryRow& partition, 
int32_t bucket,
+                                                const BucketSnapshot& 
snapshot) const;
+
+    Result<FilePositions> LookupFilePositions(const BinaryRow& partition, 
int32_t bucket,
+                                              const BucketUpdate& update) 
const;
+
+    Status MaterializeBucket(const BinaryRow& partition, int32_t bucket,
+                             const BucketSnapshot& snapshot, DeletionVectors 
baseline,
+                             const BucketUpdate& update, const FilePositions& 
positions,
+                             CommitChanges* changes) const;
+
+    /// Only for files whose provider has not yet returned successfully.
+    void CleanUpUnpublishedFiles(const CommitChanges& changes) const;

Review Comment:
   here should use `const CommitChanges&` or `CommitChanges*`?



##########
src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp:
##########
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include 
"paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.h"
+
+#include <cstdint>
+#include <limits>
+#include <map>
+#include <string>
+#include <unordered_map>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/casting/casting_utils.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/read_context.h"
+#include "paimon/utils/range.h"
+
+namespace paimon {
+namespace {
+
+Status CollectDeletedOffsets(const ManifestEntry& source, const 
std::shared_ptr<DeletionVector>& dv,
+                             const std::shared_ptr<arrow::Schema>& 
offset_schema,
+                             const std::shared_ptr<MemoryPool>& pool, 
RawFileSplitRead* read,
+                             RoaringBitmap64* deleted_offsets) {
+    std::vector<Range> ranges;
+    bool invalid_position = false;
+    PAIMON_RETURN_NOT_OK(dv->ForEachDeletedPosition([&](int64_t position) {
+        if (position < 0 || position >= source.File()->row_count ||
+            position >= std::numeric_limits<int32_t>::max()) {
+            invalid_position = true;
+        } else if (!ranges.empty() && position == ranges.back().to + 1) {
+            ranges.back().to = position;
+        } else {
+            ranges.emplace_back(position, position);
+        }
+    }));
+    if (invalid_position) {
+        return Status::Invalid("compaction input DV contains an invalid row 
position");
+    }
+    if (ranges.empty()) {
+        return Status::OK();
+    }
+    ranges = Range::SortAndMergeOverlap(ranges, /*adjacent=*/true);
+    int64_t expected_rows = 0;
+    for (const Range& range : ranges) {
+        expected_rows += range.Count();
+    }
+    // Read deleted physical rows to recover their stable offsets. Applying 
the input DV here
+    // would hide precisely the rows whose deletions need to be transferred to 
the output.
+    DeletionVector::Factory no_deletions =
+        [](const std::string&) -> Result<std::shared_ptr<DeletionVector>> {
+        return std::shared_ptr<DeletionVector>();
+    };
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<BatchReader> reader,
+        read->CreateReader(source.Partition(), source.Bucket(), 
{source.File()}, no_deletions,
+                           ranges, offset_schema, /*predicate=*/nullptr));
+    ScopeGuard reader_guard([&]() { reader->Close(); });
+    const std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(pool);
+    int64_t emitted_rows = 0;
+    while (true) {
+        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap 
batch_with_bitmap,
+                               reader->NextBatchWithBitmap());
+        if (BatchReader::IsEofBatch(batch_with_bitmap)) {
+            break;
+        }
+        auto& [batch, selected_rows] = batch_with_bitmap;
+        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+            std::shared_ptr<arrow::Array> array,
+            arrow::ImportArray(batch.first.get(), batch.second.get()));
+        if (array->type_id() != arrow::Type::STRUCT) {
+            return Status::Invalid("compaction offset reader returned a 
non-struct array");
+        }
+        std::shared_ptr<arrow::Array> raw_offsets =
+            checked_pointer_cast<arrow::StructArray>(array)->GetFieldByName(
+                SpecialFields::RealtimeOffset().Name());
+        if (raw_offsets && raw_offsets->type_id() == arrow::Type::DICTIONARY) {
+            PAIMON_ASSIGN_OR_RAISE(
+                raw_offsets,
+                CastingUtils::Cast(raw_offsets, arrow::int64(), 
arrow::compute::CastOptions::Safe(),
+                                   arrow_pool.get()));
+        }
+        if (!raw_offsets || raw_offsets->type_id() != arrow::Type::INT64) {
+            return Status::Invalid("compaction input has no valid realtime 
offsets");
+        }
+        const auto offsets = 
checked_pointer_cast<arrow::Int64Array>(raw_offsets);
+        // The raw batch may contain unselected rows. Only selected offsets 
participate in
+        // migration and row-count validation; do not materialize a filtered 
Arrow array.
+        emitted_rows += selected_rows.Cardinality();
+        for (auto selected = selected_rows.Begin(); selected != 
selected_rows.End(); ++selected) {
+            const int64_t row = *selected;
+            if (row >= offsets->length()) {
+                return Status::Invalid("compaction selected row is out of 
bounds");
+            }
+            if (offsets->IsNull(row)) {
+                return Status::Invalid("compaction input contains a null 
realtime offset");
+            }
+            if (offsets->Value(row) < 0) {
+                return Status::Invalid("compaction input contains a negative 
realtime offset");
+            }
+            deleted_offsets->Add(offsets->Value(row));
+        }
+    }
+    if (emitted_rows != expected_rows) {
+        return Status::Invalid("compaction offset reader did not return all 
deleted rows");
+    }
+    return Status::OK();
+}
+
+}  // namespace
+
+Result<RealtimeCompactionPlan> RealtimeCompactionPlan::Create(
+    const std::vector<ManifestEntry>& entries) {
+    std::unordered_map<FileEntry::Identifier, const ManifestEntry*> inputs;
+    std::unordered_map<FileEntry::Identifier, const ManifestEntry*> outputs;
+    for (const ManifestEntry& entry : entries) {
+        if (!entry.File() || entry.File()->row_count < 0) {
+            return Status::Invalid("invalid realtime compaction file");
+        }
+        if (!(entry.Kind() == FileKind::Add()) && !(entry.Kind() == 
FileKind::Delete())) {
+            return Status::Invalid("invalid realtime compaction entry kind");
+        }
+        auto& files = entry.Kind() == FileKind::Add() ? outputs : inputs;
+        if (!files.emplace(entry.CreateIdentifier(), &entry).second) {
+            return Status::Invalid("duplicate realtime compaction input or 
output: ",
+                                   entry.FileName());
+        }
+    }
+    // For F1/F2 -> F3 -> F4, delta_files contains DELETE F1/F2 and ADD F4.
+    // intermediate_files contains ADD F3, whose matching DELETE cancels it 
from the final changes.
+    RealtimeCompactionPlan plan;
+    for (const ManifestEntry& entry : entries) {
+        const auto id = entry.CreateIdentifier();
+        if (entry.Kind() == FileKind::Delete()) {
+            if (outputs.count(id) == 0) {
+                plan.delta_files.push_back(entry);
+            } else {
+                const auto& output = outputs.at(id)->File();
+                if (entry.File()->row_count != output->row_count ||
+                    entry.File()->schema_id != output->schema_id ||
+                    entry.File()->file_size != output->file_size) {
+                    return Status::Invalid("inconsistent intermediate 
compaction file: ",
+                                           entry.FileName());
+                }
+            }
+        } else if (inputs.count(id) == 0) {
+            plan.delta_files.push_back(entry);
+        } else {
+            plan.intermediate_files.push_back(entry);
+        }
+    }
+    return plan;
+}
+
+RealtimeDeduplicateCompactionChangesProvider::RealtimeDeduplicateCompactionChangesProvider(
+    RealtimeCompactionPlan plan, std::vector<ManifestEntry> changelog_files,
+    std::vector<IndexManifestEntry> index_entries, 
std::shared_ptr<TableSchema> table_schema,
+    std::shared_ptr<FileStorePathFactory> path_factory, 
std::shared_ptr<CommitScanner> scanner,
+    const CoreOptions& options, std::shared_ptr<Executor> executor,
+    std::shared_ptr<MemoryPool> pool)
+    : plan_(std::move(plan)),
+      changelog_files_(std::move(changelog_files)),
+      index_entries_(std::move(index_entries)),
+      table_schema_(std::move(table_schema)),
+      path_factory_(std::move(path_factory)),
+      options_(options),
+      executor_(std::move(executor)),
+      pool_(std::move(pool)),
+      materializer_(table_schema_, path_factory_, std::move(scanner), 
options_, pool_) {}
+
+Result<std::shared_ptr<CommitChanges>> 
RealtimeDeduplicateCompactionChangesProvider::Provide(

Review Comment:
   this func is more than 100 lines



##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and 
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+    using DeletionVectors = std::map<std::string, 
std::shared_ptr<DeletionVector>>;

Review Comment:
   this using also can write in a base file



##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and 
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+    using DeletionVectors = std::map<std::string, 
std::shared_ptr<DeletionVector>>;
+    using FilePositions = std::map<std::string, RoaringBitmap32>;

Review Comment:
   if need `using`, maybe using in 
src/paimon/core/realtime/realtime_offset_file_index_lookup.h is better?



##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and 
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+    using DeletionVectors = std::map<std::string, 
std::shared_ptr<DeletionVector>>;
+    using FilePositions = std::map<std::string, RoaringBitmap32>;
+
+    struct BucketSnapshot {
+        std::vector<ManifestEntry> files;
+        std::vector<std::shared_ptr<IndexFileMeta>> dv_index_files;
+    };
+
+    using SnapshotBuckets = std::unordered_map<BinaryRow, std::map<int32_t, 
BucketSnapshot>>;

Review Comment:
   add a comment for what does `BinaryRow` and `int32_t` mean



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to