zjw1111 commented on code in PR #449:
URL: https://github.com/apache/paimon-cpp/pull/449#discussion_r4409922733
##########
src/paimon/core/append/bucketed_append_compact_manager.h:
##########
@@ -208,6 +214,8 @@ class BucketedAppendCompactManager : public
CompactFutureManager {
std::optional<std::vector<std::shared_ptr<DataFileMeta>>> compacting_;
DataFileMetaPriorityQueue to_compact_;
std::shared_ptr<CancellationController> cancellation_controller_;
+ // Realtime deduplicate commits rebuild the bucket DV against their
publication snapshot.
Review Comment:
This comment doesn’t seem to explain the meaning of the bool variable below.
##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+ using DeletionVectors = std::map<std::string,
std::shared_ptr<DeletionVector>>;
+ using FilePositions = std::map<std::string, RoaringBitmap32>;
+
+ struct BucketSnapshot {
+ std::vector<ManifestEntry> files;
+ std::vector<std::shared_ptr<IndexFileMeta>> dv_index_files;
+ };
+
+ using SnapshotBuckets = std::unordered_map<BinaryRow, std::map<int32_t,
BucketSnapshot>>;
+
+ // Compaction F1/F2 -> F3 with F5 retained: lookup_files={F3},
final_file_names={F3, F5},
+ // and deleted_offsets contains offsets recovered from rows deleted by
F1/F2's latest DVs.
+ struct BucketUpdate {
+ std::vector<std::shared_ptr<DataFileMeta>> lookup_files;
+ // Unlike lookup_files, this includes files not participating in
compaction.
+ std::unordered_set<std::string> final_file_names;
+ RoaringBitmap64 deleted_offsets;
+ };
+
+ RealtimeDeduplicateDvMaterializer(std::shared_ptr<TableSchema>
table_schema,
+ std::shared_ptr<FileStorePathFactory>
path_factory,
+ std::shared_ptr<CommitScanner> scanner,
+ const CoreOptions& options,
std::shared_ptr<MemoryPool> pool);
+
+ Result<SnapshotBuckets> LoadSnapshot(const std::optional<Snapshot>&
snapshot,
+ const std::vector<BinaryRow>&
partitions) const;
+
+ Result<DeletionVectors> ReadDeletionVectors(const BinaryRow& partition,
int32_t bucket,
+ const BucketSnapshot&
snapshot) const;
+
+ Result<FilePositions> LookupFilePositions(const BinaryRow& partition,
int32_t bucket,
+ const BucketUpdate& update)
const;
+
+ Status MaterializeBucket(const BinaryRow& partition, int32_t bucket,
+ const BucketSnapshot& snapshot, DeletionVectors
baseline,
+ const BucketUpdate& update, const FilePositions&
positions,
+ CommitChanges* changes) const;
+
+ /// Only for files whose provider has not yet returned successfully.
+ void CleanUpUnpublishedFiles(const CommitChanges& changes) const;
Review Comment:
here should use `const CommitChanges&` or `CommitChanges*`?
##########
src/paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.cpp:
##########
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include
"paimon/core/operation/commit/realtime_deduplicate_compaction_changes_provider.h"
+
+#include <cstdint>
+#include <limits>
+#include <map>
+#include <string>
+#include <unordered_map>
+
+#include "arrow/api.h"
+#include "arrow/c/bridge.h"
+#include "paimon/common/table/special_fields.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/scope_guard.h"
+#include "paimon/core/casting/casting_utils.h"
+#include "paimon/core/deletionvectors/deletion_vectors_index_file.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/operation/raw_file_split_read.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/read_context.h"
+#include "paimon/utils/range.h"
+
+namespace paimon {
+namespace {
+
+Status CollectDeletedOffsets(const ManifestEntry& source, const
std::shared_ptr<DeletionVector>& dv,
+ const std::shared_ptr<arrow::Schema>&
offset_schema,
+ const std::shared_ptr<MemoryPool>& pool,
RawFileSplitRead* read,
+ RoaringBitmap64* deleted_offsets) {
+ std::vector<Range> ranges;
+ bool invalid_position = false;
+ PAIMON_RETURN_NOT_OK(dv->ForEachDeletedPosition([&](int64_t position) {
+ if (position < 0 || position >= source.File()->row_count ||
+ position >= std::numeric_limits<int32_t>::max()) {
+ invalid_position = true;
+ } else if (!ranges.empty() && position == ranges.back().to + 1) {
+ ranges.back().to = position;
+ } else {
+ ranges.emplace_back(position, position);
+ }
+ }));
+ if (invalid_position) {
+ return Status::Invalid("compaction input DV contains an invalid row
position");
+ }
+ if (ranges.empty()) {
+ return Status::OK();
+ }
+ ranges = Range::SortAndMergeOverlap(ranges, /*adjacent=*/true);
+ int64_t expected_rows = 0;
+ for (const Range& range : ranges) {
+ expected_rows += range.Count();
+ }
+ // Read deleted physical rows to recover their stable offsets. Applying
the input DV here
+ // would hide precisely the rows whose deletions need to be transferred to
the output.
+ DeletionVector::Factory no_deletions =
+ [](const std::string&) -> Result<std::shared_ptr<DeletionVector>> {
+ return std::shared_ptr<DeletionVector>();
+ };
+ PAIMON_ASSIGN_OR_RAISE(
+ std::unique_ptr<BatchReader> reader,
+ read->CreateReader(source.Partition(), source.Bucket(),
{source.File()}, no_deletions,
+ ranges, offset_schema, /*predicate=*/nullptr));
+ ScopeGuard reader_guard([&]() { reader->Close(); });
+ const std::shared_ptr<arrow::MemoryPool> arrow_pool = GetArrowPool(pool);
+ int64_t emitted_rows = 0;
+ while (true) {
+ PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatchWithBitmap
batch_with_bitmap,
+ reader->NextBatchWithBitmap());
+ if (BatchReader::IsEofBatch(batch_with_bitmap)) {
+ break;
+ }
+ auto& [batch, selected_rows] = batch_with_bitmap;
+ PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(
+ std::shared_ptr<arrow::Array> array,
+ arrow::ImportArray(batch.first.get(), batch.second.get()));
+ if (array->type_id() != arrow::Type::STRUCT) {
+ return Status::Invalid("compaction offset reader returned a
non-struct array");
+ }
+ std::shared_ptr<arrow::Array> raw_offsets =
+ checked_pointer_cast<arrow::StructArray>(array)->GetFieldByName(
+ SpecialFields::RealtimeOffset().Name());
+ if (raw_offsets && raw_offsets->type_id() == arrow::Type::DICTIONARY) {
+ PAIMON_ASSIGN_OR_RAISE(
+ raw_offsets,
+ CastingUtils::Cast(raw_offsets, arrow::int64(),
arrow::compute::CastOptions::Safe(),
+ arrow_pool.get()));
+ }
+ if (!raw_offsets || raw_offsets->type_id() != arrow::Type::INT64) {
+ return Status::Invalid("compaction input has no valid realtime
offsets");
+ }
+ const auto offsets =
checked_pointer_cast<arrow::Int64Array>(raw_offsets);
+ // The raw batch may contain unselected rows. Only selected offsets
participate in
+ // migration and row-count validation; do not materialize a filtered
Arrow array.
+ emitted_rows += selected_rows.Cardinality();
+ for (auto selected = selected_rows.Begin(); selected !=
selected_rows.End(); ++selected) {
+ const int64_t row = *selected;
+ if (row >= offsets->length()) {
+ return Status::Invalid("compaction selected row is out of
bounds");
+ }
+ if (offsets->IsNull(row)) {
+ return Status::Invalid("compaction input contains a null
realtime offset");
+ }
+ if (offsets->Value(row) < 0) {
+ return Status::Invalid("compaction input contains a negative
realtime offset");
+ }
+ deleted_offsets->Add(offsets->Value(row));
+ }
+ }
+ if (emitted_rows != expected_rows) {
+ return Status::Invalid("compaction offset reader did not return all
deleted rows");
+ }
+ return Status::OK();
+}
+
+} // namespace
+
+Result<RealtimeCompactionPlan> RealtimeCompactionPlan::Create(
+ const std::vector<ManifestEntry>& entries) {
+ std::unordered_map<FileEntry::Identifier, const ManifestEntry*> inputs;
+ std::unordered_map<FileEntry::Identifier, const ManifestEntry*> outputs;
+ for (const ManifestEntry& entry : entries) {
+ if (!entry.File() || entry.File()->row_count < 0) {
+ return Status::Invalid("invalid realtime compaction file");
+ }
+ if (!(entry.Kind() == FileKind::Add()) && !(entry.Kind() ==
FileKind::Delete())) {
+ return Status::Invalid("invalid realtime compaction entry kind");
+ }
+ auto& files = entry.Kind() == FileKind::Add() ? outputs : inputs;
+ if (!files.emplace(entry.CreateIdentifier(), &entry).second) {
+ return Status::Invalid("duplicate realtime compaction input or
output: ",
+ entry.FileName());
+ }
+ }
+ // For F1/F2 -> F3 -> F4, delta_files contains DELETE F1/F2 and ADD F4.
+ // intermediate_files contains ADD F3, whose matching DELETE cancels it
from the final changes.
+ RealtimeCompactionPlan plan;
+ for (const ManifestEntry& entry : entries) {
+ const auto id = entry.CreateIdentifier();
+ if (entry.Kind() == FileKind::Delete()) {
+ if (outputs.count(id) == 0) {
+ plan.delta_files.push_back(entry);
+ } else {
+ const auto& output = outputs.at(id)->File();
+ if (entry.File()->row_count != output->row_count ||
+ entry.File()->schema_id != output->schema_id ||
+ entry.File()->file_size != output->file_size) {
+ return Status::Invalid("inconsistent intermediate
compaction file: ",
+ entry.FileName());
+ }
+ }
+ } else if (inputs.count(id) == 0) {
+ plan.delta_files.push_back(entry);
+ } else {
+ plan.intermediate_files.push_back(entry);
+ }
+ }
+ return plan;
+}
+
+RealtimeDeduplicateCompactionChangesProvider::RealtimeDeduplicateCompactionChangesProvider(
+ RealtimeCompactionPlan plan, std::vector<ManifestEntry> changelog_files,
+ std::vector<IndexManifestEntry> index_entries,
std::shared_ptr<TableSchema> table_schema,
+ std::shared_ptr<FileStorePathFactory> path_factory,
std::shared_ptr<CommitScanner> scanner,
+ const CoreOptions& options, std::shared_ptr<Executor> executor,
+ std::shared_ptr<MemoryPool> pool)
+ : plan_(std::move(plan)),
+ changelog_files_(std::move(changelog_files)),
+ index_entries_(std::move(index_entries)),
+ table_schema_(std::move(table_schema)),
+ path_factory_(std::move(path_factory)),
+ options_(options),
+ executor_(std::move(executor)),
+ pool_(std::move(pool)),
+ materializer_(table_schema_, path_factory_, std::move(scanner),
options_, pool_) {}
+
+Result<std::shared_ptr<CommitChanges>>
RealtimeDeduplicateCompactionChangesProvider::Provide(
Review Comment:
this func is more than 100 lines
##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+ using DeletionVectors = std::map<std::string,
std::shared_ptr<DeletionVector>>;
Review Comment:
this using also can write in a base file
##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+ using DeletionVectors = std::map<std::string,
std::shared_ptr<DeletionVector>>;
+ using FilePositions = std::map<std::string, RoaringBitmap32>;
Review Comment:
if need `using`, maybe using in
src/paimon/core/realtime/realtime_offset_file_index_lookup.h is better?
##########
src/paimon/core/operation/commit/realtime_deduplicate_dv_materializer.h:
##########
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include "paimon/core/core_options.h"
+#include "paimon/core/operation/commit/commit_changes_provider.h"
+#include "paimon/utils/roaring_bitmap64.h"
+
+namespace paimon {
+
+class CommitScanner;
+class DeletionVector;
+class DeletionVectorsIndexFile;
+class FileStorePathFactory;
+class MemoryPool;
+class TableSchema;
+
+/// Shared, attempt-local DV processing for realtime deduplicate writes and
compaction.
+class RealtimeDeduplicateDvMaterializer {
+ public:
+ using DeletionVectors = std::map<std::string,
std::shared_ptr<DeletionVector>>;
+ using FilePositions = std::map<std::string, RoaringBitmap32>;
+
+ struct BucketSnapshot {
+ std::vector<ManifestEntry> files;
+ std::vector<std::shared_ptr<IndexFileMeta>> dv_index_files;
+ };
+
+ using SnapshotBuckets = std::unordered_map<BinaryRow, std::map<int32_t,
BucketSnapshot>>;
Review Comment:
add a comment for what does `BinaryRow` and `int32_t` mean
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]