SteNicholas commented on code in PR #357:
URL: https://github.com/apache/paimon-cpp/pull/357#discussion_r4033766730
##########
test/inte/pk_compaction_inte_test.cpp:
##########
@@ -1954,35 +1954,77 @@ TEST_F(PkCompactionInteTest, WriteAndCompactWithBranch)
{
ASSERT_EQ(compact_after[0]->level, 5);
ASSERT_EQ(compact_after[0]->row_count, 2);
- // Step 4: Fake a DataSplit from compact_after to read and verify the
compacted data.
- {
- ReadContextBuilder read_context_builder(table_path);
- read_context_builder.WithBranch("rt");
- ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
- ASSERT_OK_AND_ASSIGN(auto table_read,
TableRead::Create(std::move(read_context)));
+ // Step 4: Commit to branch-rt.
+ CommitContextBuilder commit_builder(table_path, "commit_user_1");
+ commit_builder.AddOption(Options::FILE_SYSTEM,
"local").AddOption(Options::BRANCH, "rt");
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<CommitContext> commit_context,
commit_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileStoreCommit> file_store_commit,
+ FileStoreCommit::Create(std::move(commit_context)));
+ ASSERT_OK(file_store_commit->Commit(commit_msgs));
- // Build a fake DataSplit using the compact_after file metadata.
- auto fake_split = BuildSplit(commit_msgs, /*snapshot_id=*/1);
- ASSERT_OK_AND_ASSIGN(auto batch_reader,
-
table_read->CreateReader(std::shared_ptr<Split>(fake_split)));
- ASSERT_OK_AND_ASSIGN(auto result_array,
-
ReadResultCollector::CollectResult(std::move(batch_reader)));
+ auto fs = std::make_shared<LocalFileSystem>();
+ ASSERT_OK_AND_ASSIGN(
+ bool branch_append_snapshot,
+ fs->Exists(PathUtil::JoinPath(table_path,
"branch/branch-rt/snapshot/snapshot-2")));
+ ASSERT_TRUE(branch_append_snapshot);
+ ASSERT_OK_AND_ASSIGN(
+ bool branch_compact_snapshot,
+ fs->Exists(PathUtil::JoinPath(table_path,
"branch/branch-rt/snapshot/snapshot-3")));
+ ASSERT_TRUE(branch_compact_snapshot);
+ ASSERT_OK_AND_ASSIGN(bool main_snapshot_exists,
+ fs->Exists(PathUtil::JoinPath(table_path,
"snapshot/snapshot-2")));
+ ASSERT_FALSE(main_snapshot_exists);
+
+ // Step 5: Scan branch-rt and read the compacted file.
+ std::map<std::string, std::string> branch_options =
{{Options::FILE_SYSTEM, "local"},
+ {Options::BRANCH,
"rt"}};
+ ScanContextBuilder scan_context_builder(table_path);
+ scan_context_builder.WithStreamingMode(false)
+ .SetOptions(branch_options)
+ .AddOption(Options::SCAN_MODE, StartupMode::LatestFull().ToString());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<ScanContext> scan_context,
scan_context_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableScan> table_scan,
+ TableScan::Create(std::move(scan_context)));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<Plan> plan, table_scan->CreatePlan());
- arrow::FieldVector fields_with_row_kind = fields;
- fields_with_row_kind.insert(fields_with_row_kind.begin(),
- arrow::field("_VALUE_KIND",
arrow::int8()));
- auto result_type = arrow::struct_(fields_with_row_kind);
+ std::vector<std::shared_ptr<Split>> compacted_splits;
+ for (const std::shared_ptr<Split>& split : plan->Splits()) {
+ auto* split_impl = dynamic_cast<DataSplitImpl*>(split.get());
+ ASSERT_NE(split_impl, nullptr);
+ for (const std::shared_ptr<DataFileMeta>& file :
split_impl->DataFiles()) {
+ if (file->file_name == compact_after[0]->file_name) {
+ ASSERT_EQ(split_impl->SnapshotId(), 3);
+ compacted_splits.push_back(split);
+ break;
+ }
+ }
+ }
+ ASSERT_EQ(compacted_splits.size(), 1u);
Review Comment:
Done in 0af0eea. The test no longer picks the split holding the compacted
file: it asserts `SnapshotId() == 3` for every split of `branch-rt` and reads
all of them, so the expected result is now every row of the branch.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
