SteNicholas commented on code in PR #447: URL: https://github.com/apache/paimon-cpp/pull/447#discussion_r4410730832
########## src/paimon/core/table/file_store_table.cpp: ########## @@ -0,0 +1,675 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store_table.h" + +#include <map> +#include <memory> +#include <optional> +#include <set> +#include <string> +#include <utility> + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +namespace { + +/// The escape hatch that lets an integration test commit on an object store, read where +/// `NewCommit` refuses one below. +constexpr const char kObjectStoreCommitInteTestOption[] = "enable-object-store-commit-in-inte-test"; + +} // namespace + +class Executor; +class MemoryPool; + +FileStoreTable::FileStoreTable(const std::shared_ptr<FileStore>& store, + const std::shared_ptr<TableSchema>& schema, + const Identifier& identifier, const std::string& uuid) + : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, uuid), + store_(store) {} + +Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create( + const std::shared_ptr<FileSystem>& specified_file_system, const std::string& root_path, + const std::shared_ptr<TableSchema>& schema, const Identifier& identifier, + const std::string& uuid, const std::string& branch, + const std::map<std::string, std::string>& fs_scheme_map, + const std::map<std::string, std::string>& dynamic_options) { + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema of {}", identifier.GetFullName())); + } + std::map<std::string, std::string> options = schema->Options(); + const std::string normalized_branch = BranchManager::NormalizeBranch(branch); + if (BranchManager::IsMainBranch(normalized_branch)) { + options.erase(Options::BRANCH); + } else { + options[Options::BRANCH] = normalized_branch; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options, specified_file_system, fs_scheme_map)); + std::shared_ptr<FileStore> store = FileStore::Create(root_path, std::move(core_options)); + std::shared_ptr<FileStoreTable> table; + if (!schema->PrimaryKeys().empty()) { + table = std::make_shared<PrimaryKeyFileStoreTable>(store, schema, identifier, uuid); + } else { + table = std::make_shared<AppendOnlyFileStoreTable>(store, schema, identifier, uuid); + } + table->dynamic_options_ = dynamic_options; + table->specified_file_system_ = specified_file_system; + table->fs_scheme_map_ = fs_scheme_map; + return table; +} + +Result<std::shared_ptr<Table>> FileStoreTable::Copy( + const std::map<std::string, std::string>& dynamic_options) const { + std::shared_ptr<TableSchema> loaded_schema = std::static_pointer_cast<TableSchema>(schema_); + // `type` is immutable the way Java Paimon holds it: the kind of a copy is named by the primary + // keys of the schema it serves, never by this option. A value naming another type is refused, + // and one naming the same type - whatever its case, an absent `type` reading as `table` - is + // dropped, as Java's unchanged-key check drops it. + auto type_option = dynamic_options.find(Options::TYPE); + if (type_option != dynamic_options.end()) { + PAIMON_ASSIGN_OR_RAISE(TableType schema_type, + TableTypeDefine::FromOptions(loaded_schema->Options())); + std::map<std::string, std::string> named{{Options::TYPE, type_option->second}}; + PAIMON_ASSIGN_OR_RAISE(TableType named_type, TableTypeDefine::FromOptions(named)); + if (named_type != schema_type) { + return Status::Invalid(fmt::format("Change '{}' is not supported yet.", Options::TYPE)); + } + } + // `branch` names the branch the copy serves, the way it routes a load: a copy on the branch + // this table serves keeps this table's schema, while one on another branch reads that branch's + // own latest schema and refuses a branch that holds none, as `SwitchToBranch` does. + std::string target_branch = CurrentBranch(); + auto branch_option = dynamic_options.find(Options::BRANCH); + if (branch_option != dynamic_options.end()) { + target_branch = BranchManager::NormalizeBranch(branch_option->second); + } + std::map<std::string, std::string> overridable = dynamic_options; + overridable.erase(Options::TYPE); + overridable.erase(Options::BRANCH); + std::shared_ptr<TableSchema> base_schema = loaded_schema; + // An `Identifier` is immutable once built, so the branch one is held beside this table's + // and the copy carries whichever names the branch it serves. + std::optional<Identifier> branch_identifier; + if (target_branch != CurrentBranch()) { + // The branch is read through the file system this table serves, the same read + // `SwitchToBranch` makes, and then served through the one its own schema resolves. + SchemaManager schema_manager(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest, + schema_manager.Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + base_schema = *latest; + PAIMON_ASSIGN_OR_RAISE(Identifier branch_named, + CatalogUtils::BranchIdentifier(identifier_, target_branch)); + branch_identifier.emplace(std::move(branch_named)); + } + // The schema options name the branch the copy serves, the way `SwitchToBranch` styles them: + // another branch names itself in them, the main branch carries no `branch` option. The key is + // folded into the schema alone, not into the dynamic options the copy accumulates. + std::map<std::string, std::string> schema_overridable = overridable; + if (target_branch != CurrentBranch() && target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + schema_overridable[Options::BRANCH] = target_branch; + } + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, overridable)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> merged_schema, + base_schema->MergeOptions(schema_overridable)); + std::map<std::string, std::string> accumulated = dynamic_options_; + accumulated.erase(Options::TYPE); + accumulated.erase(Options::BRANCH); + for (const auto& [key, value] : overridable) { + accumulated.insert_or_assign(key, value); Review Comment: **Merge accumulated overrides into a branch copy's schema.** When `Copy()` switches branches, `base_schema` becomes the target branch's persisted schema, but `MergeOptions()` receives only this call's overrides. Earlier `dynamic_options_` are retained in `accumulated` without reaching the new store's `CoreOptions`. For example, `Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})` followed by `Copy({{Options::BRANCH, "dev"}})` loses sequence-number support when `dev` omits that setting, causing an `_SEQUENCE_NUMBER` projection to fail even though option validation still considers the setting enabled. Please merge the accumulated overrides consistently into the target schema and add a regression test for successive copies. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
