SteNicholas commented on code in PR #447:
URL: https://github.com/apache/paimon-cpp/pull/447#discussion_r4410730825
##########
src/paimon/core/table/system/system_table.cpp:
##########
@@ -176,17 +186,68 @@ std::optional<SystemTableFactory>
FindSystemTableFactory(const std::string& syst
} // namespace
+SystemTable::SystemTable(const std::shared_ptr<FileSystem>& fs, const
std::string& table_path,
+ std::string name)
+ : Table(fs, table_path, /*schema=*/nullptr,
+ Identifier(Identifier::kUnknownDatabase, "system_table"),
/*uuid=*/""),
+ name_(std::move(name)) {}
+
+std::string SystemTable::ComposeName(const std::string& data_table_name, const
std::string& kind) {
+ return data_table_name + Catalog::SYSTEM_TABLE_SPLITTER + kind;
+}
+
+Status SystemTable::CheckNoSearch(const std::shared_ptr<ReadContext>& context)
{
+ if (context->HasVectorOrFullTextSearch()) {
+ return Status::NotImplemented(
+ "VectorSearch or FullTextSearch does not support system-table
reads yet");
+ }
+ if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) {
+ return Status::NotImplemented("FullTextMatch predicate execution is
not implemented yet");
+ }
+ return Status::OK();
+}
+
+Status SystemTable::AdoptOwnSchema() {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> arrow_schema,
ArrowSchema());
+ schema_ = std::make_shared<SystemTableSchema>(std::move(arrow_schema));
+ return Status::OK();
+}
+
+Result<std::shared_ptr<Table>> SystemTable::Copy(
+ const std::map<std::string, std::string>& /*dynamic_options*/) const {
+ return Status::NotImplemented(Name(), " system table is read-only: copying
is not supported");
Review Comment:
**[P2] Preserve dynamic scan options for system-table views.** Path-based
factories now reject system tables, but this unconditional rejection also
prevents configuring their replacement catalog-loaded handles. For example,
`NewScan()` on `tbl$ro` ignores context options `scan.mode=from-snapshot` and
`scan.snapshot-id`, using only the options captured when the table was loaded;
`Copy()` cannot apply them either. A historical scan consequently reads the
latest snapshot. Please preserve a supported way to apply dynamic options to
`read_optimized` and `audit_log` views, and test the selected snapshot through
`Catalog::GetTable()`.
##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -0,0 +1,675 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/file_store_table.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/sensitive_config_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/snapshot_commit.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/disk/io_manager.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
+#include "paimon/core/operation/expire_snapshots.h"
+#include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/append_only_file_store_table.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/primary_key_file_store_table.h"
+#include "paimon/core/table/source/append_only_table_read.h"
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+#include "paimon/core/table/source/data_table_batch_scan.h"
+#include "paimon/core/table/source/data_table_stream_scan.h"
+#include "paimon/core/table/source/fallback_table_read.h"
+#include "paimon/core/table/source/key_value_table_read.h"
+#include "paimon/core/table/source/primary_key_index_batch_scan.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
+#include "paimon/core/table/source/realtime_table_scan.h"
+#include "paimon/core/table/source/snapshot/snapshot_reader.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/context_utils.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/read_context.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/result.h"
+#include "paimon/scan_context.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/write_context.h"
+
+namespace arrow {
+class Schema;
+} // namespace arrow
+
+namespace paimon {
+
+namespace {
+
+/// The escape hatch that lets an integration test commit on an object store,
read where
+/// `NewCommit` refuses one below.
+constexpr const char kObjectStoreCommitInteTestOption[] =
"enable-object-store-commit-in-inte-test";
+
+} // namespace
+
+class Executor;
+class MemoryPool;
+
+FileStoreTable::FileStoreTable(const std::shared_ptr<FileStore>& store,
+ const std::shared_ptr<TableSchema>& schema,
+ const Identifier& identifier, const
std::string& uuid)
+ : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier,
uuid),
+ store_(store) {}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create(
+ const std::shared_ptr<FileSystem>& specified_file_system, const
std::string& root_path,
+ const std::shared_ptr<TableSchema>& schema, const Identifier& identifier,
+ const std::string& uuid, const std::string& branch,
+ const std::map<std::string, std::string>& fs_scheme_map,
+ const std::map<std::string, std::string>& dynamic_options) {
+ if (schema == nullptr) {
+ return Status::Invalid(
+ fmt::format("not found latest schema of {}",
identifier.GetFullName()));
+ }
+ std::map<std::string, std::string> options = schema->Options();
+ const std::string normalized_branch =
BranchManager::NormalizeBranch(branch);
+ if (BranchManager::IsMainBranch(normalized_branch)) {
+ options.erase(Options::BRANCH);
+ } else {
+ options[Options::BRANCH] = normalized_branch;
+ }
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(options,
specified_file_system, fs_scheme_map));
+ std::shared_ptr<FileStore> store = FileStore::Create(root_path,
std::move(core_options));
+ std::shared_ptr<FileStoreTable> table;
+ if (!schema->PrimaryKeys().empty()) {
+ table = std::make_shared<PrimaryKeyFileStoreTable>(store, schema,
identifier, uuid);
+ } else {
+ table = std::make_shared<AppendOnlyFileStoreTable>(store, schema,
identifier, uuid);
+ }
+ table->dynamic_options_ = dynamic_options;
+ table->specified_file_system_ = specified_file_system;
+ table->fs_scheme_map_ = fs_scheme_map;
+ return table;
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::Copy(
+ const std::map<std::string, std::string>& dynamic_options) const {
+ std::shared_ptr<TableSchema> loaded_schema =
std::static_pointer_cast<TableSchema>(schema_);
+ // `type` is immutable the way Java Paimon holds it: the kind of a copy is
named by the primary
+ // keys of the schema it serves, never by this option. A value naming
another type is refused,
+ // and one naming the same type - whatever its case, an absent `type`
reading as `table` - is
+ // dropped, as Java's unchanged-key check drops it.
+ auto type_option = dynamic_options.find(Options::TYPE);
+ if (type_option != dynamic_options.end()) {
+ PAIMON_ASSIGN_OR_RAISE(TableType schema_type,
+
TableTypeDefine::FromOptions(loaded_schema->Options()));
+ std::map<std::string, std::string> named{{Options::TYPE,
type_option->second}};
+ PAIMON_ASSIGN_OR_RAISE(TableType named_type,
TableTypeDefine::FromOptions(named));
+ if (named_type != schema_type) {
+ return Status::Invalid(fmt::format("Change '{}' is not supported
yet.", Options::TYPE));
+ }
+ }
+ // `branch` names the branch the copy serves, the way it routes a load: a
copy on the branch
+ // this table serves keeps this table's schema, while one on another
branch reads that branch's
+ // own latest schema and refuses a branch that holds none, as
`SwitchToBranch` does.
+ std::string target_branch = CurrentBranch();
+ auto branch_option = dynamic_options.find(Options::BRANCH);
+ if (branch_option != dynamic_options.end()) {
+ target_branch = BranchManager::NormalizeBranch(branch_option->second);
+ }
+ std::map<std::string, std::string> overridable = dynamic_options;
+ overridable.erase(Options::TYPE);
+ overridable.erase(Options::BRANCH);
+ std::shared_ptr<TableSchema> base_schema = loaded_schema;
+ // An `Identifier` is immutable once built, so the branch one is held
beside this table's
+ // and the copy carries whichever names the branch it serves.
+ std::optional<Identifier> branch_identifier;
+ if (target_branch != CurrentBranch()) {
+ // The branch is read through the file system this table serves, the
same read
+ // `SwitchToBranch` makes, and then served through the one its own
schema resolves.
+ SchemaManager schema_manager(file_system_, root_path_, target_branch);
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>>
latest,
+ schema_manager.Latest());
+ if (!latest) {
+ return Status::Invalid(fmt::format("branch {} does not exist",
target_branch));
+ }
+ base_schema = *latest;
+ PAIMON_ASSIGN_OR_RAISE(Identifier branch_named,
+ CatalogUtils::BranchIdentifier(identifier_,
target_branch));
+ branch_identifier.emplace(std::move(branch_named));
+ }
+ // The schema options name the branch the copy serves, the way
`SwitchToBranch` styles them:
+ // another branch names itself in them, the main branch carries no
`branch` option. The key is
+ // folded into the schema alone, not into the dynamic options the copy
accumulates.
+ std::map<std::string, std::string> schema_overridable = overridable;
+ if (target_branch != CurrentBranch() && target_branch !=
BranchManager::DEFAULT_MAIN_BRANCH) {
+ schema_overridable[Options::BRANCH] = target_branch;
+ }
+ PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema,
overridable));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> merged_schema,
+ base_schema->MergeOptions(schema_overridable));
+ std::map<std::string, std::string> accumulated = dynamic_options_;
+ accumulated.erase(Options::TYPE);
+ accumulated.erase(Options::BRANCH);
+ for (const auto& [key, value] : overridable) {
+ accumulated.insert_or_assign(key, value);
Review Comment:
**[P2] Merge accumulated overrides into a branch copy's schema.** When
`Copy()` switches branches, `base_schema` becomes the target branch's persisted
schema, but `MergeOptions()` receives only this call's overrides. Earlier
`dynamic_options_` are retained in `accumulated` without reaching the new
store's `CoreOptions`. For example,
`Copy({{Options::KEY_VALUE_SEQUENCE_NUMBER_ENABLED, "true"}})` followed by
`Copy({{Options::BRANCH, "dev"}})` loses sequence-number support when `dev`
omits that setting, causing an `_SEQUENCE_NUMBER` projection to fail even
though option validation still considers the setting enabled. Please merge the
accumulated overrides consistently into the target schema and add a regression
test for successive copies.
##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -0,0 +1,675 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/file_store_table.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/sensitive_config_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/snapshot_commit.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/disk/io_manager.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
+#include "paimon/core/operation/expire_snapshots.h"
+#include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/append_only_file_store_table.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/primary_key_file_store_table.h"
+#include "paimon/core/table/source/append_only_table_read.h"
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+#include "paimon/core/table/source/data_table_batch_scan.h"
+#include "paimon/core/table/source/data_table_stream_scan.h"
+#include "paimon/core/table/source/fallback_table_read.h"
+#include "paimon/core/table/source/key_value_table_read.h"
+#include "paimon/core/table/source/primary_key_index_batch_scan.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
+#include "paimon/core/table/source/realtime_table_scan.h"
+#include "paimon/core/table/source/snapshot/snapshot_reader.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/context_utils.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/read_context.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/result.h"
+#include "paimon/scan_context.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/write_context.h"
+
+namespace arrow {
+class Schema;
+} // namespace arrow
+
+namespace paimon {
+
+namespace {
+
+/// The escape hatch that lets an integration test commit on an object store,
read where
+/// `NewCommit` refuses one below.
+constexpr const char kObjectStoreCommitInteTestOption[] =
"enable-object-store-commit-in-inte-test";
+
+} // namespace
+
+class Executor;
+class MemoryPool;
+
+FileStoreTable::FileStoreTable(const std::shared_ptr<FileStore>& store,
+ const std::shared_ptr<TableSchema>& schema,
+ const Identifier& identifier, const
std::string& uuid)
+ : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier,
uuid),
+ store_(store) {}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create(
+ const std::shared_ptr<FileSystem>& specified_file_system, const
std::string& root_path,
+ const std::shared_ptr<TableSchema>& schema, const Identifier& identifier,
+ const std::string& uuid, const std::string& branch,
+ const std::map<std::string, std::string>& fs_scheme_map,
+ const std::map<std::string, std::string>& dynamic_options) {
+ if (schema == nullptr) {
+ return Status::Invalid(
+ fmt::format("not found latest schema of {}",
identifier.GetFullName()));
+ }
+ std::map<std::string, std::string> options = schema->Options();
+ const std::string normalized_branch =
BranchManager::NormalizeBranch(branch);
+ if (BranchManager::IsMainBranch(normalized_branch)) {
+ options.erase(Options::BRANCH);
+ } else {
+ options[Options::BRANCH] = normalized_branch;
+ }
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+ CoreOptions::FromMap(options,
specified_file_system, fs_scheme_map));
+ std::shared_ptr<FileStore> store = FileStore::Create(root_path,
std::move(core_options));
+ std::shared_ptr<FileStoreTable> table;
+ if (!schema->PrimaryKeys().empty()) {
+ table = std::make_shared<PrimaryKeyFileStoreTable>(store, schema,
identifier, uuid);
+ } else {
+ table = std::make_shared<AppendOnlyFileStoreTable>(store, schema,
identifier, uuid);
+ }
+ table->dynamic_options_ = dynamic_options;
+ table->specified_file_system_ = specified_file_system;
+ table->fs_scheme_map_ = fs_scheme_map;
+ return table;
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::Copy(
+ const std::map<std::string, std::string>& dynamic_options) const {
+ std::shared_ptr<TableSchema> loaded_schema =
std::static_pointer_cast<TableSchema>(schema_);
+ // `type` is immutable the way Java Paimon holds it: the kind of a copy is
named by the primary
+ // keys of the schema it serves, never by this option. A value naming
another type is refused,
+ // and one naming the same type - whatever its case, an absent `type`
reading as `table` - is
+ // dropped, as Java's unchanged-key check drops it.
+ auto type_option = dynamic_options.find(Options::TYPE);
+ if (type_option != dynamic_options.end()) {
+ PAIMON_ASSIGN_OR_RAISE(TableType schema_type,
+
TableTypeDefine::FromOptions(loaded_schema->Options()));
+ std::map<std::string, std::string> named{{Options::TYPE,
type_option->second}};
+ PAIMON_ASSIGN_OR_RAISE(TableType named_type,
TableTypeDefine::FromOptions(named));
+ if (named_type != schema_type) {
+ return Status::Invalid(fmt::format("Change '{}' is not supported
yet.", Options::TYPE));
+ }
+ }
+ // `branch` names the branch the copy serves, the way it routes a load: a
copy on the branch
+ // this table serves keeps this table's schema, while one on another
branch reads that branch's
+ // own latest schema and refuses a branch that holds none, as
`SwitchToBranch` does.
+ std::string target_branch = CurrentBranch();
+ auto branch_option = dynamic_options.find(Options::BRANCH);
+ if (branch_option != dynamic_options.end()) {
+ target_branch = BranchManager::NormalizeBranch(branch_option->second);
+ }
+ std::map<std::string, std::string> overridable = dynamic_options;
+ overridable.erase(Options::TYPE);
+ overridable.erase(Options::BRANCH);
+ std::shared_ptr<TableSchema> base_schema = loaded_schema;
+ // An `Identifier` is immutable once built, so the branch one is held
beside this table's
+ // and the copy carries whichever names the branch it serves.
+ std::optional<Identifier> branch_identifier;
+ if (target_branch != CurrentBranch()) {
+ // The branch is read through the file system this table serves, the
same read
+ // `SwitchToBranch` makes, and then served through the one its own
schema resolves.
+ SchemaManager schema_manager(file_system_, root_path_, target_branch);
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>>
latest,
+ schema_manager.Latest());
+ if (!latest) {
+ return Status::Invalid(fmt::format("branch {} does not exist",
target_branch));
+ }
+ base_schema = *latest;
+ PAIMON_ASSIGN_OR_RAISE(Identifier branch_named,
+ CatalogUtils::BranchIdentifier(identifier_,
target_branch));
+ branch_identifier.emplace(std::move(branch_named));
+ }
+ // The schema options name the branch the copy serves, the way
`SwitchToBranch` styles them:
+ // another branch names itself in them, the main branch carries no
`branch` option. The key is
+ // folded into the schema alone, not into the dynamic options the copy
accumulates.
+ std::map<std::string, std::string> schema_overridable = overridable;
+ if (target_branch != CurrentBranch() && target_branch !=
BranchManager::DEFAULT_MAIN_BRANCH) {
+ schema_overridable[Options::BRANCH] = target_branch;
+ }
+ PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema,
overridable));
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> merged_schema,
+ base_schema->MergeOptions(schema_overridable));
+ std::map<std::string, std::string> accumulated = dynamic_options_;
+ accumulated.erase(Options::TYPE);
+ accumulated.erase(Options::BRANCH);
+ for (const auto& [key, value] : overridable) {
+ accumulated.insert_or_assign(key, value);
+ }
+ // A copy resolves its file system from its own merged options, so it
takes over no manager of
+ // the table it was copied from: the store builds both over the resolution
the copy names.
+ return FileStoreTable::Create(
+ specified_file_system_, root_path_,
std::shared_ptr<TableSchema>(std::move(merged_schema)),
+ branch_identifier ? *branch_identifier : identifier_,
CatalogUuid().value_or(""),
+ target_branch, fs_scheme_map_, accumulated);
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::SwitchToBranch(const
std::string& branch) const {
+ const std::string target_branch = BranchManager::NormalizeBranch(branch);
+ if (target_branch == CurrentBranch()) {
+ return FileStoreTable::Create(specified_file_system_, root_path_,
+
std::static_pointer_cast<TableSchema>(schema_), identifier_,
+ CatalogUuid().value_or(""),
CurrentBranch(), fs_scheme_map_,
+ dynamic_options_);
+ }
+ // The branch is read through the file system this table serves, and then
served through the
+ // one its own schema resolves: the store builds the managers over that
resolution, so none of
+ // this table's is carried over.
+ SchemaManager schema_manager(file_system_, root_path_, target_branch);
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest,
+ schema_manager.Latest());
+ if (!latest) {
+ return Status::Invalid(fmt::format("branch {} does not exist",
target_branch));
+ }
+ std::map<std::string, std::string> overridable;
+ if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) {
+ overridable[Options::BRANCH] = target_branch;
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> branch_schema,
+ (*latest)->MergeOptions(overridable));
+ PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier,
+ CatalogUtils::BranchIdentifier(identifier_,
target_branch));
+ return FileStoreTable::Create(specified_file_system_, root_path_,
+
std::shared_ptr<TableSchema>(std::move(branch_schema)),
+ branch_identifier,
CatalogUuid().value_or(""), target_branch,
+ fs_scheme_map_, /*dynamic_options=*/{});
+}
+
+Status FileStoreTable::RestoreRealtimeCommittedProgress(
+ const std::shared_ptr<RealtimeContext>& realtime_context,
+ const std::shared_ptr<SnapshotManager>& snapshot_manager, const
CoreOptions& options) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl>
realtime_context_impl,
+ RealtimeContextImpl::Cast(realtime_context));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+ PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+ snapshot_manager->LatestSnapshot());
+ if (latest_snapshot) {
+ PAIMON_ASSIGN_OR_RAISE(
+ RealtimeOffsetMap realtime_committed_offsets,
+ RealtimeCommitProperties::ReadOffsets(latest_snapshot,
options.GetFileSystem()));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+ latest_snapshot->Id(), realtime_committed_offsets,
/*committed_data_files=*/{}));
+ }
+ return Status::OK();
+}
+
+Status FileStoreTable::CheckOptionsAgreeWithTable(
+ const TableSchema& table_schema,
+ const std::map<std::string, std::string>& context_options) const {
+ // The configuration the operators serve: the schema's own options as the
load styled them,
+ // with the dynamic options of the load on top. `NewXxx` reads it through
`store_->GetOptions()`
+ // and adds nothing from a context, so a context option that is not among
these keys is one the
+ // operators cannot apply.
+ std::map<std::string, std::string> table_options = table_schema.Options();
+ for (const auto& [key, value] : dynamic_options_) {
+ table_options.insert_or_assign(key, value);
+ }
+ for (const auto& [key, value] : context_options) {
+ auto table_option = table_options.find(key);
+ if (table_option != table_options.end()) {
+ if (table_option->second != value) {
+ return Status::Invalid(
+ fmt::format("option '{}' is '{}' in the context but '{}'
on the table", key,
+ SensitiveConfigUtils::RedactValue(key, value),
+ SensitiveConfigUtils::RedactValue(key,
table_option->second)));
+ }
+ continue;
+ }
+ // `branch` and `type` route a load to the table it names and are
folded into it there - by
+ // the load or by `Copy` - so the spelling a context gives them is
checked against the
+ // identifier and the schema, not against these keys.
+ if (key == Options::BRANCH || key == Options::TYPE) {
+ continue;
+ }
+ return Status::Invalid(fmt::format(
+ "option '{}' is '{}' in the context but the table was not loaded
with it: the "
+ "operator serves the table's own configuration, so the context
value would be "
+ "dropped in silence; load the table with this option or apply it
through "
+ "Table::Copy()",
+ key, SensitiveConfigUtils::RedactValue(key, value)));
+ }
+ return Status::OK();
+}
+
+Status FileStoreTable::CheckContextNamesThisTable(
+ const std::string& kind, const std::string& context_root_path,
+ const std::optional<Identifier>& context_identifier,
+ const std::optional<std::string>& context_table_id) const {
+ if (!root_path_.empty()) {
+ PAIMON_ASSIGN_OR_RAISE(std::string context_path,
+ PathUtil::NormalizePath(context_root_path));
+ PAIMON_ASSIGN_OR_RAISE(std::string table_path,
PathUtil::NormalizePath(root_path_));
+ if (context_path != table_path) {
+ return Status::Invalid(fmt::format(
+ "the {} context names root path '{}' but this table serves
'{}': the store below "
+ "reads and writes this table's files, so a context naming
another path would "
+ "leave this table's metadata there and publish it as the
other's",
+ kind, context_root_path, root_path_));
+ }
+ }
+ if (context_identifier) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> context_branch,
+ context_identifier.value().GetBranchName());
+ if (context_branch && context_branch.value() != CurrentBranch()) {
+ return Status::Invalid(fmt::format(
+ "the {} context names branch '{}' of '{}' but this table
serves branch '{}', "
+ "which is the one a {} of this table writes and publishes",
+ kind, context_branch.value(),
context_identifier.value().ToString(),
+ CurrentBranch(), kind));
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ Identifier context_table,
+ CatalogUtils::BranchIdentifier(context_identifier.value(),
CurrentBranch()));
+ PAIMON_ASSIGN_OR_RAISE(Identifier served_table,
+ CatalogUtils::BranchIdentifier(identifier_,
CurrentBranch()));
+ const bool names_this_table =
+ (served_table.GetDatabaseName() == Identifier::kUnknownDatabase ||
+ context_table.GetDatabaseName() ==
served_table.GetDatabaseName()) &&
+ context_table.GetObjectName() == served_table.GetObjectName();
+ if (!names_this_table) {
+ return Status::Invalid(fmt::format(
+ "the {} context names table '{}' but this table is '{}': the
store below serves "
+ "this table's files, so a context naming another table would
write this table's "
+ "metadata and publish it as the other's",
+ kind, context_identifier.value().ToString(),
identifier_.ToString()));
+ }
+ }
+ if (context_table_id && context_table_id != CatalogUuid()) {
+ return Status::Invalid(fmt::format(
+ "the {} context carries table id '{}' but this table's is '{}':
the id tells a "
+ "recreated table from the one the changes were prepared for, so
one this table does "
+ "not carry would publish this table's metadata as the
recreation's",
+ kind, context_table_id.value(), CatalogUuid().value_or("none")));
+ }
+ return Status::OK();
+}
+
+std::unique_ptr<FileStoreScan>
FileStoreTable::WithTablePath(std::unique_ptr<FileStoreScan>&& scan,
+ const
std::string& root_path) {
+ scan->WithTablePath(root_path);
+ return std::move(scan);
+}
+
+Result<std::unique_ptr<FileStoreWrite>> FileStoreTable::NewWrite(
+ std::unique_ptr<WriteContext> ctx) const {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write"));
+ PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write",
ctx->GetRootPath(),
+ ctx->GetIdentifier(),
+
/*context_table_id=*/std::nullopt));
Review Comment:
**[P1] Reject explicit context branches that differ from the table.** A
main-branch table accepts a `WriteContext` built with `WithBranch("dev")`
because this validation checks only the path and identifier. `NewCommit()`
similarly accepts `AddOption(Options::BRANCH, "dev")` when the schema has no
branch option. Both operators then use the store's main-branch managers, so a
write intended for `dev` can be published to `main` without an error. Please
validate explicitly supplied branch settings against `CurrentBranch()` before
constructing either operator, and cover this case through the table-based write
and commit entry points.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]