SteNicholas commented on code in PR #447:
URL: https://github.com/apache/paimon-cpp/pull/447#discussion_r4410730817


##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -0,0 +1,675 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/file_store_table.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/sensitive_config_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/snapshot_commit.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/disk/io_manager.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
+#include "paimon/core/operation/expire_snapshots.h"
+#include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/append_only_file_store_table.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/primary_key_file_store_table.h"
+#include "paimon/core/table/source/append_only_table_read.h"
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+#include "paimon/core/table/source/data_table_batch_scan.h"
+#include "paimon/core/table/source/data_table_stream_scan.h"
+#include "paimon/core/table/source/fallback_table_read.h"
+#include "paimon/core/table/source/key_value_table_read.h"
+#include "paimon/core/table/source/primary_key_index_batch_scan.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
+#include "paimon/core/table/source/realtime_table_scan.h"
+#include "paimon/core/table/source/snapshot/snapshot_reader.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/context_utils.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/read_context.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/result.h"
+#include "paimon/scan_context.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/write_context.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+namespace {
+
+/// The escape hatch that lets an integration test commit on an object store, 
read where
+/// `NewCommit` refuses one below.
+constexpr const char kObjectStoreCommitInteTestOption[] = 
"enable-object-store-commit-in-inte-test";
+
+}  // namespace
+
+class Executor;
+class MemoryPool;
+
+FileStoreTable::FileStoreTable(const std::shared_ptr<FileStore>& store,
+                               const std::shared_ptr<TableSchema>& schema,
+                               const Identifier& identifier, const 
std::string& uuid)
+    : Table(store->GetFileSystem(), store->GetRootPath(), schema, identifier, 
uuid),
+      store_(store) {}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create(
+    const std::shared_ptr<FileSystem>& specified_file_system, const 
std::string& root_path,
+    const std::shared_ptr<TableSchema>& schema, const Identifier& identifier,
+    const std::string& uuid, const std::string& branch,
+    const std::map<std::string, std::string>& fs_scheme_map,
+    const std::map<std::string, std::string>& dynamic_options) {
+    if (schema == nullptr) {
+        return Status::Invalid(
+            fmt::format("not found latest schema of {}", 
identifier.GetFullName()));
+    }
+    std::map<std::string, std::string> options = schema->Options();
+    const std::string normalized_branch = 
BranchManager::NormalizeBranch(branch);
+    if (BranchManager::IsMainBranch(normalized_branch)) {
+        options.erase(Options::BRANCH);
+    } else {
+        options[Options::BRANCH] = normalized_branch;
+    }
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
+                           CoreOptions::FromMap(options, 
specified_file_system, fs_scheme_map));
+    std::shared_ptr<FileStore> store = FileStore::Create(root_path, 
std::move(core_options));
+    std::shared_ptr<FileStoreTable> table;
+    if (!schema->PrimaryKeys().empty()) {
+        table = std::make_shared<PrimaryKeyFileStoreTable>(store, schema, 
identifier, uuid);
+    } else {
+        table = std::make_shared<AppendOnlyFileStoreTable>(store, schema, 
identifier, uuid);
+    }
+    table->dynamic_options_ = dynamic_options;
+    table->specified_file_system_ = specified_file_system;
+    table->fs_scheme_map_ = fs_scheme_map;
+    return table;
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::Copy(
+    const std::map<std::string, std::string>& dynamic_options) const {
+    std::shared_ptr<TableSchema> loaded_schema = 
std::static_pointer_cast<TableSchema>(schema_);
+    // `type` is immutable the way Java Paimon holds it: the kind of a copy is 
named by the primary
+    // keys of the schema it serves, never by this option. A value naming 
another type is refused,
+    // and one naming the same type - whatever its case, an absent `type` 
reading as `table` - is
+    // dropped, as Java's unchanged-key check drops it.
+    auto type_option = dynamic_options.find(Options::TYPE);
+    if (type_option != dynamic_options.end()) {
+        PAIMON_ASSIGN_OR_RAISE(TableType schema_type,
+                               
TableTypeDefine::FromOptions(loaded_schema->Options()));
+        std::map<std::string, std::string> named{{Options::TYPE, 
type_option->second}};
+        PAIMON_ASSIGN_OR_RAISE(TableType named_type, 
TableTypeDefine::FromOptions(named));
+        if (named_type != schema_type) {
+            return Status::Invalid(fmt::format("Change '{}' is not supported 
yet.", Options::TYPE));
+        }
+    }
+    // `branch` names the branch the copy serves, the way it routes a load: a 
copy on the branch
+    // this table serves keeps this table's schema, while one on another 
branch reads that branch's
+    // own latest schema and refuses a branch that holds none, as 
`SwitchToBranch` does.
+    std::string target_branch = CurrentBranch();
+    auto branch_option = dynamic_options.find(Options::BRANCH);
+    if (branch_option != dynamic_options.end()) {
+        target_branch = BranchManager::NormalizeBranch(branch_option->second);
+    }
+    std::map<std::string, std::string> overridable = dynamic_options;
+    overridable.erase(Options::TYPE);
+    overridable.erase(Options::BRANCH);
+    std::shared_ptr<TableSchema> base_schema = loaded_schema;
+    // An `Identifier` is immutable once built, so the branch one is held 
beside this table's
+    // and the copy carries whichever names the branch it serves.
+    std::optional<Identifier> branch_identifier;
+    if (target_branch != CurrentBranch()) {
+        // The branch is read through the file system this table serves, the 
same read
+        // `SwitchToBranch` makes, and then served through the one its own 
schema resolves.
+        SchemaManager schema_manager(file_system_, root_path_, target_branch);
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> 
latest,
+                               schema_manager.Latest());
+        if (!latest) {
+            return Status::Invalid(fmt::format("branch {} does not exist", 
target_branch));
+        }
+        base_schema = *latest;
+        PAIMON_ASSIGN_OR_RAISE(Identifier branch_named,
+                               CatalogUtils::BranchIdentifier(identifier_, 
target_branch));
+        branch_identifier.emplace(std::move(branch_named));
+    }
+    // The schema options name the branch the copy serves, the way 
`SwitchToBranch` styles them:
+    // another branch names itself in them, the main branch carries no 
`branch` option. The key is
+    // folded into the schema alone, not into the dynamic options the copy 
accumulates.
+    std::map<std::string, std::string> schema_overridable = overridable;
+    if (target_branch != CurrentBranch() && target_branch != 
BranchManager::DEFAULT_MAIN_BRANCH) {
+        schema_overridable[Options::BRANCH] = target_branch;
+    }
+    PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*base_schema, 
overridable));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> merged_schema,
+                           base_schema->MergeOptions(schema_overridable));
+    std::map<std::string, std::string> accumulated = dynamic_options_;
+    accumulated.erase(Options::TYPE);
+    accumulated.erase(Options::BRANCH);
+    for (const auto& [key, value] : overridable) {
+        accumulated.insert_or_assign(key, value);
+    }
+    // A copy resolves its file system from its own merged options, so it 
takes over no manager of
+    // the table it was copied from: the store builds both over the resolution 
the copy names.
+    return FileStoreTable::Create(
+        specified_file_system_, root_path_, 
std::shared_ptr<TableSchema>(std::move(merged_schema)),
+        branch_identifier ? *branch_identifier : identifier_, 
CatalogUuid().value_or(""),
+        target_branch, fs_scheme_map_, accumulated);
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::SwitchToBranch(const 
std::string& branch) const {
+    const std::string target_branch = BranchManager::NormalizeBranch(branch);
+    if (target_branch == CurrentBranch()) {
+        return FileStoreTable::Create(specified_file_system_, root_path_,
+                                      
std::static_pointer_cast<TableSchema>(schema_), identifier_,
+                                      CatalogUuid().value_or(""), 
CurrentBranch(), fs_scheme_map_,
+                                      dynamic_options_);
+    }
+    // The branch is read through the file system this table serves, and then 
served through the
+    // one its own schema resolves: the store builds the managers over that 
resolution, so none of
+    // this table's is carried over.
+    SchemaManager schema_manager(file_system_, root_path_, target_branch);
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest,
+                           schema_manager.Latest());
+    if (!latest) {
+        return Status::Invalid(fmt::format("branch {} does not exist", 
target_branch));
+    }
+    std::map<std::string, std::string> overridable;
+    if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) {
+        overridable[Options::BRANCH] = target_branch;
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> branch_schema,
+                           (*latest)->MergeOptions(overridable));
+    PAIMON_ASSIGN_OR_RAISE(Identifier branch_identifier,
+                           CatalogUtils::BranchIdentifier(identifier_, 
target_branch));
+    return FileStoreTable::Create(specified_file_system_, root_path_,
+                                  
std::shared_ptr<TableSchema>(std::move(branch_schema)),
+                                  branch_identifier, 
CatalogUuid().value_or(""), target_branch,
+                                  fs_scheme_map_, /*dynamic_options=*/{});
+}
+
+Status FileStoreTable::RestoreRealtimeCommittedProgress(
+    const std::shared_ptr<RealtimeContext>& realtime_context,
+    const std::shared_ptr<SnapshotManager>& snapshot_manager, const 
CoreOptions& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> 
realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context));
+    PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                           snapshot_manager->LatestSnapshot());
+    if (latest_snapshot) {
+        PAIMON_ASSIGN_OR_RAISE(
+            RealtimeOffsetMap realtime_committed_offsets,
+            RealtimeCommitProperties::ReadOffsets(latest_snapshot, 
options.GetFileSystem()));
+        PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+            latest_snapshot->Id(), realtime_committed_offsets, 
/*committed_data_files=*/{}));
+    }
+    return Status::OK();
+}
+
+Status FileStoreTable::CheckOptionsAgreeWithTable(
+    const TableSchema& table_schema,
+    const std::map<std::string, std::string>& context_options) const {
+    // The configuration the operators serve: the schema's own options as the 
load styled them,
+    // with the dynamic options of the load on top. `NewXxx` reads it through 
`store_->GetOptions()`
+    // and adds nothing from a context, so a context option that is not among 
these keys is one the
+    // operators cannot apply.
+    std::map<std::string, std::string> table_options = table_schema.Options();
+    for (const auto& [key, value] : dynamic_options_) {
+        table_options.insert_or_assign(key, value);
+    }
+    for (const auto& [key, value] : context_options) {
+        auto table_option = table_options.find(key);
+        if (table_option != table_options.end()) {
+            if (table_option->second != value) {
+                return Status::Invalid(
+                    fmt::format("option '{}' is '{}' in the context but '{}' 
on the table", key,
+                                SensitiveConfigUtils::RedactValue(key, value),
+                                SensitiveConfigUtils::RedactValue(key, 
table_option->second)));
+            }
+            continue;
+        }
+        // `branch` and `type` route a load to the table it names and are 
folded into it there - by
+        // the load or by `Copy` - so the spelling a context gives them is 
checked against the
+        // identifier and the schema, not against these keys.
+        if (key == Options::BRANCH || key == Options::TYPE) {
+            continue;
+        }
+        return Status::Invalid(fmt::format(
+            "option '{}' is '{}' in the context but the table was not loaded 
with it: the "
+            "operator serves the table's own configuration, so the context 
value would be "
+            "dropped in silence; load the table with this option or apply it 
through "
+            "Table::Copy()",
+            key, SensitiveConfigUtils::RedactValue(key, value)));
+    }
+    return Status::OK();
+}
+
+Status FileStoreTable::CheckContextNamesThisTable(
+    const std::string& kind, const std::string& context_root_path,
+    const std::optional<Identifier>& context_identifier,
+    const std::optional<std::string>& context_table_id) const {
+    if (!root_path_.empty()) {
+        PAIMON_ASSIGN_OR_RAISE(std::string context_path,
+                               PathUtil::NormalizePath(context_root_path));
+        PAIMON_ASSIGN_OR_RAISE(std::string table_path, 
PathUtil::NormalizePath(root_path_));
+        if (context_path != table_path) {
+            return Status::Invalid(fmt::format(
+                "the {} context names root path '{}' but this table serves 
'{}': the store below "
+                "reads and writes this table's files, so a context naming 
another path would "
+                "leave this table's metadata there and publish it as the 
other's",
+                kind, context_root_path, root_path_));
+        }
+    }
+    if (context_identifier) {
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> context_branch,
+                               context_identifier.value().GetBranchName());
+        if (context_branch && context_branch.value() != CurrentBranch()) {
+            return Status::Invalid(fmt::format(
+                "the {} context names branch '{}' of '{}' but this table 
serves branch '{}', "
+                "which is the one a {} of this table writes and publishes",
+                kind, context_branch.value(), 
context_identifier.value().ToString(),
+                CurrentBranch(), kind));
+        }
+        PAIMON_ASSIGN_OR_RAISE(
+            Identifier context_table,
+            CatalogUtils::BranchIdentifier(context_identifier.value(), 
CurrentBranch()));
+        PAIMON_ASSIGN_OR_RAISE(Identifier served_table,
+                               CatalogUtils::BranchIdentifier(identifier_, 
CurrentBranch()));
+        const bool names_this_table =
+            (served_table.GetDatabaseName() == Identifier::kUnknownDatabase ||
+             context_table.GetDatabaseName() == 
served_table.GetDatabaseName()) &&
+            context_table.GetObjectName() == served_table.GetObjectName();
+        if (!names_this_table) {
+            return Status::Invalid(fmt::format(
+                "the {} context names table '{}' but this table is '{}': the 
store below serves "
+                "this table's files, so a context naming another table would 
write this table's "
+                "metadata and publish it as the other's",
+                kind, context_identifier.value().ToString(), 
identifier_.ToString()));
+        }
+    }
+    if (context_table_id && context_table_id != CatalogUuid()) {
+        return Status::Invalid(fmt::format(
+            "the {} context carries table id '{}' but this table's is '{}': 
the id tells a "
+            "recreated table from the one the changes were prepared for, so 
one this table does "
+            "not carry would publish this table's metadata as the 
recreation's",
+            kind, context_table_id.value(), CatalogUuid().value_or("none")));
+    }
+    return Status::OK();
+}
+
+std::unique_ptr<FileStoreScan> 
FileStoreTable::WithTablePath(std::unique_ptr<FileStoreScan>&& scan,
+                                                             const 
std::string& root_path) {
+    scan->WithTablePath(root_path);
+    return std::move(scan);
+}
+
+Result<std::unique_ptr<FileStoreWrite>> FileStoreTable::NewWrite(
+    std::unique_ptr<WriteContext> ctx) const {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write"));
+    PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write", 
ctx->GetRootPath(),
+                                                    ctx->GetIdentifier(),
+                                                    
/*context_table_id=*/std::nullopt));

Review Comment:
   **Reject explicit context branches that differ from the table.** A 
main-branch table accepts a `WriteContext` built with `WithBranch("dev")` 
because this validation checks only the path and identifier. `NewCommit()` 
similarly accepts `AddOption(Options::BRANCH, "dev")` when the schema has no 
branch option. Both operators then use the store's main-branch managers, so a 
write intended for `dev` can be published to `main` without an error. Please 
validate explicitly supplied branch settings against `CurrentBranch()` before 
constructing either operator, and cover this case through the table-based write 
and commit entry points.



##########
src/paimon/core/table/system/system_table.cpp:
##########
@@ -176,17 +186,68 @@ std::optional<SystemTableFactory> 
FindSystemTableFactory(const std::string& syst
 
 }  // namespace
 
+SystemTable::SystemTable(const std::shared_ptr<FileSystem>& fs, const 
std::string& table_path,
+                         std::string name)
+    : Table(fs, table_path, /*schema=*/nullptr,
+            Identifier(Identifier::kUnknownDatabase, "system_table"), 
/*uuid=*/""),
+      name_(std::move(name)) {}
+
+std::string SystemTable::ComposeName(const std::string& data_table_name, const 
std::string& kind) {
+    return data_table_name + Catalog::SYSTEM_TABLE_SPLITTER + kind;
+}
+
+Status SystemTable::CheckNoSearch(const std::shared_ptr<ReadContext>& context) 
{
+    if (context->HasVectorOrFullTextSearch()) {
+        return Status::NotImplemented(
+            "VectorSearch or FullTextSearch does not support system-table 
reads yet");
+    }
+    if (PredicateUtils::ContainsFullTextMatch(context->GetPredicate())) {
+        return Status::NotImplemented("FullTextMatch predicate execution is 
not implemented yet");
+    }
+    return Status::OK();
+}
+
+Status SystemTable::AdoptOwnSchema() {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> arrow_schema, 
ArrowSchema());
+    schema_ = std::make_shared<SystemTableSchema>(std::move(arrow_schema));
+    return Status::OK();
+}
+
+Result<std::shared_ptr<Table>> SystemTable::Copy(
+    const std::map<std::string, std::string>& /*dynamic_options*/) const {
+    return Status::NotImplemented(Name(), " system table is read-only: copying 
is not supported");

Review Comment:
   **Preserve dynamic scan options for system-table views.** Path-based 
factories now reject system tables, but this unconditional rejection also 
prevents configuring their replacement catalog-loaded handles. For example, 
`NewScan()` on `tbl$ro` ignores context options `scan.mode=from-snapshot` and 
`scan.snapshot-id`, using only the options captured when the table was loaded; 
`Copy()` cannot apply them either. A historical scan consequently reads the 
latest snapshot. Please preserve a supported way to apply dynamic options to 
`read_optimized` and `audit_log` views, and test the selected snapshot through 
`Catalog::GetTable()`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to