zjw1111 commented on code in PR #447:
URL: https://github.com/apache/paimon-cpp/pull/447#discussion_r4384907908


##########
src/paimon/core/table/primary_key_file_store_table.cpp:
##########
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/primary_key_file_store_table.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "fmt/format.h"
+#include "paimon/common/global_index/btree/btree_defs.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/fields_comparator.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/manifest/index_manifest_entry.h"
+#include "paimon/core/mergetree/compact/lookup_merge_function.h"
+#include "paimon/core/mergetree/compact/merge_function.h"
+#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h"
+#include "paimon/core/operation/commit/commit_scanner.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/key_value_file_store_scan.h"
+#include "paimon/core/operation/key_value_file_store_write.h"
+#include "paimon/core/options/merge_engine.h"
+#include "paimon/core/postpone/postpone_bucket_file_store_write.h"
+#include "paimon/core/realtime/realtime_schema_layout.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/bucket_mode.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/source/merge_tree_split_generator.h"
+#include "paimon/core/table/source/split_generator.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+#include "paimon/write_context.h"
+
+namespace paimon {
+
+struct KeyValue;
+template <typename T>
+class MergeFunctionWrapper;
+
+namespace {
+
+/// Whether a schema older than `current_schema_id` carried a primary key 
b-tree index, which a
+/// write has to keep maintaining: the index files an earlier schema wrote are 
still there to be
+/// read, and dropping the definition now would leave them orphaned.
+Result<bool> HasHistoricalPrimaryKeyBTreeDefinition(
+    const std::shared_ptr<SchemaManager>& schema_manager, int64_t 
current_schema_id) {
+    if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) {
+        return false;
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::vector<int64_t> schema_ids, 
schema_manager->ListAllIds());
+    for (int64_t schema_id : schema_ids) {
+        if (schema_id >= current_schema_id) {
+            continue;
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<TableSchema> historical_schema,
+                               schema_manager->ReadSchema(schema_id));
+        const auto& historical_options = historical_schema->Options();
+        auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS);
+        if (option != historical_options.end() &&
+            !StringUtils::IsNullOrWhitespaceOnly(option->second)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+}  // namespace
+
+PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const 
std::shared_ptr<FileStore>& store,
+                                                   const Identifier& 
identifier,
+                                                   const std::string& uuid)
+    : FileStoreTable(store, identifier, uuid) {}
+
+Result<std::unique_ptr<FileStoreWrite>> PrimaryKeyFileStoreTable::CreateWrite(

Review Comment:
   this function is more than 100 lines, do you need split to some small 
functions?



##########
include/paimon/catalog/table.h:
##########
@@ -33,50 +33,45 @@
 
 namespace paimon {
 
-class TableScan;
+class CommitContext;
+class FileStoreCommit;
+class FileStoreWrite;
+class ReadContext;
+class ScanContext;
 class TableRead;
+class TableScan;
+class WriteContext;
 
 /// A table provides basic abstraction for table type.
+///
+/// The state every kind of table carries - file system, root path, schema and 
identifier - lives
+/// here, and each kind fills the operations in. The `NewXxx` operators below 
are the table-first
+/// entry points: `TableScan::Create()` and friends load the table a context 
names and hand the
+/// context to them. What a context asks for that the table cannot honour is 
refused by name rather
+/// than silently dropped.
 class PAIMON_EXPORT Table {
  public:
-    /// @param file_system File system the table's data is accessed through.
-    /// @param root_path Root path of the table in the file system.
-    /// @param uuid Metastore UUID, or empty if unavailable.
-    Table(const std::shared_ptr<FileSystem>& file_system, const std::string& 
root_path,
-          const std::shared_ptr<Schema>& schema, const Identifier& identifier,
-          const std::string& uuid = "");
-
-    ~Table() = default;
+    virtual ~Table() = default;

Review Comment:
   1. modify examples/read_write_demo.cpp with the new table api
   2. The API interfaces have changed significantly. Please check whether the 
code examples in the comments and documentation are consistent with the current 
API.



##########
src/paimon/core/table/file_store.cpp:
##########
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/file_store.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "paimon/catalog/catalog.h"
+#include "paimon/common/utils/arrow/status_utils.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/manifest/manifest_file.h"
+#include "paimon/core/manifest/manifest_list.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/index_file_path_factories.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/defs.h"
+#include "paimon/result.h"
+#include "paimon/status.h"
+
+namespace paimon {
+
+namespace {
+
+/// Whether an option key takes part in file-system resolution: the backend 
`file-system` picks,
+/// an `fs.` access key carries, or a native S3 key names - `s3.` and its 
`s3a.` spelling, the
+/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The 
scheme-map keys a
+/// context carries are the same set.
+bool IsFileSystemResolutionKey(const std::string& key) {
+    return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || 
key.rfind("s3.", 0) == 0 ||
+           key.rfind("s3a.", 0) == 0;
+}

Review Comment:
   Is there any better solution here? There are more `fs` prefixes than just 
these; various file-system plugins each have their own prefixes.



##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -0,0 +1,615 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/file_store_table.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/sensitive_config_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/snapshot_commit.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/disk/io_manager.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
+#include "paimon/core/operation/expire_snapshots.h"
+#include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/append_only_file_store_table.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/file_store_table_factory.h"
+#include "paimon/core/table/primary_key_file_store_table.h"
+#include "paimon/core/table/source/append_only_table_read.h"
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+#include "paimon/core/table/source/data_table_batch_scan.h"
+#include "paimon/core/table/source/data_table_stream_scan.h"
+#include "paimon/core/table/source/fallback_table_read.h"
+#include "paimon/core/table/source/key_value_table_read.h"
+#include "paimon/core/table/source/primary_key_index_batch_scan.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
+#include "paimon/core/table/source/realtime_table_scan.h"
+#include "paimon/core/table/source/snapshot/snapshot_reader.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/context_utils.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/read_context.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/result.h"
+#include "paimon/scan_context.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/write_context.h"
+
+namespace arrow {
+class Schema;
+}  // namespace arrow
+
+namespace paimon {
+
+class Executor;
+class MemoryPool;
+
+FileStoreTable::FileStoreTable(const std::shared_ptr<FileStore>& store,
+                               const Identifier& identifier, const 
std::string& uuid)
+    : Table(store->GetFileSystem(), store->GetRootPath(), store->GetSchema(), 
identifier, uuid),
+      store_(store) {}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create(
+    const std::shared_ptr<FileSystem>& file_system, const std::string& 
root_path,
+    const std::shared_ptr<Schema>& schema, const Identifier& identifier, const 
std::string& uuid,
+    const std::string& branch, const std::shared_ptr<SchemaManager>& 
schema_manager,
+    const std::shared_ptr<SnapshotManager>& snapshot_manager,
+    const std::map<std::string, std::string>& fs_scheme_map,
+    const std::map<std::string, std::string>& dynamic_options) {
+    // The kind the schema's primary keys name, which no schema change can add 
or drop, so the kind
+    // the table is built as is the kind every schema its root path ever held 
describes. A caller
+    // without a schema gets the append-only kind, whose operators report that 
none was found; a
+    // schema that is not a table schema - a system table's read-only one - is 
refused, not cast.
+    std::shared_ptr<TableSchema> table_schema;
+    if (schema != nullptr) {
+        table_schema = std::dynamic_pointer_cast<TableSchema>(schema);
+        if (table_schema == nullptr) {
+            return Status::Invalid(
+                fmt::format("{} is not a managed table schema", 
identifier.ToString()));
+        }
+    }
+    // The store parses the schema's options, so one that does not parse is 
refused here.
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStore> store,
+                           FileStore::Create(file_system, root_path, 
table_schema, branch,
+                                             schema_manager, snapshot_manager, 
fs_scheme_map));
+    std::shared_ptr<FileStoreTable> table;
+    if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) {
+        table = std::make_shared<PrimaryKeyFileStoreTable>(store, identifier, 
uuid);
+    } else {
+        table = std::make_shared<AppendOnlyFileStoreTable>(store, identifier, 
uuid);
+    }
+    table->dynamic_options_ = dynamic_options;
+    return table;
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::Copy(
+    const std::map<std::string, std::string>& dynamic_options) const {
+    std::shared_ptr<TableSchema> loaded_schema = 
std::static_pointer_cast<TableSchema>(schema_);
+    if (loaded_schema == nullptr) {
+        return Status::Invalid(fmt::format("not found latest schema of {}", 
FullName()));
+    }
+    // `type` is structural and read from the schema alone, the promise `Copy` 
makes: an option
+    // given at the call must not restyle the table. `branch` is the same 
promise - the copy serves
+    // the branch this table does, and a caller that means to move has 
`SwitchToBranch`. Both are
+    // dropped before the merge, so the copy's options never speak of a branch 
it does not serve.
+    std::map<std::string, std::string> overridable = dynamic_options;
+    overridable.erase(Options::TYPE);
+    overridable.erase(Options::BRANCH);
+    // The physical layout the data on the path was written by is as immovable 
as the style or the
+    // branch: the copy serves the same data.
+    
PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, 
overridable));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> merged_schema,
+                           loaded_schema->MergeOptions(overridable));
+    // The managers are handed over, not rebuilt, when the copy resolves the 
same file system this
+    // table does: its operators then read the schema cache this table already 
warmed. The copy
+    // resolves from the file system the caller opened this table through, not 
the one it resolved
+    // to, so a file-system option it was given speaks; when one does change 
the resolution, fresh
+    // managers are built over it instead. The scheme map travels the same way.
+    const bool fs_resolution_may_change =
+        FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable);
+    // The dynamic options the copy was given join the ones this table was 
loaded with - which a
+    // copy of a copy carries the same way - so a read's fallback reader 
serves what the copy's own
+    // reader does however many copies deep. The branch was dropped above, so 
none accumulates, and
+    // an option given again replaces the one carried, as the merge above does.
+    std::map<std::string, std::string> accumulated = dynamic_options_;
+    for (const auto& [key, value] : overridable) {
+        accumulated.insert_or_assign(key, value);
+    }
+    return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_,
+                                  
std::shared_ptr<TableSchema>(std::move(merged_schema)),
+                                  identifier_, CatalogUuid().value_or(""), 
CurrentBranch(),
+                                  fs_resolution_may_change ? nullptr : 
GetSchemaManager(),
+                                  fs_resolution_may_change ? nullptr : 
GetSnapshotManager(),
+                                  store_->GetFsSchemeMap(), accumulated);
+}
+
+Result<std::shared_ptr<Table>> FileStoreTable::SwitchToBranch(const 
std::string& branch) const {
+    const std::string target_branch = BranchManager::NormalizeBranch(branch);
+    if (target_branch == CurrentBranch()) {
+        // The table already serves this branch, so a copy of it is the same 
table to the caller.
+        return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), 
root_path_, schema_,
+                                      identifier_, CatalogUuid().value_or(""), 
CurrentBranch(),
+                                      GetSchemaManager(), GetSnapshotManager(),
+                                      store_->GetFsSchemeMap(), 
dynamic_options_);
+    }
+    // The schema of the branch switched to, read from its own directory.
+    auto schema_manager = std::make_shared<SchemaManager>(file_system_, 
root_path_, target_branch);
+    PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest,
+                           schema_manager->Latest());
+    if (!latest) {
+        return Status::Invalid(fmt::format("branch {} does not exist", 
target_branch));
+    }
+    // A branch other than main is named in the options the schema carries - 
the way a read context
+    // names it - so what reads through them serves the branch the managers 
point at.
+    std::map<std::string, std::string> overridable;
+    if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) {
+        overridable[Options::BRANCH] = target_branch;
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> branch_schema,
+                           (*latest)->MergeOptions(overridable));
+    // The branch's schema may name another file system than this table's 
options resolved - a
+    // recreation that changed the access configuration - and a manager kept 
through that
+    // disagreement would read the branch's metadata through the backend this 
table left, so the
+    // branch builds its own. The schema is in hand already, so only the cache 
is lost.
+    const std::shared_ptr<SchemaManager> branch_schema_manager =
+        (store_->GetSpecifiedFileSystem() == nullptr &&
+         (!store_->GetFsSchemeMap().empty() ||
+          !FileStore::FileSystemResolutionAgrees(
+              branch_schema->Options(), 
std::static_pointer_cast<TableSchema>(schema_)->Options())))
+            ? nullptr
+            : schema_manager;
+    return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_,
+                                  
std::shared_ptr<TableSchema>(std::move(branch_schema)),
+                                  identifier_, CatalogUuid().value_or(""), 
target_branch,
+                                  branch_schema_manager, 
/*snapshot_manager=*/nullptr,
+                                  store_->GetFsSchemeMap());
+}
+
+Status FileStoreTable::RestoreRealtimeCommittedProgress(
+    const std::shared_ptr<RealtimeContext>& realtime_context,
+    const std::shared_ptr<SnapshotManager>& snapshot_manager, const 
CoreOptions& options) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> 
realtime_context_impl,
+                           RealtimeContextImpl::Cast(realtime_context));
+    PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+                           snapshot_manager->LatestSnapshot());
+    if (latest_snapshot) {
+        PAIMON_ASSIGN_OR_RAISE(
+            RealtimeOffsetMap realtime_committed_offsets,
+            RealtimeCommitProperties::ReadOffsets(latest_snapshot, 
options.GetFileSystem()));
+        PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+            latest_snapshot->Id(), realtime_committed_offsets, 
/*committed_data_files=*/{}));
+    }
+    return Status::OK();
+}
+
+Status FileStoreTable::CheckOptionsAgreeWithTable(
+    const TableSchema& table_schema, const std::map<std::string, std::string>& 
context_options) {
+    const std::map<std::string, std::string>& table_options = 
table_schema.Options();
+    for (const auto& [key, value] : context_options) {
+        auto table_option = table_options.find(key);
+        if (table_option != table_options.end() && table_option->second != 
value) {
+            // A value may be a credential and the error travels into logs, so 
it is redacted.
+            return Status::Invalid(
+                fmt::format("option '{}' is '{}' in the context but '{}' on 
the table", key,
+                            SensitiveConfigUtils::RedactValue(key, value),
+                            SensitiveConfigUtils::RedactValue(key, 
table_option->second)));
+        }
+    }
+    return Status::OK();
+}
+
+std::unique_ptr<FileStoreScan> 
FileStoreTable::WithTablePath(std::unique_ptr<FileStoreScan>&& scan,
+                                                             const 
std::string& root_path) {
+    scan->WithTablePath(root_path);
+    return std::move(scan);
+}
+
+Result<std::unique_ptr<FileStoreWrite>> FileStoreTable::NewWrite(
+    std::unique_ptr<WriteContext> ctx) const {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write"));
+    PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog()));
+    std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem();
+    // A catalog write names its table; the file system comes from the table's 
own credentials.
+    if (specific_fs == nullptr && 
ctx->GetFileSystemSchemeToIdentifierMap().empty() &&
+        ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) {
+        return Status::Invalid("a catalog write requires a table identifier");
+    }
+    // The write goes to the branch the table serves, not the context's own.
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Identifier> catalog_identifier,
+                           
CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch()));
+    const std::shared_ptr<TableSchema>& schema = store_->GetSchema();
+    if (schema == nullptr) {
+        return Status::Invalid(
+            fmt::format("not found latest schema in branch {}", 
CurrentBranch()));
+    }
+    PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, 
ctx->GetOptions()));
+    CoreOptions options = store_->GetOptions();
+    auto arrow_schema = 
DataField::ConvertDataFieldsToArrowSchema(schema->Fields());
+    PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options));
+    PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, 
options));
+    if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) {
+        return Status::NotImplemented(
+            "Dedicated VECTOR storage does not yet support custom write 
schemas or partial writes");
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema,
+                           FieldMapping::GetPartitionSchema(arrow_schema, 
schema->PartitionKeys()));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<FileStorePathFactory> file_store_path_factory,
+        store_->NewPathFactory(arrow_schema, options,
+                               
options.GetWriteFileFormat(/*level=*/0)->Identifier(),
+                               ctx->GetMemoryPool()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<SnapshotManager> snapshot_manager,
+        store_->NewSnapshotManager(options, ctx->GetCatalog(), 
catalog_identifier));
+    std::shared_ptr<IOManager> io_manager;
+    const auto& io_temp_dir = ctx->GetTempDirectory();
+    if (!io_temp_dir.empty()) {
+        io_manager = std::make_shared<IOManager>(io_temp_dir, 
options.GetFileSystem());
+    }
+
+    bool ignore_previous_files = ctx->IgnorePreviousFiles();
+    if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) {
+        return Status::Invalid("real-time write requires 
realtime.enabled=true");
+    }
+    if (ctx->GetRealtimeContext()) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> 
realtime_context_impl,
+                               
RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
+        PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+    }
+    return CreateWrite(*ctx, arrow_schema, partition_schema, options, 
file_store_path_factory,
+                       snapshot_manager, io_manager, ignore_previous_files);
+}
+
+Result<std::unique_ptr<FileStoreWrite>> 
FileStoreWrite::Create(std::unique_ptr<WriteContext> ctx) {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write"));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, 
FileStoreTableFactory::Create(*ctx));
+    return table->NewWrite(std::move(ctx));
+}
+
+Result<std::unique_ptr<TableScan>> FileStoreTable::NewScan(
+    std::unique_ptr<ScanContext> context) const {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan"));
+    const std::shared_ptr<TableSchema>& table_schema = store_->GetSchema();
+    if (table_schema == nullptr) {
+        return Status::Invalid("not found latest schema");
+    }
+    PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, 
context->GetOptions()));
+    PAIMON_ASSIGN_OR_RAISE(bool read_optimized,
+                           
OptionsUtils::GetValueFromMap<bool>(context->GetOptions(),
+                                                               
kReadOptimizedScanOption, false));
+    // The branch in the store's options is the table's, so one the context 
carries is not honoured.
+    CoreOptions core_options = store_->GetOptions();
+    // The cache is a runtime setting of this scan, not of the table.
+    core_options.WithCache(context->GetCache());
+
+    PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, 
core_options));
+
+    PAIMON_RETURN_NOT_OK(
+        RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, 
read_optimized));
+    // validate options
+    if (core_options.GetBucket() == -1) {
+        if (!table_schema->PrimaryKeys().empty()) {
+            return Status::NotImplemented(fmt::format(
+                "do not support pk table bucket={} in scan process", 
core_options.GetBucket()));
+        }
+    } else if (core_options.GetBucket() < 1 &&
+               !SchemaValidation::IsPostponeBucketTable(*table_schema, 
core_options.GetBucket())) {
+        return Status::Invalid(
+            fmt::format("do not support bucket={} in scan process", 
core_options.GetBucket()));
+    }
+
+    // validate schema and scan filter
+    auto arrow_schema = 
DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+    if (context->GetScanFilters() && 
context->GetScanFilters()->GetPredicate()) {
+        PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema(
+            *arrow_schema, context->GetScanFilters()->GetPredicate(),
+            /*validate_field_idx=*/false));
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStorePathFactory> path_factory,
+                           store_->NewPathFactory(arrow_schema, core_options,
+                                                  
core_options.GetFileFormat()->Identifier(),
+                                                  context->GetMemoryPool()));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<ManifestList> manifest_list,
+        store_->NewManifestList(core_options, path_factory, 
context->GetMemoryPool()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<arrow::Schema> partition_schema,
+        FieldMapping::GetPartitionSchema(arrow_schema, 
table_schema->PartitionKeys()));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ManifestFile> manifest_file,
+                           store_->NewManifestFile(core_options, path_factory, 
partition_schema,
+                                                   context->GetMemoryPool()));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<FileStoreScan> file_store_scan,
+        NewFileStoreScan(manifest_list, manifest_file, arrow_schema, 
core_options,
+                         context->GetExecutor(), context->GetMemoryPool(), 
context.get()));
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SplitGenerator> split_generator,
+                           NewSplitGenerator(core_options));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<IndexFileHandler> index_file_handler,
+        store_->NewIndexFileHandler(core_options, path_factory, 
context->GetMemoryPool()));
+    auto snapshot_reader = std::make_shared<SnapshotReader>(
+        file_store_scan, path_factory, std::move(split_generator), 
std::move(index_file_handler));
+    const bool pk_table = !table_schema->PrimaryKeys().empty();
+    if (read_optimized && pk_table && context->IsStreamingMode()) {
+        return Status::NotImplemented(
+            "read-optimized system table does not support streaming scan for 
primary key table");
+    }
+    if (context->IsStreamingMode()) {
+        return std::make_unique<DataTableStreamScan>(core_options, 
snapshot_reader);
+    }
+    auto batch_scan = std::make_unique<DataTableBatchScan>(
+        /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, 
context->GetLimit(),
+        /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != 
nullptr);
+    if (context->GetRealtimeContext()) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> 
realtime_context,
+                               
RealtimeContextImpl::Cast(context->GetRealtimeContext()));
+        return std::make_unique<RealtimeTableScan>(
+            std::move(batch_scan), pk_table, realtime_context, path_factory,
+            snapshot_reader->GetSnapshotManager(), 
core_options.GetFileSystem(),
+            context->GetScanFilters(), 
core_options.GetRealtimeReadViewTtlMillis());
+    }
+    if (core_options.DataEvolutionEnabled()) {
+        return std::make_unique<DataEvolutionBatchScan>(
+            root_path_, snapshot_reader, std::move(batch_scan), table_schema,
+            context->GetGlobalIndexResult(), core_options, 
context->GetMemoryPool(),
+            context->GetExecutor());
+    }
+    if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) {
+        PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions,
+                               
PrimaryKeyIndexDefinitions::Create(*table_schema));
+        if (!definitions.ScalarDefinitions().empty()) {
+            return PrimaryKeyIndexBatchScan::Create(snapshot_reader, 
std::move(batch_scan),
+                                                    table_schema, 
path_factory, core_options,
+                                                    context->GetMemoryPool());
+        }
+    }
+    return batch_scan;
+}
+
+Result<std::unique_ptr<TableScan>> 
TableScan::Create(std::unique_ptr<ScanContext> context) {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan"));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, 
FileStoreTableFactory::Create(*context));
+    return table->NewScan(std::move(context));
+}
+
+Result<std::unique_ptr<TableRead>> FileStoreTable::CreateTableRead(
+    const std::shared_ptr<InternalReadContext>& internal_context,
+    const std::shared_ptr<MemoryPool>& memory_pool,
+    const std::shared_ptr<Executor>& executor) const {
+    if 
(PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) {
+        return Status::NotImplemented("FullTextMatch predicate execution is 
not implemented yet");
+    }
+    if (!internal_context->GetIndexScoreBindings().empty()) {
+        return Status::NotImplemented("Per-field index score output is not 
implemented yet");
+    }
+    const auto& core_options = internal_context->GetCoreOptions();
+    const auto& table_schema = internal_context->GetTableSchema();
+    if (internal_context->GetRealtimeContext() && 
!core_options.RealtimeEnabled()) {
+        return Status::Invalid("real-time read requires 
realtime.enabled=true");
+    }
+    PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, 
core_options));
+    auto arrow_schema = 
DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<FileStorePathFactory> path_factory,
+        store_->NewPathFactory(arrow_schema, core_options,
+                               core_options.GetFileFormat()->Identifier(), 
memory_pool));
+    // The reader is selected by the schema the read serves, not by the 
table's kind: an
+    // append-only table can have a primary-key fallback branch and vice 
versa, so a reader typed
+    // after the main table would fail the fallback's splits.
+    if (internal_context->GetPrimaryKeys().empty()) {
+        return std::make_unique<AppendOnlyTableRead>(path_factory, 
internal_context, memory_pool,
+                                                     executor);
+    }
+    return KeyValueTableRead::Create(path_factory, internal_context, 
memory_pool, executor);
+}
+
+Result<std::unique_ptr<TableRead>> FileStoreTable::NewRead(
+    std::unique_ptr<ReadContext> context) const {
+    std::shared_ptr<ReadContext> read_context = std::move(context);
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, 
"read"));
+    const std::shared_ptr<TableSchema>& loaded_schema = store_->GetSchema();
+    if (loaded_schema == nullptr) {
+        return Status::Invalid("not found latest schema");
+    }
+    PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, 
read_context->GetOptions()));
+    // The read reads through the branch the table serves, not the context's 
own.
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> 
internal_context,
+                           InternalReadContext::Create(read_context, 
loaded_schema,
+                                                       store_->GetOptions(), 
&root_path_));
+
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read,
+                           CreateTableRead(internal_context, 
read_context->GetMemoryPool(),
+                                           read_context->GetExecutor()));
+
+    std::optional<std::string> scan_fallback_branch =
+        internal_context->GetCoreOptions().GetScanFallbackBranch();
+    if (!scan_fallback_branch ||
+        StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) {
+        return std::move(table_read);
+    }
+
+    // The fallback branch is not the table's, so its schema is read through 
its own manager.
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> 
fallback_context,
+                           InternalReadContext::CreateForFallbackBranch(
+                               read_context, 
/*branch=*/scan_fallback_branch.value(), file_system_,
+                               root_path_, dynamic_options_));
+
+    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> fallback_table_read,
+                           CreateTableRead(fallback_context, 
read_context->GetMemoryPool(),
+                                           read_context->GetExecutor()));
+    const std::shared_ptr<arrow::MemoryPool>& arrow_pool = 
internal_context->GetArrowMemoryPool();
+    return std::make_unique<FallbackTableRead>(std::move(table_read),
+                                               std::move(fallback_table_read), 
arrow_pool);
+}
+
+Result<std::unique_ptr<TableRead>> 
TableRead::Create(std::unique_ptr<ReadContext> ctx) {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read"));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, 
FileStoreTableFactory::Create(*ctx));
+    return table->NewRead(std::move(ctx));
+}
+
+Result<std::unique_ptr<FileStoreCommit>> FileStoreTable::NewCommit(
+    std::unique_ptr<CommitContext> ctx) const {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit"));
+    PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog()));
+    std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem();
+    // A catalog commit names its table; the file system comes from the 
table's own credentials.
+    if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && 
!ctx->GetIdentifier()) {
+        return Status::Invalid("a catalog commit requires a table identifier");
+    }
+    // A commit commits to the root path and branch the table was loaded with, 
not the context's.
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Identifier> catalog_identifier,
+                           
CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch()));
+    // The id a snapshot records the catalog's current schema by, read through 
the loader below
+    // rather than trusted from a schema this commit will not re-read.
+    FileStoreCommitImpl::SchemaIdLoader schema_id_loader;
+    if (ctx->GetCatalog() != nullptr) {
+        if (!catalog_identifier) {
+            return Status::Invalid("a catalog commit requires a table 
identifier");
+        }
+        std::shared_ptr<Catalog> catalog = ctx->GetCatalog();
+        Identifier identifier = catalog_identifier.value();
+        schema_id_loader = [catalog, identifier]() -> Result<int64_t> {
+            PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> schema,
+                                   catalog->LoadTableSchema(identifier));
+            std::shared_ptr<DataSchema> data_schema = 
std::dynamic_pointer_cast<DataSchema>(schema);
+            if (data_schema == nullptr) {
+                return Status::Invalid(
+                    "the catalog did not hand back a data table schema, which 
has the id a "
+                    "snapshot records");
+            }
+            return data_schema->Id();
+        };
+    }
+    const std::shared_ptr<TableSchema>& schema = store_->GetSchema();
+    if (schema == nullptr) {
+        return Status::Invalid(
+            fmt::format("not found latest schema in branch {}", 
CurrentBranch()));
+    }
+    std::shared_ptr<arrow::Schema> arrow_schema =
+        DataField::ConvertDataFieldsToArrowSchema(schema->Fields());
+    PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, 
ctx->GetOptions()));
+    CoreOptions options = store_->GetOptions();
+    PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options));
+    assert(options.GetFileSystem());
+    assert(options.GetFileFormat());
+    PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options));
+    PAIMON_ASSIGN_OR_RAISE(bool is_object_store, 
FileSystem::IsObjectStore(root_path_));
+    // Catalog commits do not require atomic rename on the object store. The 
inte-test marker is
+    // not a table option, so it is read from the context the commit was 
handed.
+    if (is_object_store && !ctx->UseRESTCatalogCommit() &&
+        AsVersionManaged(ctx->GetCatalog()) == nullptr &&
+        ctx->GetOptions().find("enable-object-store-commit-in-inte-test") ==
+            ctx->GetOptions().end()) {
+        return Status::NotImplemented(
+            "commit operation does not support object store file system for 
now");
+    }
+    PAIMON_ASSIGN_OR_RAISE(
+        std::unique_ptr<BinaryRowPartitionComputer> partition_computer,
+        store_->NewPartitionComputer(arrow_schema, options, 
ctx->GetMemoryPool()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<FileStorePathFactory> path_factory,
+        store_->NewPathFactory(arrow_schema, options, 
options.GetFileFormat()->Identifier(),
+                               ctx->GetMemoryPool()));
+
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<SnapshotManager> snapshot_manager,
+        store_->NewSnapshotManager(options, ctx->GetCatalog(), 
catalog_identifier));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ManifestList> manifest_list,
+                           store_->NewManifestList(options, path_factory, 
ctx->GetMemoryPool()));
+
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema,
+                           FieldMapping::GetPartitionSchema(arrow_schema, 
schema->PartitionKeys()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<ManifestFile> manifest_file,
+        store_->NewManifestFile(options, path_factory, partition_schema, 
ctx->GetMemoryPool()));
+    PAIMON_ASSIGN_OR_RAISE(
+        std::shared_ptr<IndexManifestFile> index_manifest_file,
+        store_->NewIndexManifestFile(options, path_factory, 
ctx->GetMemoryPool()));
+
+    auto expire_snapshots = std::make_shared<ExpireSnapshots>(
+        snapshot_manager, path_factory, manifest_list, manifest_file, 
index_manifest_file,
+        options.GetFileSystem(), options.GetExpireConfig(), 
options.RealtimeEnabled(),
+        ctx->GetExecutor());
+
+    CommitScanner::ScanSupplier scan_supplier =
+        NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, 
arrow_schema, options,
+                              ctx->GetExecutor(), ctx->GetMemoryPool());
+
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SnapshotCommit> snapshot_commit,
+                           SnapshotCommit::Create(ctx->GetCatalog(), 
catalog_identifier,
+                                                  ctx->GetTableId(), 
ctx->UseRESTCatalogCommit(),
+                                                  options.GetFileSystem(), 
snapshot_manager));
+
+    return std::make_unique<FileStoreCommitImpl>(
+        ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, 
ctx->GetCommitUser(),
+        options, path_factory, std::move(partition_computer), 
snapshot_manager, snapshot_commit,
+        ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema,
+        std::move(schema_id_loader), manifest_file, manifest_list, 
index_manifest_file,
+        expire_snapshots, GetSchemaManager(), std::move(scan_supplier));
+}
+
+Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create(
+    std::unique_ptr<CommitContext> ctx) {
+    PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit"));
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, 
FileStoreTableFactory::Create(*ctx));
+    return table->NewCommit(std::move(ctx));
+}

Review Comment:
   the `XxxOp::Create` impl is not in xxx_op.cpp



##########
src/paimon/core/table/file_store_table_factory.cpp:
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/file_store_table_factory.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/catalog/table.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/file_store_table.h"
+#include "paimon/core/table/system/system_table.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/predicate/predicate_utils.h"
+#include "paimon/read_context.h"
+#include "paimon/result.h"
+#include "paimon/scan_context.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/table/format/format_table.h"
+#include "paimon/write_context.h"
+
+namespace paimon {
+
+namespace {
+
+/// Recognises a format table behind `table_path`, or returns null when it is 
not one. The scan,
+/// read, write and commit entry points below all dispatch on the same 
question, so one place
+/// answers it and they cannot disagree. Null is also the answer when no 
schema was found under
+/// the path at all: what to say about a table that is not there is the 
managed path's to decide.
+///
+/// The table type is read from the schema, never from `options`: `type` is 
structural, and one
+/// call's options must not decide what kind of table this is.
+Result<std::shared_ptr<FormatTable>> TryLoadFormatTable(
+    const std::shared_ptr<FileSystem>& file_system, const std::string& 
table_path,
+    const std::string& branch, const std::map<std::string, std::string>& 
options,
+    const std::shared_ptr<TableSchema>& table_schema) {
+    if (table_schema == nullptr) {
+        return std::shared_ptr<FormatTable>();
+    }
+    // From the schema alone: see `FormatTable::Create()`.
+    PAIMON_ASSIGN_OR_RAISE(bool is_format_table,
+                           
TableTypeDefine::IsFormatTable(table_schema->Options()));
+    if (!is_format_table) {
+        return std::shared_ptr<FormatTable>();
+    }
+    // A format table has no branches, so it is refused rather than served 
from the table path
+    // under another branch's name. See 
`SchemaValidation::ValidateFormatTableSchema()`.
+    if (BranchManager::NormalizeBranch(branch) != 
BranchManager::DEFAULT_MAIN_BRANCH) {
+        return Status::Invalid(
+            fmt::format("a format table has no branches, so {} cannot be read 
on branch '{}'",
+                        table_path, branch));
+    }
+    Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, 
branch);
+    return FormatTable::Create(file_system, table_path, identifier,
+                               checked_pointer_cast<DataSchema>(table_schema),
+                               /*location_carries_paimon_metadata=*/true, 
options);
+}
+
+/// The options a context carries, merged into the table the load hands back: 
a table is the
+/// product of the load, so the options speak there - the way `Copy` merges 
them - and `NewXxx` on
+/// the product serves them without merging again. A caller holding a table 
already changes it
+/// through `Copy` itself, and an option that contradicts the table that way 
is refused.
+Result<std::shared_ptr<Table>> WithContextOptions(
+    const std::shared_ptr<Table>& table, const std::shared_ptr<TableSchema>& 
schema,
+    const std::map<std::string, std::string>& options) {
+    if (table == nullptr || schema == nullptr || options.empty()) {
+        return table;
+    }
+    const std::map<std::string, std::string>& schema_options = 
schema->Options();
+    for (const auto& [key, value] : options) {
+        auto schema_option = schema_options.find(key);
+        // A key the schema does not carry, or carries differently: the merge 
has something to do.
+        if (schema_option == schema_options.end() || schema_option->second != 
value) {
+            return table->Copy(options);
+        }
+    }
+    return table;
+}
+
+/// The tail every `Create` ends with: the schema one of them read says what 
the path names. A
+/// format table is handed back as the `FormatTable` it is, everything else as 
the `FileStoreTable`
+/// a managed table is, carrying the schema for its `NewXxx` to build from. A 
null schema is handed
+/// over as it is - see `TryLoadFormatTable`.
+///
+/// @param file_system The file system the schema above was read through, 
which a format table
+///        is served by - it has no schema options of its own to name another 
one.
+/// @param explicit_file_system The file system the context supplied, which a 
managed table is
+///        opened through when there is one. A null one leaves the table to 
resolve from its
+///        schema's own access configuration rather than being pinned to the 
file system this load
+///        happened to read the schema through.
+Result<std::shared_ptr<Table>> TableOfLoadedSchema(
+    const std::shared_ptr<FileSystem>& file_system,
+    const std::shared_ptr<FileSystem>& explicit_file_system, const 
std::string& path,
+    const std::string& branch, const std::map<std::string, std::string>& 
options,
+    const std::shared_ptr<SchemaManager>& schema_manager,
+    const std::shared_ptr<TableSchema>& latest_schema, const 
std::optional<Identifier>& identifier,
+    const std::string& uuid = "", const std::map<std::string, std::string>& 
fs_scheme_map = {}) {
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> format_table,
+                           TryLoadFormatTable(file_system, path, branch, 
options, latest_schema));
+    if (format_table != nullptr) {
+        return std::shared_ptr<Table>(std::move(format_table));
+    }
+    // The branch the schema was read on is folded into the object name.
+    Identifier table_identifier =
+        identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch));
+    // The manager was built over the file system the context's own options 
resolve, while the
+    // store resolves the schema's: a schema carrying access configuration 
names a backend the
+    // context's options never did, and a manager kept through that 
disagreement would read the
+    // table's metadata through the wrong one. The schema is in hand already, 
so dropping the
+    // manager costs its cache alone. A context that supplied a file system 
wins on both sides and
+    // keeps it; a scheme map only the store's resolution takes part in is the 
same disagreement.
+    std::shared_ptr<SchemaManager> manager = schema_manager;
+    if (manager != nullptr && latest_schema != nullptr && explicit_file_system 
== nullptr &&
+        (!fs_scheme_map.empty() ||
+         !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), 
options))) {
+        manager = nullptr;
+    }
+    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStoreTable> table,
+                           FileStoreTable::Create(explicit_file_system, path, 
latest_schema,
+                                                  table_identifier, uuid, 
branch, manager,
+                                                  
/*snapshot_manager=*/nullptr, fs_scheme_map));
+    return WithContextOptions(table, latest_schema, options);
+}
+
+/// The managed table a context names behind a path: the schema is read 
through a manager of the
+/// context's branch - handed over instead when the caller already read it - 
and what the schema
+/// says is there decides the table `TableOfLoadedSchema` hands back.
+///
+/// @param schema_json The schema of the table, when the caller already read 
it, or null when it
+///        has to be read here. It is honoured on the main branch only; a 
branch publishes its own
+///        under `branch/branch-<name>`, which is where the read below goes.
+/// @param explicit_file_system The file system the context supplied, which 
the managed table the
+///        schema describes is opened through - see `TableOfLoadedSchema`.
+Result<std::shared_ptr<Table>> ManagedTableOf(
+    const std::shared_ptr<FileSystem>& file_system,
+    const std::shared_ptr<FileSystem>& explicit_file_system, const 
std::string& path,
+    const std::string& branch, const std::map<std::string, std::string>& 
options,
+    const std::optional<std::string>& schema_json, const 
std::optional<Identifier>& identifier,
+    const std::string& uuid = "", const std::map<std::string, std::string>& 
fs_scheme_map = {}) {
+    auto schema_manager = std::make_shared<SchemaManager>(file_system, path, 
branch);
+    std::shared_ptr<TableSchema> latest_schema;
+    if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) {
+        // Handing the schema over only saves reading it. A context always 
names a table path and
+        // this library writes the metadata under it, so `schema` and `branch` 
stay metadata.
+        PAIMON_ASSIGN_OR_RAISE(latest_schema, 
TableSchema::CreateFromJson(schema_json.value()));
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> 
latest,
+                               schema_manager->Latest());
+        if (latest) {
+            latest_schema = *latest;
+        }
+    }
+    return TableOfLoadedSchema(file_system, explicit_file_system, path, 
branch, options,
+                               schema_manager, latest_schema, identifier, 
uuid, fs_scheme_map);
+}
+
+/// The schema a catalog may hold for the table, read from it on the main 
branch only and null
+/// otherwise: a branch publishes its own under `branch/branch-<name>`, which 
is where
+/// `ManagedTableOf` reads instead, so a catalog answering for no branch 
schema does not fail the
+/// `kind` operation to one. `kind` also names the operator the messages speak 
of.
+Result<std::optional<std::string>> CatalogMainBranchSchemaJson(
+    const std::shared_ptr<Catalog>& catalog, const std::optional<Identifier>& 
catalog_identifier,
+    const std::string& branch, const char* kind) {
+    if (catalog == nullptr) {
+        return std::optional<std::string>();
+    }
+    if (!catalog_identifier) {
+        return Status::Invalid(fmt::format("a catalog {} requires a table 
identifier", kind));
+    }
+    if (BranchManager::IsMainBranch(branch)) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> schema,
+                               
catalog->LoadTableSchema(catalog_identifier.value()));
+        PAIMON_ASSIGN_OR_RAISE(std::string schema_json, 
schema->GetJsonSchema());
+        return std::optional<std::string>(std::move(schema_json));
+    }
+    return std::optional<std::string>();
+}
+
+}  // namespace
+
+/// The table a scan context names: the format table the builder was handed, a 
system table behind
+/// a `T$system` path, or the table the schema under the context's path says 
is there.
+Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const 
ScanContext& ctx) {
+    if (ctx.GetFormatTable() != nullptr) {
+        return std::shared_ptr<Table>(ctx.GetFormatTable());
+    }
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
+                           CoreOptions::FromMap(ctx.GetOptions(), 
ctx.GetSpecificFileSystem(), {}));
+    PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path,
+                           SystemTableLoader::TryParsePath(ctx.GetPath()));
+    if (system_table_path) {
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table,
+                               
SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(),
+                                                               ctx.GetPath(), 
ctx.GetOptions()));
+        return std::shared_ptr<Table>(std::move(system_table));
+    }
+    // One `TableScan` interface serves a format table too, the way Java 
Paimon serves both
+    // through one `ReadBuilder`; a format table is planned by listing 
directories instead.
+    const std::string branch = 
BranchManager::NormalizeBranch(tmp_options.GetBranch());
+    return ManagedTableOf(tmp_options.GetFileSystem(), 
ctx.GetSpecificFileSystem(), ctx.GetPath(),
+                          branch, ctx.GetOptions(), 
ctx.GetSpecificTableSchema(),
+                          ctx.GetIdentifier());
+}
+
+/// The table a read context names: the format table the builder was handed, a 
system table behind
+/// a `T$system` path, or the table the schema under the context's path says 
is there.
+Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const 
ReadContext& ctx) {
+    if (ctx.GetFormatTable() != nullptr) {
+        return std::shared_ptr<Table>(ctx.GetFormatTable());
+    }
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options,
+                           CoreOptions::FromMap(ctx.GetOptions(), 
ctx.GetSpecificFileSystem(),
+                                                
ctx.GetFileSystemSchemeToIdentifierMap()));
+    PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path,
+                           SystemTableLoader::TryParsePath(ctx.GetPath()));
+    if (system_table_path) {
+        if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) {
+            return Status::NotImplemented(
+                "FullTextMatch predicate execution is not implemented yet");
+        }
+        if (ctx.HasVectorOrFullTextSearch()) {
+            return Status::NotImplemented(
+                "VectorSearch or FullTextSearch is not supported for system 
tables");
+        }
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table,
+                               
SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(),
+                                                               ctx.GetPath(), 
ctx.GetOptions()));
+        return std::shared_ptr<Table>(std::move(system_table));
+    }
+
+    // A format table has no manifest and a file another engine wrote carries 
no field ids, so how
+    // a row is put back together is its own; how a file is opened is not - 
both paths go through
+    // `DataFileReaderFactory`.
+    const std::string branch = ctx.GetBranch();
+    // The scheme map the schema was read through, so the table's data 
resolves custom schemes the
+    // same way its metadata does.
+    return ManagedTableOf(tmp_core_options.GetFileSystem(), 
ctx.GetSpecificFileSystem(),
+                          ctx.GetPath(), branch, ctx.GetOptions(), 
ctx.GetSpecificTableSchema(),
+                          ctx.GetIdentifier(), /*uuid=*/"",
+                          ctx.GetFileSystemSchemeToIdentifierMap());
+}
+
+/// The table a write context names: the format table the builder was handed, 
or the table the
+/// schema under the context's root path says is there.
+Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const 
WriteContext& ctx) {
+    if (ctx.GetFormatTable() != nullptr) {
+        return std::shared_ptr<Table>(ctx.GetFormatTable());
+    }
+    PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog()));
+    std::shared_ptr<FileSystem> specific_fs = ctx.GetSpecificFileSystem();
+    if (specific_fs == nullptr && 
ctx.GetFileSystemSchemeToIdentifierMap().empty() &&
+        ctx.GetCatalog() != nullptr) {
+        if (!ctx.GetIdentifier()) {
+            return Status::Invalid("a catalog write requires a table 
identifier");
+        }
+        // A catalog issuing per-table temporary credentials only hands them 
out through
+        // GetTableFileSystem, so the write uses this table's.
+        PAIMON_ASSIGN_OR_RAISE(specific_fs,
+                               
ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value()));
+    }
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
+                           CoreOptions::FromMap(ctx.GetOptions(), specific_fs,
+                                                
ctx.GetFileSystemSchemeToIdentifierMap()));
+    std::string branch = ctx.GetBranch();
+    PAIMON_ASSIGN_OR_RAISE(std::optional<Identifier> catalog_identifier,
+                           CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), 
branch));
+    // Only the main branch may have a current schema the catalog alone holds: 
a branch publishes
+    // its own under `branch/branch-<name>`, which is where the load below 
reads it instead.
+    PAIMON_ASSIGN_OR_RAISE(
+        std::optional<std::string> catalog_table_schema,
+        CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, 
branch, "write"));
+    return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, 
ctx.GetRootPath(), branch,
+                          ctx.GetOptions(), catalog_table_schema, 
ctx.GetIdentifier(), /*uuid=*/"",
+                          ctx.GetFileSystemSchemeToIdentifierMap());
+}
+
+/// The table a commit context names: the format table the builder was handed, 
or the table the
+/// schema under the context's root path says is there.
+Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const 
CommitContext& ctx) {
+    if (ctx.GetFormatTable() != nullptr) {
+        return std::shared_ptr<Table>(ctx.GetFormatTable());
+    }
+    PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog()));
+    // Use this table's catalog credentials unless the caller supplied a file 
system: a catalog
+    // issuing per-table temporary ones only hands them out through 
GetTableFileSystem.
+    std::shared_ptr<FileSystem> specific_fs = ctx.GetSpecificFileSystem();
+    if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) {
+        if (!ctx.GetIdentifier()) {
+            return Status::Invalid("a catalog commit requires a table 
identifier");
+        }
+        PAIMON_ASSIGN_OR_RAISE(specific_fs,
+                               
ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value()));
+    }
+    PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
+                           CoreOptions::FromMap(ctx.GetOptions(), 
specific_fs));
+    const std::string& root_path = ctx.GetRootPath();
+    // The branch the context was built for, already validated by 
`CommitContextBuilder`.
+    PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch(

Review Comment:
   why different with WriteContext's Create? One use `std::string branch = 
ctx.GetBranch();`, the other one use `BranchManager::ResolveBranch`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to