SteNicholas commented on code in PR #556:
URL: https://github.com/apache/paimon-cpp/pull/556#discussion_r4583893881


##########
src/paimon/core/operation/read_context_test.cpp:
##########
@@ -375,97 +170,10 @@ TEST(ReadContextTest, 
TestSearchRequiresPreciseFileIndexEvaluate) {
     }
 }
 
-TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) {
-    ReadContextBuilder builder("table_root_path");
-    builder.AddOption("old", "value");
-    builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}});
-
-    ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
-
-    std::map<std::string, std::string> expected_options = {{"key1", "value1"}, 
{"key2", "value2"}};
-    ASSERT_EQ(expected_options, ctx->GetOptions());
-}
-
-TEST(ReadContextTest, TestBranch) {
-    ReadContextBuilder option_builder("table_root_path");
-    ASSERT_OK_AND_ASSIGN(auto option_ctx, 
option_builder.AddOption(Options::BRANCH, "rt").Finish());
-    ASSERT_EQ("rt", option_ctx->GetBranch());
-
-    ReadContextBuilder agreeing_builder("table_root_path");
-    ASSERT_OK_AND_ASSIGN(
-        auto agreeing_ctx,
-        agreeing_builder.WithBranch("rt").AddOption(Options::BRANCH, 
"rt").Finish());
-    ASSERT_EQ("rt", agreeing_ctx->GetBranch());
-
-    ReadContextBuilder mixed_builder("table_root_path");
-    
ASSERT_NOK_WITH_MSG(mixed_builder.WithBranch("rt").AddOption(Options::BRANCH, 
"dev").Finish(),
-                        "but both 'dev' and 'rt' were named");
-
-    // An empty branch selects the main branch and stays accepted.
-    ReadContextBuilder main_builder("table_root_path");
-    main_builder.WithBranch("");
-    ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish());
-    ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch());
-    ReadContextBuilder empty_builder("table_root_path");
-    
ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH, 
"dev").Finish(),
-                        "but both 'dev' and 'main' were named");
-
-    // The branch names a directory under the table path, so a value that is 
not a single path
-    // component is rejected when the context is built.
-    ReadContextBuilder escaping_builder("table_root_path");
-    escaping_builder.WithBranch("rt/../../../../../outside");
-    ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain 
path separators");
-}
-
-TEST(ReadContextTest, TestBranchAgreesWithIdentifier) {
-    // A read through a catalog names its table with an Identifier that can 
itself carry a branch,
-    // so that branch and the one WithBranch() names must agree, exactly as 
the write and commit
-    // builders already require: one read cannot be aimed at two branches at 
once.
-    auto catalog = std::make_shared<MockVersionManagedCatalog>();
-    catalog->SetTableFileSystem(std::make_shared<MockFileSystem>());
-
-    // The identifier's branch is honored when WithBranch() stays silent.
-    ReadContextBuilder identifier_builder("table_root_path");
-    identifier_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev"));
-    ASSERT_OK_AND_ASSIGN(auto identifier_ctx, identifier_builder.Finish());
-    ASSERT_EQ("dev", identifier_ctx->GetBranch());
-
-    // An agreeing WithBranch() is accepted.
-    ReadContextBuilder agreeing_builder("table_root_path");
-    agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1", 
"dev")).WithBranch("dev");
-    ASSERT_OK_AND_ASSIGN(auto agreeing_ctx, agreeing_builder.Finish());
-    ASSERT_EQ("dev", agreeing_ctx->GetBranch());
-
-    // A disagreeing WithBranch() is refused, and both branches are named.
-    ReadContextBuilder mixed_builder("table_root_path");
-    mixed_builder.WithCatalog(catalog, Identifier("db1", "t1", 
"dev")).WithBranch("rt");
-    ASSERT_NOK_WITH_MSG(mixed_builder.Finish(), "but both 'dev' and 'rt' were 
named");
-}
-
-TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) {
-    ReadContextBuilder builder("table_root_path");
-    auto fs = std::make_shared<MockFileSystem>();
-    builder.WithFileSystem(fs);
-    builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
-    ASSERT_NOK_WITH_MSG(
-        builder.Finish(),
-        "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be 
used together");
-}
-
-TEST(ReadContextTest, TestSchemeMapWithoutFileSystem) {
-    ReadContextBuilder builder("table_root_path");
-    builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
-    ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
-    std::map<std::string, std::string> expected_fs_map = {{"file", "local"}};
-    ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap());
-    ASSERT_FALSE(ctx->GetSpecificFileSystem());
-}
-
 TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) {
-    ReadContextBuilder builder("table_root_path");
+    ReadContextBuilder builder;

Review Comment:
   **Restore the assertion that calls `Finish()`.** This test now only sets the 
prefetch fields and never executes the validation it is named for, so it passes 
even if zero prefetch parallelism is accepted. The check still exists in 
`ReadContextBuilder::Finish()` and is an execution-context concern, so 
separating table loading does not make this regression coverage obsolete. 
Please retain `ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num 
should be greater than 0")`.



##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -460,24 +439,19 @@ Result<std::unique_ptr<TableScan>> 
FileStoreTable::NewScan(
     std::map<std::string, std::string> read_options;
     PAIMON_ASSIGN_OR_RAISE(
         read_options, CatalogUtils::ResolveQueryAuthReadOptions(
-                          context->GetCatalog(), context->GetIdentifier(), 
context->GetOptions(),
-                          CurrentBranch(), file_system_, root_path_, true, 
"scan"));
+                          catalog_, identifier_, dynamic_options, 
CurrentBranch(), file_system_,

Review Comment:
   **Use the retained catalog to resolve scan snapshots too.** Writers and 
committers created from a catalog-loaded table now use `catalog_` and can 
publish a snapshot only through the catalog, but `NewScan()` still reaches the 
filesystem-only `GetSnapshotManager()` through both managed scan 
implementations. Consequently, writing and committing one row through a 
version-managed catalog and then calling `NewScan()` on the same table returns 
an empty plan (`SnapshotId() == nullopt` although the catalog accepted snapshot 
1); an existing filesystem snapshot instead produces stale results. Please 
supply the catalog-aware snapshot manager to the scan path as well, and add a 
commit-then-scan regression covering a snapshot that has not been published to 
the filesystem.



##########
docs/source/user_guide/format_table.rst:
##########
@@ -48,33 +48,27 @@ Write and commit contexts must name the loaded table's 
location and, when they
 carry a format-table handle, its identity. Conflicting contexts are rejected
 before creating an operator, including for overwrite commits.
 
-Reading and writing go through the entry points every other table uses: 
``TableScan::Create``,
-``TableRead::Create``, ``FileStoreWrite::Create`` and 
``FileStoreCommit::Create``, each from its
-usual context builder. Each reads the table's schema - from under the table 
path, or from the one
-the context carries - and dispatches to a format table when its ``type`` says 
so.
-
-That path needs a schema under the table's own location, which is what every 
table a file system
-catalog serves has. A table loaded from a catalog that keeps the schema 
elsewhere, such as the
-REST catalog, has none: nothing under the location says that it is a format 
table, nor that
-``schema`` and ``branch`` below it are data rather than metadata. Such a table 
is read and written
-by handing the loaded ``FormatTable`` to the context instead of a path - every 
context builder has
-a constructor taking one - which is what Java does through 
``FormatTable.newReadBuilder()`` and
+Static ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create``
+and ``FileStoreCommit::Create`` load tables through ``TableLoadContext``. They
+read the schema from the table path or ``table_schema_json`` and dispatch by 
its
+type. Operation Contexts contain execution settings; dynamic options are a
+separate final argument.
+
+A format table whose schema lives only in a catalog is used through its loaded
+Table handle, as Java uses ``FormatTable.newReadBuilder()`` and
 ``newBatchWriteBuilder()``:
 
 .. code-block:: cpp
 
    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::FormatTable> table,
                           catalog->GetFormatTable(paimon::Identifier("db", 
"tbl")));
-   paimon::ScanContextBuilder scan_builder(table);
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::ScanContext> scan_context,
-                          scan_builder.Finish());
+                          paimon::ScanContextBuilder().Finish());
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::TableScan> scan,
-                          paimon::TableScan::Create(std::move(scan_context)));
+                          table->NewScan(std::move(scan_context)));
 
-The table already carries its schema and the file system it was loaded 
through, so a context built
-from one refuses ``SetTableSchema()``, ``WithFileSystem()``,
-``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch. 
Options
-given at the call still win over the ones the schema stored, as they do 
everywhere else.
+The loaded table supplies its schema and file system. Dynamic options passed to
+``New*()`` override stored options.

Review Comment:
   **Document `CopyWithoutTimeTravel()` as the way to apply dynamic options.** 
The new `NewScan/NewRead/NewWrite/NewCommit` signatures accept only an 
operation context, and the context builders no longer expose `SetOptions()`, so 
callers cannot pass overrides to `New*()` as instructed here. Show copying the 
loaded table with `CopyWithoutTimeTravel(overrides)` and creating the operator 
from that copy. The same instruction in `docs/source/api/format_table.rst` 
needs updating, together with the paragraphs above about legacy format-table 
context constructors, which this PR removes.



##########
src/paimon/core/operation/internal_read_context.cpp:
##########
@@ -507,62 +510,71 @@ Result<std::shared_ptr<InternalReadContext>> 
InternalReadContext::CreateWithSche
                                 original->options_, original->arrow_pool_));
     result->path_ = original->path_;
     result->fs_scheme_map_ = original->fs_scheme_map_;
+    result->catalog_ = original->catalog_;
+    if (original->identifier_) {
+        result->identifier_.emplace(*original->identifier_);
+    }
+    result->dynamic_options_ = original->dynamic_options_;
     return result;
 }
 
 Result<std::unique_ptr<InternalReadContext>> 
InternalReadContext::CreateForTable(
     const std::shared_ptr<ReadContext>& context, const 
std::shared_ptr<TableSchema>& schema,
     CoreOptions options, const std::string& root_path,
-    const std::map<std::string, std::string>& fs_scheme_map) {
+    const std::map<std::string, std::string>& fs_scheme_map,
+    const std::shared_ptr<Catalog>& catalog, const std::optional<Identifier>& 
identifier,
+    const std::map<std::string, std::string>& dynamic_options) {
     PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InternalReadContext> result,
                            CreateFromCoreOptions(context, schema, 
std::move(options)));
     result->path_ = root_path;
     result->fs_scheme_map_ = fs_scheme_map;
+    result->catalog_ = catalog;
+    if (identifier) {
+        result->identifier_.emplace(*identifier);
+    }
+    result->dynamic_options_ = dynamic_options;
     return result;
 }
 Result<std::unique_ptr<InternalReadContext>> 
InternalReadContext::CreateForFallbackBranch(
     const std::shared_ptr<ReadContext>& context, const std::string& branch,
     const std::shared_ptr<FileSystem>& file_system, const std::string& 
root_path,
-    const std::map<std::string, std::string>& fs_scheme_map, bool 
use_context_load) {
+    const std::map<std::string, std::string>& fs_scheme_map,
+    const std::shared_ptr<Catalog>& catalog, const std::optional<Identifier>& 
identifier,
+    const std::map<std::string, std::string>& dynamic_options,
+    const std::optional<std::string>& supplied_main_schema,
+    const std::optional<std::shared_ptr<FileSystem>>& fallback_file_system) {
     if (context->GetRealtimeContext()) {
         PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch));
     }
-    std::shared_ptr<FileSystem> fallback_fs = file_system;
-    if (use_context_load) {
-        // Preserve static-factory loading: metadata uses the caller's storage 
configuration;
-        // data storage is resolved below after the fallback schema options 
are known.
+    std::shared_ptr<FileSystem> metadata_fs = file_system;
+    if (fallback_file_system) {
         PAIMON_ASSIGN_OR_RAISE(
             CoreOptions metadata_options,
-            CoreOptions::FromMap(context->GetOptions(), 
context->GetSpecificFileSystem(),
-                                 fs_scheme_map));
-        fallback_fs = metadata_options.GetFileSystem();
-    }
-    std::shared_ptr<TableSchema> schema;
-    if (use_context_load && branch == BranchManager::DEFAULT_MAIN_BRANCH &&
-        context->GetSpecificTableSchema()) {
-        PAIMON_ASSIGN_OR_RAISE(
-            schema, 
TableSchema::CreateFromJson(context->GetSpecificTableSchema().value()));
+            CoreOptions::FromMap(dynamic_options, *fallback_file_system, 
fs_scheme_map));
+        metadata_fs = metadata_options.GetFileSystem();
+    }
+    SchemaManager manager(metadata_fs, root_path, branch);
+    std::optional<std::shared_ptr<TableSchema>> schema;
+    if (BranchManager::IsMainBranch(branch) && supplied_main_schema) {
+        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> supplied,
+                               
TableSchema::CreateFromJson(*supplied_main_schema));
+        schema = std::shared_ptr<TableSchema>(std::move(supplied));
+    } else {
+        PAIMON_ASSIGN_OR_RAISE(schema, manager.Latest());

Review Comment:
   **Resolve the fallback schema through the bound catalog when metadata is 
catalog-only.** The new catalog-only loading path can successfully load `dev` 
with no filesystem schema metadata, but `CreateForFallbackBranch()` ignores its 
`catalog` argument here and always uses `manager.Latest()` unless a main schema 
was explicitly supplied. A catalog-only load with identifier `tbl$branch_dev` 
and `scan.fallback-branch=main` therefore fails during `TableRead::Create()` 
with `fallback schema not found`, even when the catalog serves the main schema. 
A stale filesystem fallback schema is also used in preference to the catalog's 
schema. Please resolve the fallback branch's identifier and schema from the 
catalog while preserving the explicit supplied-main-schema precedence, and 
cover this combination in the catalog-only load tests.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to