SteNicholas commented on code in PR #556:
URL: https://github.com/apache/paimon-cpp/pull/556#discussion_r4583893881
##########
src/paimon/core/operation/read_context_test.cpp:
##########
@@ -375,97 +170,10 @@ TEST(ReadContextTest,
TestSearchRequiresPreciseFileIndexEvaluate) {
}
}
-TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) {
- ReadContextBuilder builder("table_root_path");
- builder.AddOption("old", "value");
- builder.SetOptions({{"key1", "value1"}, {"key2", "value2"}});
-
- ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
-
- std::map<std::string, std::string> expected_options = {{"key1", "value1"},
{"key2", "value2"}};
- ASSERT_EQ(expected_options, ctx->GetOptions());
-}
-
-TEST(ReadContextTest, TestBranch) {
- ReadContextBuilder option_builder("table_root_path");
- ASSERT_OK_AND_ASSIGN(auto option_ctx,
option_builder.AddOption(Options::BRANCH, "rt").Finish());
- ASSERT_EQ("rt", option_ctx->GetBranch());
-
- ReadContextBuilder agreeing_builder("table_root_path");
- ASSERT_OK_AND_ASSIGN(
- auto agreeing_ctx,
- agreeing_builder.WithBranch("rt").AddOption(Options::BRANCH,
"rt").Finish());
- ASSERT_EQ("rt", agreeing_ctx->GetBranch());
-
- ReadContextBuilder mixed_builder("table_root_path");
-
ASSERT_NOK_WITH_MSG(mixed_builder.WithBranch("rt").AddOption(Options::BRANCH,
"dev").Finish(),
- "but both 'dev' and 'rt' were named");
-
- // An empty branch selects the main branch and stays accepted.
- ReadContextBuilder main_builder("table_root_path");
- main_builder.WithBranch("");
- ASSERT_OK_AND_ASSIGN(auto ctx, main_builder.Finish());
- ASSERT_EQ(BranchManager::DEFAULT_MAIN_BRANCH, ctx->GetBranch());
- ReadContextBuilder empty_builder("table_root_path");
-
ASSERT_NOK_WITH_MSG(empty_builder.WithBranch("").AddOption(Options::BRANCH,
"dev").Finish(),
- "but both 'dev' and 'main' were named");
-
- // The branch names a directory under the table path, so a value that is
not a single path
- // component is rejected when the context is built.
- ReadContextBuilder escaping_builder("table_root_path");
- escaping_builder.WithBranch("rt/../../../../../outside");
- ASSERT_NOK_WITH_MSG(escaping_builder.Finish(), "branch name cannot contain
path separators");
-}
-
-TEST(ReadContextTest, TestBranchAgreesWithIdentifier) {
- // A read through a catalog names its table with an Identifier that can
itself carry a branch,
- // so that branch and the one WithBranch() names must agree, exactly as
the write and commit
- // builders already require: one read cannot be aimed at two branches at
once.
- auto catalog = std::make_shared<MockVersionManagedCatalog>();
- catalog->SetTableFileSystem(std::make_shared<MockFileSystem>());
-
- // The identifier's branch is honored when WithBranch() stays silent.
- ReadContextBuilder identifier_builder("table_root_path");
- identifier_builder.WithCatalog(catalog, Identifier("db1", "t1", "dev"));
- ASSERT_OK_AND_ASSIGN(auto identifier_ctx, identifier_builder.Finish());
- ASSERT_EQ("dev", identifier_ctx->GetBranch());
-
- // An agreeing WithBranch() is accepted.
- ReadContextBuilder agreeing_builder("table_root_path");
- agreeing_builder.WithCatalog(catalog, Identifier("db1", "t1",
"dev")).WithBranch("dev");
- ASSERT_OK_AND_ASSIGN(auto agreeing_ctx, agreeing_builder.Finish());
- ASSERT_EQ("dev", agreeing_ctx->GetBranch());
-
- // A disagreeing WithBranch() is refused, and both branches are named.
- ReadContextBuilder mixed_builder("table_root_path");
- mixed_builder.WithCatalog(catalog, Identifier("db1", "t1",
"dev")).WithBranch("rt");
- ASSERT_NOK_WITH_MSG(mixed_builder.Finish(), "but both 'dev' and 'rt' were
named");
-}
-
-TEST(ReadContextTest, TestFileSystemAndSchemeMapConflict) {
- ReadContextBuilder builder("table_root_path");
- auto fs = std::make_shared<MockFileSystem>();
- builder.WithFileSystem(fs);
- builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
- ASSERT_NOK_WITH_MSG(
- builder.Finish(),
- "WithFileSystem() and WithFileSystemSchemeToIdentifierMap() cannot be
used together");
-}
-
-TEST(ReadContextTest, TestSchemeMapWithoutFileSystem) {
- ReadContextBuilder builder("table_root_path");
- builder.WithFileSystemSchemeToIdentifierMap({{"file", "local"}});
- ASSERT_OK_AND_ASSIGN(auto ctx, builder.Finish());
- std::map<std::string, std::string> expected_fs_map = {{"file", "local"}};
- ASSERT_EQ(expected_fs_map, ctx->GetFileSystemSchemeToIdentifierMap());
- ASSERT_FALSE(ctx->GetSpecificFileSystem());
-}
-
TEST(ReadContextTest, TestPrefetchMaxParallelNumZero) {
- ReadContextBuilder builder("table_root_path");
+ ReadContextBuilder builder;
Review Comment:
**Restore the assertion that calls `Finish()`.** This test now only sets the
prefetch fields and never executes the validation it is named for, so it passes
even if zero prefetch parallelism is accepted. The check still exists in
`ReadContextBuilder::Finish()` and is an execution-context concern, so
separating table loading does not make this regression coverage obsolete.
Please retain `ASSERT_NOK_WITH_MSG(builder.Finish(), "prefetch max parallel num
should be greater than 0")`.
##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -460,24 +439,19 @@ Result<std::unique_ptr<TableScan>>
FileStoreTable::NewScan(
std::map<std::string, std::string> read_options;
PAIMON_ASSIGN_OR_RAISE(
read_options, CatalogUtils::ResolveQueryAuthReadOptions(
- context->GetCatalog(), context->GetIdentifier(),
context->GetOptions(),
- CurrentBranch(), file_system_, root_path_, true,
"scan"));
+ catalog_, identifier_, dynamic_options,
CurrentBranch(), file_system_,
Review Comment:
**Use the retained catalog to resolve scan snapshots too.** Writers and
committers created from a catalog-loaded table now use `catalog_` and can
publish a snapshot only through the catalog, but `NewScan()` still reaches the
filesystem-only `GetSnapshotManager()` through both managed scan
implementations. Consequently, writing and committing one row through a
version-managed catalog and then calling `NewScan()` on the same table returns
an empty plan (`SnapshotId() == nullopt` although the catalog accepted snapshot
1); an existing filesystem snapshot instead produces stale results. Please
supply the catalog-aware snapshot manager to the scan path as well, and add a
commit-then-scan regression covering a snapshot that has not been published to
the filesystem.
##########
docs/source/user_guide/format_table.rst:
##########
@@ -48,33 +48,27 @@ Write and commit contexts must name the loaded table's
location and, when they
carry a format-table handle, its identity. Conflicting contexts are rejected
before creating an operator, including for overwrite commits.
-Reading and writing go through the entry points every other table uses:
``TableScan::Create``,
-``TableRead::Create``, ``FileStoreWrite::Create`` and
``FileStoreCommit::Create``, each from its
-usual context builder. Each reads the table's schema - from under the table
path, or from the one
-the context carries - and dispatches to a format table when its ``type`` says
so.
-
-That path needs a schema under the table's own location, which is what every
table a file system
-catalog serves has. A table loaded from a catalog that keeps the schema
elsewhere, such as the
-REST catalog, has none: nothing under the location says that it is a format
table, nor that
-``schema`` and ``branch`` below it are data rather than metadata. Such a table
is read and written
-by handing the loaded ``FormatTable`` to the context instead of a path - every
context builder has
-a constructor taking one - which is what Java does through
``FormatTable.newReadBuilder()`` and
+Static ``TableScan::Create``, ``TableRead::Create``, ``FileStoreWrite::Create``
+and ``FileStoreCommit::Create`` load tables through ``TableLoadContext``. They
+read the schema from the table path or ``table_schema_json`` and dispatch by
its
+type. Operation Contexts contain execution settings; dynamic options are a
+separate final argument.
+
+A format table whose schema lives only in a catalog is used through its loaded
+Table handle, as Java uses ``FormatTable.newReadBuilder()`` and
``newBatchWriteBuilder()``:
.. code-block:: cpp
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<paimon::FormatTable> table,
catalog->GetFormatTable(paimon::Identifier("db",
"tbl")));
- paimon::ScanContextBuilder scan_builder(table);
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::ScanContext> scan_context,
- scan_builder.Finish());
+ paimon::ScanContextBuilder().Finish());
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<paimon::TableScan> scan,
- paimon::TableScan::Create(std::move(scan_context)));
+ table->NewScan(std::move(scan_context)));
-The table already carries its schema and the file system it was loaded
through, so a context built
-from one refuses ``SetTableSchema()``, ``WithFileSystem()``,
-``WithFileSystemSchemeToIdentifierMap()``, ``WithCatalog()`` and a branch.
Options
-given at the call still win over the ones the schema stored, as they do
everywhere else.
+The loaded table supplies its schema and file system. Dynamic options passed to
+``New*()`` override stored options.
Review Comment:
**Document `CopyWithoutTimeTravel()` as the way to apply dynamic options.**
The new `NewScan/NewRead/NewWrite/NewCommit` signatures accept only an
operation context, and the context builders no longer expose `SetOptions()`, so
callers cannot pass overrides to `New*()` as instructed here. Show copying the
loaded table with `CopyWithoutTimeTravel(overrides)` and creating the operator
from that copy. The same instruction in `docs/source/api/format_table.rst`
needs updating, together with the paragraphs above about legacy format-table
context constructors, which this PR removes.
##########
src/paimon/core/operation/internal_read_context.cpp:
##########
@@ -507,62 +510,71 @@ Result<std::shared_ptr<InternalReadContext>>
InternalReadContext::CreateWithSche
original->options_, original->arrow_pool_));
result->path_ = original->path_;
result->fs_scheme_map_ = original->fs_scheme_map_;
+ result->catalog_ = original->catalog_;
+ if (original->identifier_) {
+ result->identifier_.emplace(*original->identifier_);
+ }
+ result->dynamic_options_ = original->dynamic_options_;
return result;
}
Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::CreateForTable(
const std::shared_ptr<ReadContext>& context, const
std::shared_ptr<TableSchema>& schema,
CoreOptions options, const std::string& root_path,
- const std::map<std::string, std::string>& fs_scheme_map) {
+ const std::map<std::string, std::string>& fs_scheme_map,
+ const std::shared_ptr<Catalog>& catalog, const std::optional<Identifier>&
identifier,
+ const std::map<std::string, std::string>& dynamic_options) {
PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InternalReadContext> result,
CreateFromCoreOptions(context, schema,
std::move(options)));
result->path_ = root_path;
result->fs_scheme_map_ = fs_scheme_map;
+ result->catalog_ = catalog;
+ if (identifier) {
+ result->identifier_.emplace(*identifier);
+ }
+ result->dynamic_options_ = dynamic_options;
return result;
}
Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::CreateForFallbackBranch(
const std::shared_ptr<ReadContext>& context, const std::string& branch,
const std::shared_ptr<FileSystem>& file_system, const std::string&
root_path,
- const std::map<std::string, std::string>& fs_scheme_map, bool
use_context_load) {
+ const std::map<std::string, std::string>& fs_scheme_map,
+ const std::shared_ptr<Catalog>& catalog, const std::optional<Identifier>&
identifier,
+ const std::map<std::string, std::string>& dynamic_options,
+ const std::optional<std::string>& supplied_main_schema,
+ const std::optional<std::shared_ptr<FileSystem>>& fallback_file_system) {
if (context->GetRealtimeContext()) {
PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch));
}
- std::shared_ptr<FileSystem> fallback_fs = file_system;
- if (use_context_load) {
- // Preserve static-factory loading: metadata uses the caller's storage
configuration;
- // data storage is resolved below after the fallback schema options
are known.
+ std::shared_ptr<FileSystem> metadata_fs = file_system;
+ if (fallback_file_system) {
PAIMON_ASSIGN_OR_RAISE(
CoreOptions metadata_options,
- CoreOptions::FromMap(context->GetOptions(),
context->GetSpecificFileSystem(),
- fs_scheme_map));
- fallback_fs = metadata_options.GetFileSystem();
- }
- std::shared_ptr<TableSchema> schema;
- if (use_context_load && branch == BranchManager::DEFAULT_MAIN_BRANCH &&
- context->GetSpecificTableSchema()) {
- PAIMON_ASSIGN_OR_RAISE(
- schema,
TableSchema::CreateFromJson(context->GetSpecificTableSchema().value()));
+ CoreOptions::FromMap(dynamic_options, *fallback_file_system,
fs_scheme_map));
+ metadata_fs = metadata_options.GetFileSystem();
+ }
+ SchemaManager manager(metadata_fs, root_path, branch);
+ std::optional<std::shared_ptr<TableSchema>> schema;
+ if (BranchManager::IsMainBranch(branch) && supplied_main_schema) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> supplied,
+
TableSchema::CreateFromJson(*supplied_main_schema));
+ schema = std::shared_ptr<TableSchema>(std::move(supplied));
+ } else {
+ PAIMON_ASSIGN_OR_RAISE(schema, manager.Latest());
Review Comment:
**Resolve the fallback schema through the bound catalog when metadata is
catalog-only.** The new catalog-only loading path can successfully load `dev`
with no filesystem schema metadata, but `CreateForFallbackBranch()` ignores its
`catalog` argument here and always uses `manager.Latest()` unless a main schema
was explicitly supplied. A catalog-only load with identifier `tbl$branch_dev`
and `scan.fallback-branch=main` therefore fails during `TableRead::Create()`
with `fallback schema not found`, even when the catalog serves the main schema.
A stale filesystem fallback schema is also used in preference to the catalog's
schema. Please resolve the fallback branch's identifier and schema from the
catalog while preserving the explicit supplied-main-schema precedence, and
cover this combination in the catalog-only load tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]