zjw1111 commented on code in PR #527:
URL: https://github.com/apache/paimon-cpp/pull/527#discussion_r4559499384
##########
src/paimon/core/operation/internal_read_context.cpp:
##########
@@ -493,9 +502,59 @@ Result<std::shared_ptr<InternalReadContext>>
InternalReadContext::CreateWithSche
}
// Create a new InternalReadContext sharing all properties except
read_schema.
// The new read_schema is the minimal column set for COUNT(*).
- return std::shared_ptr<InternalReadContext>(
+ auto result = std::shared_ptr<InternalReadContext>(
new InternalReadContext(original->read_context_,
original->table_schema_, new_read_schema,
original->options_, original->arrow_pool_));
+ result->path_ = original->path_;
+ result->fs_scheme_map_ = original->fs_scheme_map_;
+ return result;
+}
+
+Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::CreateForTable(
+ const std::shared_ptr<ReadContext>& context, const
std::shared_ptr<TableSchema>& schema,
+ CoreOptions options, const std::string& root_path,
+ const std::map<std::string, std::string>& fs_scheme_map) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<InternalReadContext> result,
+ CreateFromCoreOptions(context, schema,
std::move(options)));
+ result->path_ = root_path;
+ result->fs_scheme_map_ = fs_scheme_map;
+ return result;
+}
+Result<std::unique_ptr<InternalReadContext>>
InternalReadContext::CreateForFallbackBranch(
+ const std::shared_ptr<ReadContext>& context, const std::string& branch,
+ const std::shared_ptr<FileSystem>& file_system, const std::string&
root_path,
+ const std::map<std::string, std::string>& fs_scheme_map) {
+ if (context->GetRealtimeContext()) {
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch));
+ }
+ std::shared_ptr<TableSchema> schema;
+ if (branch == BranchManager::DEFAULT_MAIN_BRANCH &&
context->GetSpecificTableSchema()) {
+ // A loaded table may receive a context for another path; its supplied
schema must not
+ // replace this table's fallback schema. Static entrypoints use the
context's own path.
+ PAIMON_ASSIGN_OR_RAISE(std::string context_path,
+ PathUtil::NormalizePath(context->GetPath()));
+ PAIMON_ASSIGN_OR_RAISE(std::string table_path,
PathUtil::NormalizePath(root_path));
+ if (context_path == table_path) {
+ PAIMON_ASSIGN_OR_RAISE(
+ schema,
TableSchema::CreateFromJson(context->GetSpecificTableSchema().value()));
+ }
+ }
+ if (!schema) {
+ SchemaManager manager(file_system, root_path, branch);
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>>
latest,
+ manager.Latest());
+ if (!latest) {
+ return Status::Invalid("fallback schema not found");
+ }
+ schema = latest.value();
+ }
+ auto options = schema->Options();
+ for (const auto& [key, value] : context->GetOptions()) {
+ options[key] = value;
+ }
+ options[Options::BRANCH] = branch;
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
CoreOptions::FromMap(options, file_system));
Review Comment:
The static read path now reuses the primary reader's filesystem for
fallback. It previously resolved the fallback data filesystem from the fallback
schema and context options when the caller supplied no filesystem. Fallback can
now fail if the branches rely on different endpoints or credentials. On the
loaded-table path, matching path strings also lets `SetTableSchema()` supply
the main fallback schema even when the context refers to a different
filesystem. Java also reuses one `FileIO`, but loads the fallback schema from
the target branch and reads only the current branch if that schema is missing.
The static filesystem regression and same-path JSON override are specific to
C++.
##########
src/paimon/core/table/file_store_table.cpp:
##########
@@ -0,0 +1,776 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "paimon/core/table/file_store_table.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/identifier.h"
+#include "paimon/commit_context.h"
+#include "paimon/common/fs/resolving_file_system.h"
+#include "paimon/common/predicate/predicate_validator.h"
+#include "paimon/common/types/data_field.h"
+#include "paimon/common/utils/arrow/mem_utils.h"
+#include "paimon/common/utils/binary_row_partition_computer.h"
+#include "paimon/common/utils/checked_cast.h"
+#include "paimon/common/utils/options_utils.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/common/utils/sensitive_config_utils.h"
+#include "paimon/common/utils/string_utils.h"
+#include "paimon/core/catalog/catalog_snapshot_commit.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/commit_lock.h"
+#include "paimon/core/catalog/renaming_snapshot_commit.h"
+#include "paimon/core/catalog/snapshot_commit.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/disk/io_manager.h"
+#include "paimon/core/index/index_file_handler.h"
+#include "paimon/core/index/pk/primary_key_index_definitions.h"
+#include "paimon/core/manifest/index_manifest_file.h"
+#include "paimon/core/operation/commit/realtime_commit_properties.h"
+#include "paimon/core/operation/expire_snapshots.h"
+#include "paimon/core/operation/file_store_commit_impl.h"
+#include "paimon/core/operation/file_store_scan.h"
+#include "paimon/core/operation/internal_read_context.h"
+#include "paimon/core/options/table_type.h"
+#include "paimon/core/realtime/realtime_context_impl.h"
+#include "paimon/core/realtime/realtime_utils.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/schema_validation.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/snapshot.h"
+#include "paimon/core/table/append_only_file_store_table.h"
+#include "paimon/core/table/bucket_mode.h"
+#include "paimon/core/table/file_store.h"
+#include "paimon/core/table/primary_key_file_store_table.h"
+#include "paimon/core/table/source/append_only_table_read.h"
+#include "paimon/core/table/source/data_evolution_batch_scan.h"
+#include "paimon/core/table/source/data_table_batch_scan.h"
+#include "paimon/core/table/source/data_table_stream_scan.h"
+#include "paimon/core/table/source/fallback_table_read.h"
+#include "paimon/core/table/source/key_value_table_read.h"
+#include "paimon/core/table/source/primary_key_index_batch_scan.h"
+#include "paimon/core/table/source/query_auth_table_read.h"
+#include "paimon/core/table/source/query_auth_table_scan.h"
+#include "paimon/core/table/source/read_optimized_scan_options.h"
+#include "paimon/core/table/source/realtime_table_scan.h"
+#include "paimon/core/table/source/snapshot/snapshot_reader.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/context_utils.h"
+#include "paimon/core/utils/field_mapping.h"
+#include "paimon/core/utils/file_store_path_factory.h"
+#include "paimon/core/utils/primary_key_table_utils.h"
+#include "paimon/core/utils/snapshot_manager.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/format/file_format.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/read_context.h"
+#include "paimon/realtime/realtime_context.h"
+#include "paimon/result.h"
+#include "paimon/scan_context.h"
+#include "paimon/schema/schema.h"
+#include "paimon/status.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/write_context.h"
+
+namespace arrow {
+class Schema;
+} // namespace arrow
+
+namespace paimon {
+
+namespace {
+
+/// Allows object-store commits in integration tests.
+constexpr const char kObjectStoreCommitInteTestOption[] =
"enable-object-store-commit-in-inte-test";
+
+Status ValidateRealtimeScan(const TableSchema& table_schema, const
CoreOptions& core_options,
+ const ScanContext& context, bool read_optimized) {
+ if (!context.GetRealtimeContext()) {
+ return Status::OK();
+ }
+ if (!core_options.RealtimeEnabled()) {
+ return Status::Invalid("real-time scan requires
realtime.enabled=true");
+ }
+ if (core_options.GetBucket() <= 0) {
+ return Status::Invalid("real-time union read requires fixed bucket
mode");
+ }
+ if (core_options.DataEvolutionEnabled()) {
+ return Status::Invalid("real-time union read does not support data
evolution");
+ }
+ if (!table_schema.PrimaryKeys().empty()) {
+ if (read_optimized) {
+ return Status::NotImplemented(
+ "PK real-time union read does not support read-optimized
scans");
+ }
+ PAIMON_RETURN_NOT_OK(
+ PrimaryKeyTableUtils::ValidateRealtimeOptions(core_options,
table_schema));
+ }
+ if (context.IsStreamingMode()) {
+ return Status::Invalid("real-time union read currently supports batch
scans only");
+ }
+ if (context.GetLimit()) {
+ return Status::Invalid("real-time union read does not support scan
limit pushdown");
+ }
+ if (context.GetGlobalIndexResult()) {
+ return Status::Invalid("real-time union read does not support global
index splits");
+ }
+ StartupMode startup_mode = core_options.GetStartupMode();
+ const bool specified_snapshot =
+ startup_mode == StartupMode::FromSnapshot() &&
core_options.GetScanSnapshotId().has_value();
+ if (!(startup_mode == StartupMode::LatestFull() || startup_mode ==
StartupMode::Latest() ||
+ specified_snapshot)) {
+ return Status::Invalid(
+ "real-time union read requires the latest snapshot or an explicit
snapshot id");
+ }
+ return Status::OK();
+}
+
+bool IsCatalogCommit(const CommitContext& ctx) {
+ return AsVersionManaged(ctx.GetCatalog()) != nullptr;
+}
+
+Result<std::shared_ptr<SnapshotCommit>> NewSnapshotCommit(
+ const CommitContext& ctx, const std::optional<Identifier>&
catalog_identifier,
+ const CoreOptions& options, const std::shared_ptr<SnapshotManager>&
snapshot_manager) {
+ if (IsCatalogCommit(ctx)) {
+ if (!catalog_identifier) {
+ return Status::Invalid("a catalog commit requires a table
identifier");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<CommitLock> lock,
+ AsVersionManaged(ctx.GetCatalog())
+
->CreateCommitLock(catalog_identifier.value(), ctx.GetTableId(),
+ ctx.GetCommitUser(),
options));
+ // Reuse the caller's UUID so table recreation cannot redirect this
commit.
+ return
std::shared_ptr<SnapshotCommit>(std::make_shared<CatalogSnapshotCommit>(
+ ctx.GetCatalog(), catalog_identifier.value(), ctx.GetTableId(),
std::move(lock)));
+ }
+ if (ctx.UseRESTCatalogCommit()) {
+ if (options.RestCommitAcquireLock()) {
+ return Status::Invalid(fmt::format(
+ "'{}' needs a commit through a REST catalog with
WithCatalog(); a commit table "
+ "request built for the caller to send cannot hold a commit
lease",
+ Options::REST_COMMIT_ACQUIRE_LOCK));
+ }
+ return std::shared_ptr<SnapshotCommit>(
+ std::make_shared<CatalogSnapshotCommit>(ctx.GetTableId()));
+ }
+ // Catalogs without version management may still supply a table ID.
+ if (ctx.GetTableId() && ctx.GetCatalog() == nullptr) {
+ return Status::Invalid(
+ "a commit which publishes the snapshot by writing it to the table
directory sends no "
+ "commit table request, so there is nowhere to put a table id;
commit through a "
+ "catalog with WithCatalog() or ask for the request with
UseRESTCatalogCommit()");
+ }
+ return std::shared_ptr<SnapshotCommit>(
+ std::make_shared<RenamingSnapshotCommit>(options.GetFileSystem(),
snapshot_manager));
+}
+
+} // namespace
+
+class Executor;
+class MemoryPool;
+
+FileStoreTable::FileStoreTable(const std::shared_ptr<FileSystem>& file_system,
+ const std::string& root_path,
+ const std::shared_ptr<TableSchema>& schema,
+ const Identifier& identifier, const
std::string& uuid,
+ const std::map<std::string, std::string>&
options)
+ : Table(file_system, root_path, schema, identifier, uuid),
+ options_(options),
+ schema_manager_(std::make_shared<SchemaManager>(file_system, root_path,
CurrentBranch())) {}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create(
+ const std::shared_ptr<FileSystem>& specified_file_system, const
std::string& root_path,
+ const std::shared_ptr<TableSchema>& schema, const Identifier& identifier,
+ const std::string& uuid, const std::string& branch,
+ const std::map<std::string, std::string>& fs_scheme_map) {
+ if (schema == nullptr) {
+ return Status::Invalid(
+ fmt::format("not found latest schema of {}",
identifier.GetFullName()));
+ }
+ PAIMON_RETURN_NOT_OK(
+ CatalogUtils::CheckManagedTableType(identifier, schema,
"FileStoreTable::Create"));
+ PAIMON_RETURN_NOT_OK(BranchManager::CheckValidBranch(branch));
+ std::map<std::string, std::string> options = schema->Options();
+ const std::string normalized_branch =
BranchManager::NormalizeBranch(branch);
+ if (BranchManager::IsMainBranch(normalized_branch)) {
+ options.erase(Options::BRANCH);
+ } else {
+ options[Options::BRANCH] = normalized_branch;
+ }
+ std::shared_ptr<TableSchema> effective_schema = schema;
+ if (options != schema->Options()) {
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> normalized_schema,
+ schema->Copy(options));
+ effective_schema =
std::shared_ptr<TableSchema>(std::move(normalized_schema));
+ }
+ return CreateWithOptions(specified_file_system, root_path,
effective_schema, identifier, uuid,
+ options, fs_scheme_map);
+}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create(
+ std::unique_ptr<FileStore> store, const std::shared_ptr<TableSchema>&
schema,
+ const Identifier& identifier, const std::string& uuid,
+ const std::map<std::string, std::string>& fs_scheme_map,
+ const std::shared_ptr<SchemaManager>& schema_manager) {
+ if (!store || !schema) {
+ return Status::Invalid("managed table requires a store and schema");
+ }
+ PAIMON_RETURN_NOT_OK(
+ CatalogUtils::CheckManagedTableType(identifier, schema,
"FileStoreTable::Create"));
+
PAIMON_RETURN_NOT_OK(BranchManager::CheckValidBranch(store->CurrentBranch()));
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FileStoreTable> table,
+ CreateWithOptions(store->GetFileSystem(), store->GetRootPath(),
schema, identifier, uuid,
+ store->GetOptions().ToMap(), fs_scheme_map));
+ if (schema_manager) {
+ table->schema_manager_ = schema_manager;
+ }
+ return table;
+}
+
+Result<std::shared_ptr<FileStoreTable>> FileStoreTable::CreateWithOptions(
+ const std::shared_ptr<FileSystem>& specified_file_system, const
std::string& root_path,
+ const std::shared_ptr<TableSchema>& schema, const Identifier& identifier,
+ const std::string& uuid, const std::map<std::string, std::string>& options,
+ const std::map<std::string, std::string>& fs_scheme_map) {
+ // Loading metadata must not require the data-format plugins used by
operators.
+ std::shared_ptr<FileSystem> file_system = specified_file_system;
+ if (!file_system) {
+ PAIMON_ASSIGN_OR_RAISE(std::string default_fs,
OptionsUtils::GetValueFromMap<std::string>(
+ options,
Options::FILE_SYSTEM, "local"));
+ file_system = std::make_shared<ResolvingFileSystem>(
+ fs_scheme_map, StringUtils::ToLowerCase(default_fs), options);
+ }
+ std::shared_ptr<FileStoreTable> table;
+ if (!schema->PrimaryKeys().empty()) {
+ table = std::make_shared<PrimaryKeyFileStoreTable>(file_system,
root_path, schema,
+ identifier, uuid,
options);
+ } else {
+ table = std::make_shared<AppendOnlyFileStoreTable>(file_system,
root_path, schema,
+ identifier, uuid,
options);
+ }
+ table->fs_scheme_map_ = fs_scheme_map;
+ return table;
+}
+
+Result<CoreOptions> FileStoreTable::ResolveOptions(
+ const std::map<std::string, std::string>& overrides) const {
+ auto options = options_;
+ for (const auto& [key, value] : overrides) {
+ options[key] = value;
+ }
+ auto branch = overrides.find(Options::BRANCH);
+ if (branch != overrides.end() &&
+ BranchManager::NormalizeBranch(branch->second) != CurrentBranch()) {
+ return Status::Invalid("context branch contradicts the loaded table");
+ }
+ PAIMON_ASSIGN_OR_RAISE(TableType type,
TableTypeDefine::FromOptions(options));
+ PAIMON_ASSIGN_OR_RAISE(TableType bound_type,
TableTypeDefine::FromOptions(options_));
+ if (type != bound_type) {
+ return Status::Invalid("context type contradicts the loaded table");
+ }
+ return CoreOptions::FromMap(options, file_system_, fs_scheme_map_);
+}
+
+Status FileStoreTable::RestoreRealtimeCommittedProgress(
+ const std::shared_ptr<RealtimeContext>& realtime_context,
+ const std::shared_ptr<SnapshotManager>& snapshot_manager, const
CoreOptions& options) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl>
realtime_context_impl,
+ RealtimeContextImpl::Cast(realtime_context));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+ PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot,
+ snapshot_manager->LatestSnapshot());
+ if (latest_snapshot) {
+ PAIMON_ASSIGN_OR_RAISE(
+ RealtimeOffsetMap realtime_committed_offsets,
+ RealtimeCommitProperties::ReadOffsets(latest_snapshot,
options.GetFileSystem()));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress(
+ latest_snapshot->Id(), realtime_committed_offsets,
/*committed_data_files=*/{}));
+ }
+ return Status::OK();
+}
+
+Status FileStoreTable::CheckContextNamesThisTable(
+ const std::string& kind, const std::string& context_root_path,
+ const std::optional<Identifier>& context_identifier,
+ const std::optional<std::string>& context_table_id) const {
+ if (!root_path_.empty()) {
+ PAIMON_ASSIGN_OR_RAISE(std::string context_path,
+ PathUtil::NormalizePath(context_root_path));
+ PAIMON_ASSIGN_OR_RAISE(std::string table_path,
PathUtil::NormalizePath(root_path_));
+ if (context_path != table_path) {
+ return Status::Invalid(fmt::format(
+ "the {} context names root path '{}' but this table serves
'{}': the store below "
+ "reads and writes this table's files, so a context naming
another path would "
+ "leave this table's metadata there and publish it as the
other's",
+ kind, context_root_path, root_path_));
+ }
+ }
+ if (context_identifier) {
+ PAIMON_ASSIGN_OR_RAISE(std::optional<std::string> context_branch,
+ context_identifier.value().GetBranchName());
+ if (context_branch) {
+ PAIMON_ASSIGN_OR_RAISE(context_branch,
+
context_identifier.value().GetBranchNameOrDefault());
+ }
+ if (context_branch && context_branch.value() != CurrentBranch()) {
+ return Status::Invalid(fmt::format(
+ "the {} context names branch '{}' of '{}' but this table
serves branch '{}', "
+ "which is the one a {} of this table writes and publishes",
+ kind, context_branch.value(),
context_identifier.value().ToString(),
+ CurrentBranch(), kind));
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ Identifier context_table,
+ CatalogUtils::BranchIdentifier(context_identifier.value(),
CurrentBranch()));
+ PAIMON_ASSIGN_OR_RAISE(Identifier served_table,
+ CatalogUtils::BranchIdentifier(identifier_,
CurrentBranch()));
+ const bool names_this_table =
+ (served_table.GetDatabaseName() == Identifier::kUnknownDatabase ||
+ context_table.GetDatabaseName() ==
served_table.GetDatabaseName()) &&
+ context_table.GetObjectName() == served_table.GetObjectName();
+ if (!names_this_table) {
+ return Status::Invalid(fmt::format(
+ "the {} context names table '{}' but this table is '{}': the
store below serves "
+ "this table's files, so a context naming another table would
write this table's "
+ "metadata and publish it as the other's",
+ kind, context_identifier.value().ToString(),
identifier_.ToString()));
+ }
+ }
+ if (context_table_id && context_table_id != CatalogUuid()) {
+ return Status::Invalid(fmt::format(
+ "the {} context carries table id '{}' but this table's is '{}':
the id tells a "
+ "recreated table from the one the changes were prepared for, so
one this table does "
+ "not carry would publish this table's metadata as the
recreation's",
+ kind, context_table_id.value(), CatalogUuid().value_or("none")));
+ }
+ return Status::OK();
+}
+
+std::unique_ptr<FileStoreScan>
FileStoreTable::WithTablePath(std::unique_ptr<FileStoreScan>&& scan,
+ const
std::string& root_path) {
+ scan->WithTablePath(root_path);
+ return std::move(scan);
+}
+
+Result<std::unique_ptr<FileStoreWrite>> FileStoreTable::NewWrite(
+ std::unique_ptr<WriteContext> ctx) const {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write"));
+ PAIMON_RETURN_NOT_OK(CheckContextNamesThisTable("write",
ctx->GetRootPath(),
+ ctx->GetIdentifier(),
+
/*context_table_id=*/std::nullopt));
+ if (AsVersionManaged(ctx->GetCatalog()) != nullptr) {
+
PAIMON_RETURN_NOT_OK(BranchManager::CheckCatalogAddressableBranch(CurrentBranch()));
+ }
+ PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog()));
+ std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem();
+ if (specific_fs == nullptr &&
ctx->GetFileSystemSchemeToIdentifierMap().empty() &&
+ ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) {
+ return Status::Invalid("a catalog write requires a table identifier");
+ }
+ std::optional<Identifier> catalog_identifier;
+ if (ctx->GetIdentifier()) {
+ PAIMON_ASSIGN_OR_RAISE(Identifier identifier,
CatalogUtils::BranchIdentifier(
+
*ctx->GetIdentifier(), CurrentBranch()));
+ catalog_identifier.emplace(std::move(identifier));
+ }
+ if (ctx->GetRealtimeContext()) {
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch()));
+ }
+ std::shared_ptr<TableSchema> schema =
checked_pointer_cast<TableSchema>(schema_);
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions options,
ResolveOptions(ctx->GetOptions()));
+ FileStore file_store(root_path_, options);
+ auto arrow_schema =
DataField::ConvertDataFieldsToArrowSchema(schema->Fields());
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options));
+ PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema,
options));
+ if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) {
+ return Status::NotImplemented(
+ "Dedicated VECTOR storage does not yet support custom write
schemas or partial writes");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema,
+ FieldMapping::GetPartitionSchema(arrow_schema,
schema->PartitionKeys()));
+
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FileStorePathFactory> file_store_path_factory,
+ file_store.NewPathFactory(arrow_schema, schema->PartitionKeys(),
+
options.GetWriteFileFormat(/*level=*/0)->Identifier(),
+ ctx->GetMemoryPool()));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SnapshotManager> snapshot_manager,
+ file_store.NewSnapshotManager(ctx->GetCatalog(),
catalog_identifier));
+ std::shared_ptr<IOManager> io_manager;
+ const auto& io_temp_dir = ctx->GetTempDirectory();
+ if (!io_temp_dir.empty()) {
+ io_manager = std::make_shared<IOManager>(io_temp_dir,
options.GetFileSystem());
+ }
+
+ bool ignore_previous_files = ctx->IgnorePreviousFiles();
+ if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) {
+ return Status::Invalid("real-time write requires
realtime.enabled=true");
+ }
+ if (ctx->GetRealtimeContext()) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl>
realtime_context_impl,
+
RealtimeContextImpl::Cast(ctx->GetRealtimeContext()));
+ PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable());
+ }
+ return CreateWrite(*ctx, arrow_schema, partition_schema, options,
file_store_path_factory,
+ snapshot_manager, io_manager, ignore_previous_files);
+}
+
+Result<std::unique_ptr<TableScan>> FileStoreTable::NewScan(
+ std::unique_ptr<ScanContext> context) const {
+ return NewScan(std::shared_ptr<ScanContext>(std::move(context)), false);
+}
+
+Result<std::unique_ptr<TableScan>> FileStoreTable::NewScan(
+ const std::shared_ptr<ScanContext>& context, bool query_auth_applied)
const {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan"));
+ if (context->GetRealtimeContext()) {
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(CurrentBranch()));
+ }
+ std::shared_ptr<TableSchema> table_schema =
checked_pointer_cast<TableSchema>(schema_);
+ std::map<std::string, std::string> read_options;
+ PAIMON_ASSIGN_OR_RAISE(
+ read_options, CatalogUtils::ResolveQueryAuthReadOptions(
+ context->GetCatalog(), context->GetIdentifier(),
context->GetOptions(),
+ CurrentBranch(), file_system_, root_path_, true,
"scan"));
+ PAIMON_ASSIGN_OR_RAISE(bool query_auth,
SchemaValidation::QueryAuthReadEnabled(
+ table_schema->Options(),
read_options));
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options,
ResolveOptions(read_options));
+ if (query_auth && !query_auth_applied) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FileStoreTable> table,
+ FileStoreTable::Create(std::make_unique<FileStore>(root_path_,
core_options),
+ table_schema, identifier_,
CatalogUuid().value_or(""),
+ fs_scheme_map_));
+ return QueryAuthTableScan::Create(
Review Comment:
If A and B both enable query auth and have matching branches, field names,
and field IDs, passing B's context to A's `NewScan()` and `NewRead()` can apply
B's rules to A's data. The inner scan plans A's files, but the wrapper fetches
B's rules and labels the splits with B's path. The reader checks that label
against B's context and reads A's files. A's row filters or column masks can be
bypassed if B's rules are looser. Java's standard `ReadBuilder` uses the same
table for planning and authorization, so this context mix-up is specific to C++.
##########
src/paimon/core/table/system/in_memory_system_table.cpp:
##########
@@ -104,19 +107,42 @@ class InMemorySystemTableRead : public TableRead {
} // namespace
-InMemorySystemTable::InMemorySystemTable(std::string table_path)
- : table_path_(std::move(table_path)) {}
+InMemorySystemTable::InMemorySystemTable(const std::shared_ptr<FileSystem>& fs,
+ std::string table_path, std::string
name,
+ std::optional<std::string> branch)
+ : SystemTable(fs, table_path, std::move(name)), branch_(std::move(branch))
{}
+
+Status InMemorySystemTable::CheckBranch(const std::map<std::string,
std::string>& options) const {
+ auto branch = options.find(Options::BRANCH);
+ if (branch_ && branch != options.end() &&
+ BranchManager::NormalizeBranch(branch->second) !=
+ BranchManager::NormalizeBranch(*branch_)) {
+ return Status::Invalid("context branch contradicts the loaded table");
+ }
+ return Status::OK();
+}
Result<std::unique_ptr<TableScan>> InMemorySystemTable::NewScan(
- const std::shared_ptr<ScanContext>& /*context*/) const {
- return std::make_unique<SystemTableScan>(table_path_);
+ std::unique_ptr<ScanContext> context) const {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan"));
+ PAIMON_RETURN_NOT_OK(CheckBranch(context->GetOptions()));
+ return std::make_unique<SystemTableScan>(GetRootPath());
}
Result<std::unique_ptr<TableRead>> InMemorySystemTable::NewRead(
- const std::shared_ptr<ReadContext>& context) const {
- return std::make_unique<InMemorySystemTableRead>(
- checked_pointer_cast<const InMemorySystemTable>(shared_from_this()),
- context->GetMemoryPool());
+ std::unique_ptr<ReadContext> ctx) const {
+ const std::shared_ptr<ReadContext> context = std::move(ctx);
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read"));
+ PAIMON_RETURN_NOT_OK(CheckNoSearch(context));
+ PAIMON_RETURN_NOT_OK(CheckBranch(context->GetOptions()));
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const InMemorySystemTable> table,
+ ForRead(context->GetOptions()));
Review Comment:
Load `$files` or `$manifests` before query auth is enabled, then enable it
on the base table. `NewRead()` on the existing handle still returns raw
statistics. It also accepts `query-auth.enabled=true` in the read options
without running the loader's check. An existing managed-table handle on a
non-main branch can likewise miss a newly enabled policy when no Catalog is
supplied. Java has the same stale-handle problem for managed tables and
`$files`. Java's `$manifests` also returns raw partition stats on a fresh load
because its loader allows that view on auth-enabled tables.
##########
src/paimon/core/table/system/system_table.cpp:
##########
@@ -261,27 +355,31 @@ Result<std::shared_ptr<SystemTable>>
SystemTableLoader::LoadFromPath(
// The warehouse is the grandparent of the sys/<table> path
context.warehouse =
PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path));
context.catalog_options = dynamic_options;
- // Note: context.catalog is intentionally left as nullptr here.
// Global tables loaded from path do not have a Catalog reference and
// cannot enumerate databases/tables. Only tables that don't require
// catalog enumeration (e.g. catalog_options) will work in this path.
return GlobalSystemTableLoader::Load(parsed.system_table_name,
context);
}
- PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch(
-
/*identifier=*/std::nullopt, dynamic_options,
- parsed.branch, "system
table read"));
+ const Identifier path_identifier(Identifier::kUnknownDatabase,
PathUtil::GetName(path));
+ PAIMON_ASSIGN_OR_RAISE(std::string branch,
+ BranchManager::ResolveBranch(path_identifier,
dynamic_options,
Review Comment:
For a static path such as `tbl$branch_MAIN$snapshots`, the context builder
sets `branch=MAIN`, then this loader resolves the same suffix to `main`. The
two values conflict, so a path that previously read an existing `branch-MAIN`
directory now fails. Java's filesystem catalog preserves `MAIN` and reads that
branch if it exists. Java's REST catalog sends a request for the bare table
name when resolving this system table. Neither Java path produces this branch
conflict.
##########
src/paimon/core/table/table_factory.cpp:
##########
@@ -0,0 +1,391 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "paimon/core/table/table_factory.h"
+
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+
+#include "fmt/format.h"
+#include "paimon/catalog/catalog.h"
+#include "paimon/catalog/table.h"
+#include "paimon/commit_context.h"
+#include "paimon/core/catalog/catalog_utils.h"
+#include "paimon/core/catalog/version_managed_catalog.h"
+#include "paimon/core/core_options.h"
+#include "paimon/core/realtime/realtime_utils.h"
+#include "paimon/core/schema/schema_manager.h"
+#include "paimon/core/schema/table_schema.h"
+#include "paimon/core/table/file_store_table.h"
+#include "paimon/core/table/format/format_table_loader.h"
+#include "paimon/core/table/system/system_table.h"
+#include "paimon/core/utils/branch_manager.h"
+#include "paimon/core/utils/context_utils.h"
+#include "paimon/file_store_commit.h"
+#include "paimon/file_store_write.h"
+#include "paimon/read_context.h"
+#include "paimon/scan_context.h"
+#include "paimon/table/format/format_table.h"
+#include "paimon/table/source/table_read.h"
+#include "paimon/table/source/table_scan.h"
+#include "paimon/write_context.h"
+
+namespace paimon {
+namespace {
+Result<std::shared_ptr<Table>> ManagedTable(
+ const std::string& path, const std::string& branch, const
std::shared_ptr<TableSchema>& schema,
+ CoreOptions options, const std::optional<Identifier>& identifier,
+ const std::optional<std::string>& table_id,
+ const std::map<std::string, std::string>& fs_scheme_map,
+ const std::shared_ptr<SchemaManager>& schema_manager = nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FileStoreTable> table,
+ FileStoreTable::Create(std::make_unique<FileStore>(path,
std::move(options)), schema,
+
identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)),
+ table_id.value_or(""), fs_scheme_map,
schema_manager));
+ return std::shared_ptr<Table>(std::move(table));
+}
+} // namespace
+
+Result<std::shared_ptr<Table>> TableFactory::Create(const WriteContext* ctx) {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write"));
+
+ // A table the caller already loaded says what it is, so nothing is read
to find out.
+ if (ctx->GetFormatTable() != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> given_table,
+ FormatTable::Copy(ctx->GetFormatTable(),
ctx->GetOptions()));
+ return std::shared_ptr<Table>(given_table);
+ }
+
+ PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog()));
+ std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem();
+ if (specific_fs == nullptr &&
ctx->GetFileSystemSchemeToIdentifierMap().empty() &&
+ ctx->GetCatalog() != nullptr) {
+ if (!ctx->GetIdentifier()) {
+ return Status::Invalid("a catalog write requires a table
identifier");
+ }
+ // A catalog issuing per-table temporary credentials only hands them
out through
+ // GetTableFileSystem, so the write uses the credentials of this table.
+ PAIMON_ASSIGN_OR_RAISE(specific_fs,
+
ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
+ CoreOptions::FromMap(ctx->GetOptions(), specific_fs,
+
ctx->GetFileSystemSchemeToIdentifierMap()));
+ std::string branch = ctx->GetBranch();
+ if (ctx->GetRealtimeContext()) {
+ PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(branch));
+ }
+ std::optional<Identifier> catalog_identifier;
+ if (ctx->GetIdentifier()) {
+ PAIMON_ASSIGN_OR_RAISE(
+ Identifier branch_identifier,
+ CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(),
branch));
+ catalog_identifier.emplace(std::move(branch_identifier));
+ }
+ // Only the main branch may have a current schema the catalog alone holds.
A branch publishes
+ // its own under `branch/branch-<name>`, which is where the load below and
a read of the branch
+ // both read it, so a catalog answering for no branch schema does not fail
a write to one.
+ std::optional<std::string> catalog_table_schema;
+ if (ctx->GetCatalog() != nullptr) {
+ if (!catalog_identifier) {
+ return Status::Invalid("a catalog write requires a table
identifier");
+ }
+ if (BranchManager::IsMainBranch(branch)) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> schema,
+
ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value()));
+ PAIMON_ASSIGN_OR_RAISE(catalog_table_schema,
schema->GetJsonSchema());
+ }
+ }
+ // A format table writes plain data files into a directory, so it never
reaches the manifest
+ // path below.
+ auto schema_manager =
+ std::make_shared<SchemaManager>(tmp_options.GetFileSystem(),
ctx->GetRootPath(), branch);
+ std::shared_ptr<TableSchema> latest_schema;
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FormatTable> format_table,
+ FormatTableLoader::TryLoad(tmp_options.GetFileSystem(),
ctx->GetRootPath(), branch,
+ ctx->GetOptions(), catalog_table_schema,
schema_manager.get(),
+ &latest_schema));
+ if (format_table != nullptr) {
+ return std::shared_ptr<Table>(format_table);
+ }
+ // The schema the dispatch above already read through `schema_manager`,
rather than a second
+ // read of the same file.
+ if (latest_schema == nullptr) {
+ return Status::Invalid(fmt::format("not found latest schema in branch
{}", branch));
+ }
+ const std::shared_ptr<TableSchema>& schema = latest_schema;
+ auto opts = schema->Options();
+ for (const auto& [key, value] : ctx->GetOptions()) {
+ opts[key] = value;
+ }
+ // The metadata paths above are built for this branch, so what reads
through these options,
+ // whose cache keys are scoped by it, has to read the same one.
+ opts[Options::BRANCH] = branch;
+ PAIMON_ASSIGN_OR_RAISE(
+ CoreOptions options,
+ CoreOptions::FromMap(opts, specific_fs,
ctx->GetFileSystemSchemeToIdentifierMap()));
+ return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options),
+ ctx->GetIdentifier(), std::nullopt,
+ ctx->GetFileSystemSchemeToIdentifierMap(),
schema_manager);
+}
+
+Result<std::shared_ptr<Table>> TableFactory::Create(const CommitContext* ctx) {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit"));
+
+ // A table the caller already loaded says what it is, so nothing is read
to find out.
+ if (ctx->GetFormatTable() != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> given_table,
+ FormatTable::Copy(ctx->GetFormatTable(),
ctx->GetOptions()));
+ return std::shared_ptr<Table>(given_table);
+ }
+
+ PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog()));
+ // Use catalog credentials unless the caller supplied a file system.
+ std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem();
+ if (specific_fs == nullptr && ctx->GetCatalog() != nullptr) {
+ if (!ctx->GetIdentifier()) {
+ return Status::Invalid("a catalog commit requires a table
identifier");
+ }
+ // Use the credentials of this table, which a catalog issuing
per-table temporary
+ // ones only hands out through GetTableFileSystem.
+ PAIMON_ASSIGN_OR_RAISE(specific_fs,
+
ctx->GetCatalog()->GetTableFileSystem(ctx->GetIdentifier().value()));
+ }
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options,
+ CoreOptions::FromMap(ctx->GetOptions(),
specific_fs));
+ const std::string& root_path = ctx->GetRootPath();
+ // The branch the context was built for, already validated by
`CommitContextBuilder`.
+ PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch(
+ ctx->GetIdentifier(),
ctx->GetOptions(),
+
/*explicit_branch=*/std::nullopt, "commit"));
+ std::optional<Identifier> catalog_identifier;
+ if (ctx->GetIdentifier()) {
+ PAIMON_ASSIGN_OR_RAISE(
+ Identifier branch_identifier,
+ CatalogUtils::BranchIdentifier(ctx->GetIdentifier().value(),
branch));
+ catalog_identifier.emplace(std::move(branch_identifier));
+ }
+ // Catalog-managed schemas may be absent from the table directory, but
only on the main branch:
+ // a branch publishes its own under `branch/branch-<name>`, which is where
the load below and a
+ // read of the branch both read it, so a catalog answering for no branch
schema does not fail
+ // a commit to one.
+ std::optional<std::string> catalog_table_schema;
+ if (ctx->GetCatalog() != nullptr) {
+ if (!catalog_identifier) {
+ return Status::Invalid("a catalog commit requires a table
identifier");
+ }
+ if (BranchManager::IsMainBranch(branch)) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> loaded_schema,
+
ctx->GetCatalog()->LoadTableSchema(catalog_identifier.value()));
+ PAIMON_ASSIGN_OR_RAISE(std::string schema_json,
loaded_schema->GetJsonSchema());
+ catalog_table_schema = std::move(schema_json);
+ }
+ }
+
+ // A format table commits by renaming files into place, so it never
reaches the snapshot path
+ // below. The managed path here reads the commit's branch, so this reads
the same one: the two
+ // must not dispatch on different schemas.
+ auto schema_manager =
+ std::make_shared<SchemaManager>(tmp_options.GetFileSystem(),
root_path, branch);
+ std::shared_ptr<TableSchema> latest_schema;
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> format_table,
+ FormatTableLoader::TryLoad(
+ tmp_options.GetFileSystem(), root_path, branch,
ctx->GetOptions(),
+ catalog_table_schema, schema_manager.get(),
&latest_schema));
+ if (format_table != nullptr) {
+ return std::shared_ptr<Table>(format_table);
+ }
+ if (latest_schema == nullptr) {
+ return Status::Invalid(fmt::format("not found latest schema in branch
{}", branch));
+ }
+ const std::shared_ptr<TableSchema>& schema = latest_schema;
+ auto opts = schema->Options();
+ for (const auto& [key, value] : ctx->GetOptions()) {
+ opts[key] = value;
+ }
+ // The metadata paths below are built for this branch, so what reads
through these options,
+ // whose cache keys are scoped by it, has to read the same one.
+ opts[Options::BRANCH] = branch;
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(opts,
specific_fs));
+ return ManagedTable(ctx->GetRootPath(), branch, schema, std::move(options),
+ ctx->GetIdentifier(), ctx->GetTableId(), {},
schema_manager);
+}
+
+Result<std::shared_ptr<Table>> TableFactory::Create(const ScanContext*
context) {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan"));
+
+ // A table the caller already loaded says what it is, so nothing is read
to find out.
+ if (context->GetFormatTable() != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> format_table,
+ FormatTable::Copy(context->GetFormatTable(),
context->GetOptions()));
+ return std::shared_ptr<Table>(format_table);
+ }
+ // load schema
+ PAIMON_ASSIGN_OR_RAISE(
+ CoreOptions tmp_options,
+ CoreOptions::FromMap(context->GetOptions(),
context->GetSpecificFileSystem(), {}));
+ if (context->GetRealtimeContext()) {
+
PAIMON_RETURN_NOT_OK(RealtimeUtils::ValidateBranch(tmp_options.GetBranch()));
+ }
+ std::map<std::string, std::string> read_options;
+ PAIMON_ASSIGN_OR_RAISE(
+ read_options,
+ CatalogUtils::ResolveQueryAuthReadOptions(
+ context->GetCatalog(), context->GetIdentifier(),
context->GetOptions(),
+ BranchManager::NormalizeBranch(tmp_options.GetBranch()),
tmp_options.GetFileSystem(),
+ context->GetPath(), context->GetSpecificTableSchema().has_value(),
"scan"));
+ PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path,
+
SystemTableLoader::TryParsePath(context->GetPath()));
+ if (system_table_path) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table,
+
SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(),
+
context->GetPath(), read_options));
+ return std::shared_ptr<Table>(system_table);
+ }
+ // A format table is planned by listing directories, so it never reaches
the manifest path
+ // below. One `TableScan` interface serves both, as Java Paimon serves
both through one
+ // `ReadBuilder`.
+ std::shared_ptr<TableSchema> latest_schema;
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<FormatTable> format_table,
+ FormatTableLoader::TryLoad(tmp_options.GetFileSystem(),
context->GetPath(),
+
BranchManager::NormalizeBranch(tmp_options.GetBranch()),
+ read_options,
context->GetSpecificTableSchema(),
+ /*schema_manager=*/nullptr,
&latest_schema));
+ if (format_table != nullptr) {
+ return std::shared_ptr<Table>(format_table);
+ }
+ if (!latest_schema) {
+ return Status::Invalid("not found latest schema");
+ }
+ auto options_map = latest_schema->Options();
+ for (const auto& [key, value] : read_options) {
+ options_map[key] = value;
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ std::string branch,
+ BranchManager::ResolveBranch(std::nullopt, options_map, std::nullopt,
"scan"));
+ if (options_map.count(Options::BRANCH) != 0) {
+ options_map[Options::BRANCH] = branch;
+ }
+ PAIMON_ASSIGN_OR_RAISE(CoreOptions options,
+ CoreOptions::FromMap(options_map,
context->GetSpecificFileSystem(), {}));
+ return ManagedTable(context->GetPath(), branch, latest_schema,
std::move(options),
+ context->GetIdentifier(), std::nullopt, {});
+}
+
+Result<std::shared_ptr<Table>> TableFactory::Create(const ReadContext*
context) {
+ PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "read"));
+
+ // A table the caller already loaded says what it is, so nothing is read
to find out.
+ if (context->GetFormatTable() != nullptr) {
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> format_table,
+ FormatTable::Copy(context->GetFormatTable(),
context->GetOptions()));
+ return std::shared_ptr<Table>(format_table);
+ }
+ PAIMON_ASSIGN_OR_RAISE(
+ CoreOptions tmp_core_options,
+ CoreOptions::FromMap(context->GetOptions(),
context->GetSpecificFileSystem(),
+ context->GetFileSystemSchemeToIdentifierMap()));
+ std::map<std::string, std::string> read_options;
+ PAIMON_ASSIGN_OR_RAISE(
+ read_options,
+ CatalogUtils::ResolveQueryAuthReadOptions(
+ context->GetCatalog(), context->GetIdentifier(),
context->GetOptions(),
+ context->GetBranch(), tmp_core_options.GetFileSystem(),
context->GetPath(),
+ context->GetSpecificTableSchema().has_value(), "read"));
+ PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path,
+
SystemTableLoader::TryParsePath(context->GetPath()));
+ if (system_table_path) {
+ if (context->EnablePreciseFileIndexEvaluate()) {
+ return Status::NotImplemented(
+ "Precise file-index evaluation is not supported for system
tables");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table,
+ SystemTableLoader::LoadFromPath(
Review Comment:
Static `$ro` scan and read used to forward `SetTableSchema()` to the
underlying operator. They now use the schema loaded from disk, so a projection
or filter that uses the supplied schema can fail after schema evolution. The
scan also lets context options override `__paimon.internal.read-optimized`, so
`false` can include lower-level PK files. Java has no per-operation
`SetTableSchema()` on `$ro` and applies the top-level filter directly. Both
regressions are specific to C++.
##########
src/paimon/core/table/format/format_table.cpp:
##########
@@ -192,20 +204,188 @@ Result<std::shared_ptr<FormatTable>> FormatTable::Copy(
for (const auto& [key, value] : dynamic_options) {
options[key] = value;
}
- return Create(table->file_system_, table->location_, table->identifier_,
table->schema_,
+ return Create(table->file_system_, table->root_path_, table->identifier_,
table->GetSchema(),
table->location_carries_paimon_metadata_, options);
}
const std::vector<std::string>& FormatTable::PartitionKeys() const {
- return schema_->PartitionKeys();
+ return GetSchema()->PartitionKeys();
}
std::string FormatTable::FullName() const {
return identifier_.GetFullName();
}
Result<std::unique_ptr<::ArrowSchema>> FormatTable::GetArrowSchema() const {
- return schema_->GetArrowSchema();
+ return GetSchema()->GetArrowSchema();
+}
+
+namespace {
+Result<std::unique_ptr<FileStoreWrite>> NewFormatTableWrite(
+ const std::shared_ptr<FormatTable>& table, const WriteContext& ctx) {
+ if (ctx.GetCatalog() != nullptr) {
+ return Status::Invalid(
+ "WithCatalog() requires a native table; use
WriteContextBuilder(FormatTable) for a "
+ "format table");
+ }
+ if (ctx.IsStreamingMode()) {
+ return Status::NotImplemented(
+ "a format table has no snapshots, so there is nothing a streaming
write could "
+ "commit against");
+ }
+ if (ctx.GetRealtimeContext() != nullptr) {
+ return Status::NotImplemented("a format table has no real-time store
to write into");
+ }
+ if (!ctx.GetWriteSchema().empty()) {
+ return Status::NotImplemented(
+ "a format table write takes the table's own columns; a write
schema naming a "
+ "subset of them is not supported yet");
+ }
+ // A write id prefixes a postpone-bucket writer's files so that one
compaction reader can
+ // put them back in order. A format table has no buckets, so it would
identify nothing.
+ if (ctx.GetWriteId().has_value()) {
+ return Status::NotImplemented(
+ "a format table has no buckets, so a write id would name nothing");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableFileStoreWrite>
format_write,
+ FormatTableFileStoreWrite::Create(table,
ctx.GetMemoryPool()));
+ return std::unique_ptr<FileStoreWrite>(std::move(format_write));
+}
+
+Result<std::unique_ptr<TableScan>> NewFormatTableScan(const
std::shared_ptr<FormatTable>& table,
+ const
std::shared_ptr<ScanContext>& context) {
+ // Anything a `ScanContext` carries that a format table cannot honour is
refused rather than
+ // silently dropped.
+ if (context->IsStreamingMode()) {
+ return Status::NotImplemented(
+ "a format table has no snapshots, so there is nothing for a
streaming scan to follow");
+ }
+ if (context->GetRealtimeContext() != nullptr) {
+ return Status::NotImplemented(
+ "a format table has no real-time store to union with what is on
disk");
+ }
+ if (context->GetGlobalIndexResult() != nullptr) {
+ return Status::NotImplemented("a format table carries no global
index");
+ }
+ std::map<std::string, std::string> partition_filter;
+ std::shared_ptr<ScanFilter> filters = context->GetScanFilters();
+ if (filters != nullptr) {
+ if (filters->GetPredicate() != nullptr) {
+ return Status::NotImplemented(
+ "a format table scan does not take a predicate: its files
carry no statistics to "
+ "skip by, and pruning by the partition columns a predicate
names is not "
+ "implemented yet; give the partition values as a partition
filter instead");
+ }
+ if (filters->GetBucketFilter()) {
+ return Status::NotImplemented("a format table has no buckets to
filter by");
+ }
+ const std::vector<std::map<std::string, std::string>>&
partition_filters =
+ filters->GetPartitionFilters();
+ if (partition_filters.size() > 1) {
+ return Status::NotImplemented("a format table scan takes at most
one partition filter");
+ }
+ if (partition_filters.size() == 1) {
+ partition_filter = partition_filters.front();
+ }
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableScan> scan,
+ FormatTableScan::Create(table, partition_filter,
context->GetLimit()));
+ return std::unique_ptr<TableScan>(std::move(scan));
+}
+
+Result<std::unique_ptr<TableRead>> NewFormatTableRead(const
std::shared_ptr<FormatTable>& table,
+ const
std::shared_ptr<ReadContext>& context) {
+ if (context->EnablePreciseFileIndexEvaluate()) {
+ return Status::NotImplemented(
+ "Precise file-index evaluation is not supported for format
tables");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableRead> read,
+ FormatTableRead::Create(table, context));
+ return std::unique_ptr<TableRead>(std::move(read));
+}
+
+Result<std::unique_ptr<FileStoreCommit>> NewFormatTableCommit(
+ const std::shared_ptr<FormatTable>& table, const CommitContext& ctx) {
+ if (!ctx.IgnoreEmptyCommit()) {
+ return Status::NotImplemented(
+ "a format table cannot record an empty commit: keeping one means
writing a "
+ "snapshot that adds no files, and there are no snapshots here");
+ }
+ if (ctx.UseRESTCatalogCommit()) {
+ return Status::NotImplemented(
+ "a format table commits by renaming files into place, not by
sending a snapshot "
+ "to a rest catalog");
+ }
+ if (ctx.GetTableId()) {
+ return Status::NotImplemented(
+ "a format table sends no commit table request, so there is nowhere
to put a table id");
+ }
+ if (ctx.GetCatalog() != nullptr) {
+ return Status::NotImplemented(
+ "a format table keeps no snapshot for a catalog to take, so it
commits by renaming "
+ "files into place rather than through the catalog it was loaded
from");
+ }
+ if (ctx.AppendCommitCheckConflict()) {
+ return Status::NotImplemented(
+ "a format table has no manifests to check a concurrent commit
against");
+ }
+ PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<FormatTableFileStoreCommit>
format_commit,
+ FormatTableFileStoreCommit::Create(table));
+ return std::unique_ptr<FileStoreCommit>(std::move(format_commit));
+}
+} // namespace
+Status FormatTable::CheckWriteContext(const std::string& path,
+ const std::shared_ptr<FormatTable>&
context_table) const {
+ PAIMON_ASSIGN_OR_RAISE(std::string context_path,
PathUtil::NormalizePath(path));
+ PAIMON_ASSIGN_OR_RAISE(std::string table_path,
PathUtil::NormalizePath(Location()));
+ if (context_path != table_path) {
+ return Status::Invalid("write/commit context path contradicts the
loaded format table");
+ }
+ if (context_table) {
+ const Identifier& other = context_table->identifier_;
+ const bool database_matches =
+ identifier_.GetDatabaseName() == Identifier::kUnknownDatabase ||
+ other.GetDatabaseName() == Identifier::kUnknownDatabase ||
+ identifier_.GetDatabaseName() == other.GetDatabaseName();
+ if (!database_matches || identifier_.GetObjectName() !=
other.GetObjectName()) {
Review Comment:
Two catalogs can return the same `db.tbl` and `s3://bucket/path` while
binding different storage endpoints. A typed context from B then passes this
check on A, and an empty `Overwrite()` can clear A's partition. Comparing raw
names also rejects handles for the same table loaded as `tbl` and
`tbl$branch_MAIN`. Java's standard `FormatBatchWriteBuilder` gets its write and
commit inputs from a single table, so this two-handle mismatch is specific to
C++.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]