zjw1111 commented on code in PR #447: URL: https://github.com/apache/paimon-cpp/pull/447#discussion_r4384907908
########## src/paimon/core/table/primary_key_file_store_table.cpp: ########## @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/primary_key_file_store_table.h" + +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "fmt/format.h" +#include "paimon/common/global_index/btree/btree_defs.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/deletionvectors/bucketed_dv_maintainer.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/bucketed_primary_key_index_maintainer.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/mergetree/compact/lookup_merge_function.h" +#include "paimon/core/mergetree/compact/merge_function.h" +#include "paimon/core/mergetree/compact/reducer_merge_function_wrapper.h" +#include "paimon/core/operation/commit/commit_scanner.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_scan.h" +#include "paimon/core/operation/key_value_file_store_write.h" +#include "paimon/core/options/merge_engine.h" +#include "paimon/core/postpone/postpone_bucket_file_store_write.h" +#include "paimon/core/realtime/realtime_schema_layout.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/bucket_mode.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/source/merge_tree_split_generator.h" +#include "paimon/core/table/source/split_generator.h" +#include "paimon/core/utils/primary_key_table_utils.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/write_context.h" + +namespace paimon { + +struct KeyValue; +template <typename T> +class MergeFunctionWrapper; + +namespace { + +/// Whether a schema older than `current_schema_id` carried a primary key b-tree index, which a +/// write has to keep maintaining: the index files an earlier schema wrote are still there to be +/// read, and dropping the definition now would leave them orphaned. +Result<bool> HasHistoricalPrimaryKeyBTreeDefinition( + const std::shared_ptr<SchemaManager>& schema_manager, int64_t current_schema_id) { + if (current_schema_id <= TableSchema::FIRST_SCHEMA_ID) { + return false; + } + PAIMON_ASSIGN_OR_RAISE(std::vector<int64_t> schema_ids, schema_manager->ListAllIds()); + for (int64_t schema_id : schema_ids) { + if (schema_id >= current_schema_id) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<TableSchema> historical_schema, + schema_manager->ReadSchema(schema_id)); + const auto& historical_options = historical_schema->Options(); + auto option = historical_options.find(Options::PK_BTREE_INDEX_COLUMNS); + if (option != historical_options.end() && + !StringUtils::IsNullOrWhitespaceOnly(option->second)) { + return true; + } + } + return false; +} + +} // namespace + +PrimaryKeyFileStoreTable::PrimaryKeyFileStoreTable(const std::shared_ptr<FileStore>& store, + const Identifier& identifier, + const std::string& uuid) + : FileStoreTable(store, identifier, uuid) {} + +Result<std::unique_ptr<FileStoreWrite>> PrimaryKeyFileStoreTable::CreateWrite( Review Comment: this function is more than 100 lines, do you need split to some small functions? ########## include/paimon/catalog/table.h: ########## @@ -33,50 +33,45 @@ namespace paimon { -class TableScan; +class CommitContext; +class FileStoreCommit; +class FileStoreWrite; +class ReadContext; +class ScanContext; class TableRead; +class TableScan; +class WriteContext; /// A table provides basic abstraction for table type. +/// +/// The state every kind of table carries - file system, root path, schema and identifier - lives +/// here, and each kind fills the operations in. The `NewXxx` operators below are the table-first +/// entry points: `TableScan::Create()` and friends load the table a context names and hand the +/// context to them. What a context asks for that the table cannot honour is refused by name rather +/// than silently dropped. class PAIMON_EXPORT Table { public: - /// @param file_system File system the table's data is accessed through. - /// @param root_path Root path of the table in the file system. - /// @param uuid Metastore UUID, or empty if unavailable. - Table(const std::shared_ptr<FileSystem>& file_system, const std::string& root_path, - const std::shared_ptr<Schema>& schema, const Identifier& identifier, - const std::string& uuid = ""); - - ~Table() = default; + virtual ~Table() = default; Review Comment: 1. modify examples/read_write_demo.cpp with the new table api 2. The API interfaces have changed significantly. Please check whether the code examples in the comments and documentation are consistent with the current API. ########## src/paimon/core/table/file_store.cpp: ########## @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store.h" + +#include <map> +#include <memory> +#include <optional> +#include <string> +#include <utility> +#include <vector> + +#include "paimon/catalog/catalog.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_list.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +namespace { + +/// Whether an option key takes part in file-system resolution: the backend `file-system` picks, +/// an `fs.` access key carries, or a native S3 key names - `s3.` and its `s3a.` spelling, the +/// prefixes the S3 file system consumes alongside `fs.s3.`/`fs.s3a.`. The scheme-map keys a +/// context carries are the same set. +bool IsFileSystemResolutionKey(const std::string& key) { + return key == Options::FILE_SYSTEM || key.rfind("fs.", 0) == 0 || key.rfind("s3.", 0) == 0 || + key.rfind("s3a.", 0) == 0; +} Review Comment: Is there any better solution here? There are more `fs` prefixes than just these; various file-system plugins each have their own prefixes. ########## src/paimon/core/table/file_store_table.cpp: ########## @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/file_store_table.h" + +#include <map> +#include <memory> +#include <optional> +#include <string> +#include <utility> + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/commit_context.h" +#include "paimon/common/predicate/predicate_validator.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/binary_row_partition_computer.h" +#include "paimon/common/utils/options_utils.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/snapshot_commit.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/disk/io_manager.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/operation/commit/realtime_commit_properties.h" +#include "paimon/core/operation/expire_snapshots.h" +#include "paimon/core/operation/file_store_commit_impl.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/realtime/realtime_context_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/append_only_file_store_table.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table_factory.h" +#include "paimon/core/table/primary_key_file_store_table.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/table/source/data_evolution_batch_scan.h" +#include "paimon/core/table/source/data_table_batch_scan.h" +#include "paimon/core/table/source/data_table_stream_scan.h" +#include "paimon/core/table/source/fallback_table_read.h" +#include "paimon/core/table/source/key_value_table_read.h" +#include "paimon/core/table/source/primary_key_index_batch_scan.h" +#include "paimon/core/table/source/read_optimized_scan_options.h" +#include "paimon/core/table/source/realtime_table_scan.h" +#include "paimon/core/table/source/snapshot/snapshot_reader.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/context_utils.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/file_store_commit.h" +#include "paimon/file_store_write.h" +#include "paimon/format/file_format.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/realtime/realtime_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" +#include "paimon/write_context.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +class Executor; +class MemoryPool; + +FileStoreTable::FileStoreTable(const std::shared_ptr<FileStore>& store, + const Identifier& identifier, const std::string& uuid) + : Table(store->GetFileSystem(), store->GetRootPath(), store->GetSchema(), identifier, uuid), + store_(store) {} + +Result<std::shared_ptr<FileStoreTable>> FileStoreTable::Create( + const std::shared_ptr<FileSystem>& file_system, const std::string& root_path, + const std::shared_ptr<Schema>& schema, const Identifier& identifier, const std::string& uuid, + const std::string& branch, const std::shared_ptr<SchemaManager>& schema_manager, + const std::shared_ptr<SnapshotManager>& snapshot_manager, + const std::map<std::string, std::string>& fs_scheme_map, + const std::map<std::string, std::string>& dynamic_options) { + // The kind the schema's primary keys name, which no schema change can add or drop, so the kind + // the table is built as is the kind every schema its root path ever held describes. A caller + // without a schema gets the append-only kind, whose operators report that none was found; a + // schema that is not a table schema - a system table's read-only one - is refused, not cast. + std::shared_ptr<TableSchema> table_schema; + if (schema != nullptr) { + table_schema = std::dynamic_pointer_cast<TableSchema>(schema); + if (table_schema == nullptr) { + return Status::Invalid( + fmt::format("{} is not a managed table schema", identifier.ToString())); + } + } + // The store parses the schema's options, so one that does not parse is refused here. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStore> store, + FileStore::Create(file_system, root_path, table_schema, branch, + schema_manager, snapshot_manager, fs_scheme_map)); + std::shared_ptr<FileStoreTable> table; + if (table_schema != nullptr && !table_schema->PrimaryKeys().empty()) { + table = std::make_shared<PrimaryKeyFileStoreTable>(store, identifier, uuid); + } else { + table = std::make_shared<AppendOnlyFileStoreTable>(store, identifier, uuid); + } + table->dynamic_options_ = dynamic_options; + return table; +} + +Result<std::shared_ptr<Table>> FileStoreTable::Copy( + const std::map<std::string, std::string>& dynamic_options) const { + std::shared_ptr<TableSchema> loaded_schema = std::static_pointer_cast<TableSchema>(schema_); + if (loaded_schema == nullptr) { + return Status::Invalid(fmt::format("not found latest schema of {}", FullName())); + } + // `type` is structural and read from the schema alone, the promise `Copy` makes: an option + // given at the call must not restyle the table. `branch` is the same promise - the copy serves + // the branch this table does, and a caller that means to move has `SwitchToBranch`. Both are + // dropped before the merge, so the copy's options never speak of a branch it does not serve. + std::map<std::string, std::string> overridable = dynamic_options; + overridable.erase(Options::TYPE); + overridable.erase(Options::BRANCH); + // The physical layout the data on the path was written by is as immovable as the style or the + // branch: the copy serves the same data. + PAIMON_RETURN_NOT_OK(SchemaValidation::CheckImmutableOptions(*loaded_schema, overridable)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> merged_schema, + loaded_schema->MergeOptions(overridable)); + // The managers are handed over, not rebuilt, when the copy resolves the same file system this + // table does: its operators then read the schema cache this table already warmed. The copy + // resolves from the file system the caller opened this table through, not the one it resolved + // to, so a file-system option it was given speaks; when one does change the resolution, fresh + // managers are built over it instead. The scheme map travels the same way. + const bool fs_resolution_may_change = + FileStore::FileSystemResolutionMayChange(*loaded_schema, overridable); + // The dynamic options the copy was given join the ones this table was loaded with - which a + // copy of a copy carries the same way - so a read's fallback reader serves what the copy's own + // reader does however many copies deep. The branch was dropped above, so none accumulates, and + // an option given again replaces the one carried, as the merge above does. + std::map<std::string, std::string> accumulated = dynamic_options_; + for (const auto& [key, value] : overridable) { + accumulated.insert_or_assign(key, value); + } + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, + std::shared_ptr<TableSchema>(std::move(merged_schema)), + identifier_, CatalogUuid().value_or(""), CurrentBranch(), + fs_resolution_may_change ? nullptr : GetSchemaManager(), + fs_resolution_may_change ? nullptr : GetSnapshotManager(), + store_->GetFsSchemeMap(), accumulated); +} + +Result<std::shared_ptr<Table>> FileStoreTable::SwitchToBranch(const std::string& branch) const { + const std::string target_branch = BranchManager::NormalizeBranch(branch); + if (target_branch == CurrentBranch()) { + // The table already serves this branch, so a copy of it is the same table to the caller. + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, schema_, + identifier_, CatalogUuid().value_or(""), CurrentBranch(), + GetSchemaManager(), GetSnapshotManager(), + store_->GetFsSchemeMap(), dynamic_options_); + } + // The schema of the branch switched to, read from its own directory. + auto schema_manager = std::make_shared<SchemaManager>(file_system_, root_path_, target_branch); + PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest, + schema_manager->Latest()); + if (!latest) { + return Status::Invalid(fmt::format("branch {} does not exist", target_branch)); + } + // A branch other than main is named in the options the schema carries - the way a read context + // names it - so what reads through them serves the branch the managers point at. + std::map<std::string, std::string> overridable; + if (target_branch != BranchManager::DEFAULT_MAIN_BRANCH) { + overridable[Options::BRANCH] = target_branch; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableSchema> branch_schema, + (*latest)->MergeOptions(overridable)); + // The branch's schema may name another file system than this table's options resolved - a + // recreation that changed the access configuration - and a manager kept through that + // disagreement would read the branch's metadata through the backend this table left, so the + // branch builds its own. The schema is in hand already, so only the cache is lost. + const std::shared_ptr<SchemaManager> branch_schema_manager = + (store_->GetSpecifiedFileSystem() == nullptr && + (!store_->GetFsSchemeMap().empty() || + !FileStore::FileSystemResolutionAgrees( + branch_schema->Options(), std::static_pointer_cast<TableSchema>(schema_)->Options()))) + ? nullptr + : schema_manager; + return FileStoreTable::Create(store_->GetSpecifiedFileSystem(), root_path_, + std::shared_ptr<TableSchema>(std::move(branch_schema)), + identifier_, CatalogUuid().value_or(""), target_branch, + branch_schema_manager, /*snapshot_manager=*/nullptr, + store_->GetFsSchemeMap()); +} + +Status FileStoreTable::RestoreRealtimeCommittedProgress( + const std::shared_ptr<RealtimeContext>& realtime_context, + const std::shared_ptr<SnapshotManager>& snapshot_manager, const CoreOptions& options) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl, + RealtimeContextImpl::Cast(realtime_context)); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + PAIMON_ASSIGN_OR_RAISE(std::optional<Snapshot> latest_snapshot, + snapshot_manager->LatestSnapshot()); + if (latest_snapshot) { + PAIMON_ASSIGN_OR_RAISE( + RealtimeOffsetMap realtime_committed_offsets, + RealtimeCommitProperties::ReadOffsets(latest_snapshot, options.GetFileSystem())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->AdvanceCommittedProgress( + latest_snapshot->Id(), realtime_committed_offsets, /*committed_data_files=*/{})); + } + return Status::OK(); +} + +Status FileStoreTable::CheckOptionsAgreeWithTable( + const TableSchema& table_schema, const std::map<std::string, std::string>& context_options) { + const std::map<std::string, std::string>& table_options = table_schema.Options(); + for (const auto& [key, value] : context_options) { + auto table_option = table_options.find(key); + if (table_option != table_options.end() && table_option->second != value) { + // A value may be a credential and the error travels into logs, so it is redacted. + return Status::Invalid( + fmt::format("option '{}' is '{}' in the context but '{}' on the table", key, + SensitiveConfigUtils::RedactValue(key, value), + SensitiveConfigUtils::RedactValue(key, table_option->second))); + } + } + return Status::OK(); +} + +std::unique_ptr<FileStoreScan> FileStoreTable::WithTablePath(std::unique_ptr<FileStoreScan>&& scan, + const std::string& root_path) { + scan->WithTablePath(root_path); + return std::move(scan); +} + +Result<std::unique_ptr<FileStoreWrite>> FileStoreTable::NewWrite( + std::unique_ptr<WriteContext> ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem(); + // A catalog write names its table; the file system comes from the table's own credentials. + if (specific_fs == nullptr && ctx->GetFileSystemSchemeToIdentifierMap().empty() && + ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // The write goes to the branch the table serves, not the context's own. + PAIMON_ASSIGN_OR_RAISE(std::optional<Identifier> catalog_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + const std::shared_ptr<TableSchema>& schema = store_->GetSchema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->GetOptions(); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateVectorStore(*schema, options)); + if (options.GetVectorFileFormat() && !ctx->GetWriteSchema().empty()) { + return Status::NotImplemented( + "Dedicated VECTOR storage does not yet support custom write schemas or partial writes"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<FileStorePathFactory> file_store_path_factory, + store_->NewPathFactory(arrow_schema, options, + options.GetWriteFileFormat(/*level=*/0)->Identifier(), + ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<SnapshotManager> snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); + std::shared_ptr<IOManager> io_manager; + const auto& io_temp_dir = ctx->GetTempDirectory(); + if (!io_temp_dir.empty()) { + io_manager = std::make_shared<IOManager>(io_temp_dir, options.GetFileSystem()); + } + + bool ignore_previous_files = ctx->IgnorePreviousFiles(); + if (ctx->GetRealtimeContext() && !options.RealtimeEnabled()) { + return Status::Invalid("real-time write requires realtime.enabled=true"); + } + if (ctx->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context_impl, + RealtimeContextImpl::Cast(ctx->GetRealtimeContext())); + PAIMON_RETURN_NOT_OK(realtime_context_impl->CheckUsable()); + } + return CreateWrite(*ctx, arrow_schema, partition_schema, options, file_store_path_factory, + snapshot_manager, io_manager, ignore_previous_files); +} + +Result<std::unique_ptr<FileStoreWrite>> FileStoreWrite::Create(std::unique_ptr<WriteContext> ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "write")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, FileStoreTableFactory::Create(*ctx)); + return table->NewWrite(std::move(ctx)); +} + +Result<std::unique_ptr<TableScan>> FileStoreTable::NewScan( + std::unique_ptr<ScanContext> context) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + const std::shared_ptr<TableSchema>& table_schema = store_->GetSchema(); + if (table_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*table_schema, context->GetOptions())); + PAIMON_ASSIGN_OR_RAISE(bool read_optimized, + OptionsUtils::GetValueFromMap<bool>(context->GetOptions(), + kReadOptimizedScanOption, false)); + // The branch in the store's options is the table's, so one the context carries is not honoured. + CoreOptions core_options = store_->GetOptions(); + // The cache is a runtime setting of this scan, not of the table. + core_options.WithCache(context->GetCache()); + + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + + PAIMON_RETURN_NOT_OK( + RealtimeTableScan::ValidateScan(*table_schema, core_options, *context, read_optimized)); + // validate options + if (core_options.GetBucket() == -1) { + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented(fmt::format( + "do not support pk table bucket={} in scan process", core_options.GetBucket())); + } + } else if (core_options.GetBucket() < 1 && + !SchemaValidation::IsPostponeBucketTable(*table_schema, core_options.GetBucket())) { + return Status::Invalid( + fmt::format("do not support bucket={} in scan process", core_options.GetBucket())); + } + + // validate schema and scan filter + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + if (context->GetScanFilters() && context->GetScanFilters()->GetPredicate()) { + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + *arrow_schema, context->GetScanFilters()->GetPredicate(), + /*validate_field_idx=*/false)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStorePathFactory> path_factory, + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<ManifestList> manifest_list, + store_->NewManifestList(core_options, path_factory, context->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<arrow::Schema> partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ManifestFile> manifest_file, + store_->NewManifestFile(core_options, path_factory, partition_schema, + context->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<FileStoreScan> file_store_scan, + NewFileStoreScan(manifest_list, manifest_file, arrow_schema, core_options, + context->GetExecutor(), context->GetMemoryPool(), context.get())); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<SplitGenerator> split_generator, + NewSplitGenerator(core_options)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr<IndexFileHandler> index_file_handler, + store_->NewIndexFileHandler(core_options, path_factory, context->GetMemoryPool())); + auto snapshot_reader = std::make_shared<SnapshotReader>( + file_store_scan, path_factory, std::move(split_generator), std::move(index_file_handler)); + const bool pk_table = !table_schema->PrimaryKeys().empty(); + if (read_optimized && pk_table && context->IsStreamingMode()) { + return Status::NotImplemented( + "read-optimized system table does not support streaming scan for primary key table"); + } + if (context->IsStreamingMode()) { + return std::make_unique<DataTableStreamScan>(core_options, snapshot_reader); + } + auto batch_scan = std::make_unique<DataTableBatchScan>( + /*pk_table=*/pk_table, core_options, snapshot_reader, read_optimized, context->GetLimit(), + /*realtime_pk_scan=*/pk_table && context->GetRealtimeContext() != nullptr); + if (context->GetRealtimeContext()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<RealtimeContextImpl> realtime_context, + RealtimeContextImpl::Cast(context->GetRealtimeContext())); + return std::make_unique<RealtimeTableScan>( + std::move(batch_scan), pk_table, realtime_context, path_factory, + snapshot_reader->GetSnapshotManager(), core_options.GetFileSystem(), + context->GetScanFilters(), core_options.GetRealtimeReadViewTtlMillis()); + } + if (core_options.DataEvolutionEnabled()) { + return std::make_unique<DataEvolutionBatchScan>( + root_path_, snapshot_reader, std::move(batch_scan), table_schema, + context->GetGlobalIndexResult(), core_options, context->GetMemoryPool(), + context->GetExecutor()); + } + if (pk_table && !read_optimized && core_options.GlobalIndexEnabled()) { + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(*table_schema)); + if (!definitions.ScalarDefinitions().empty()) { + return PrimaryKeyIndexBatchScan::Create(snapshot_reader, std::move(batch_scan), + table_schema, path_factory, core_options, + context->GetMemoryPool()); + } + } + return batch_scan; +} + +Result<std::unique_ptr<TableScan>> TableScan::Create(std::unique_ptr<ScanContext> context) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(context, "scan")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, FileStoreTableFactory::Create(*context)); + return table->NewScan(std::move(context)); +} + +Result<std::unique_ptr<TableRead>> FileStoreTable::CreateTableRead( + const std::shared_ptr<InternalReadContext>& internal_context, + const std::shared_ptr<MemoryPool>& memory_pool, + const std::shared_ptr<Executor>& executor) const { + if (PredicateUtils::ContainsFullTextMatch(internal_context->GetPredicate())) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + if (!internal_context->GetIndexScoreBindings().empty()) { + return Status::NotImplemented("Per-field index score output is not implemented yet"); + } + const auto& core_options = internal_context->GetCoreOptions(); + const auto& table_schema = internal_context->GetTableSchema(); + if (internal_context->GetRealtimeContext() && !core_options.RealtimeEnabled()) { + return Status::Invalid("real-time read requires realtime.enabled=true"); + } + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*table_schema, core_options)); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<FileStorePathFactory> path_factory, + store_->NewPathFactory(arrow_schema, core_options, + core_options.GetFileFormat()->Identifier(), memory_pool)); + // The reader is selected by the schema the read serves, not by the table's kind: an + // append-only table can have a primary-key fallback branch and vice versa, so a reader typed + // after the main table would fail the fallback's splits. + if (internal_context->GetPrimaryKeys().empty()) { + return std::make_unique<AppendOnlyTableRead>(path_factory, internal_context, memory_pool, + executor); + } + return KeyValueTableRead::Create(path_factory, internal_context, memory_pool, executor); +} + +Result<std::unique_ptr<TableRead>> FileStoreTable::NewRead( + std::unique_ptr<ReadContext> context) const { + std::shared_ptr<ReadContext> read_context = std::move(context); + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(read_context, "read")); + const std::shared_ptr<TableSchema>& loaded_schema = store_->GetSchema(); + if (loaded_schema == nullptr) { + return Status::Invalid("not found latest schema"); + } + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*loaded_schema, read_context->GetOptions())); + // The read reads through the branch the table serves, not the context's own. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> internal_context, + InternalReadContext::Create(read_context, loaded_schema, + store_->GetOptions(), &root_path_)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> table_read, + CreateTableRead(internal_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + + std::optional<std::string> scan_fallback_branch = + internal_context->GetCoreOptions().GetScanFallbackBranch(); + if (!scan_fallback_branch || + StringUtils::IsNullOrWhitespaceOnly(scan_fallback_branch.value())) { + return std::move(table_read); + } + + // The fallback branch is not the table's, so its schema is read through its own manager. + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<InternalReadContext> fallback_context, + InternalReadContext::CreateForFallbackBranch( + read_context, /*branch=*/scan_fallback_branch.value(), file_system_, + root_path_, dynamic_options_)); + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<TableRead> fallback_table_read, + CreateTableRead(fallback_context, read_context->GetMemoryPool(), + read_context->GetExecutor())); + const std::shared_ptr<arrow::MemoryPool>& arrow_pool = internal_context->GetArrowMemoryPool(); + return std::make_unique<FallbackTableRead>(std::move(table_read), + std::move(fallback_table_read), arrow_pool); +} + +Result<std::unique_ptr<TableRead>> TableRead::Create(std::unique_ptr<ReadContext> ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "read")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, FileStoreTableFactory::Create(*ctx)); + return table->NewRead(std::move(ctx)); +} + +Result<std::unique_ptr<FileStoreCommit>> FileStoreTable::NewCommit( + std::unique_ptr<CommitContext> ctx) const { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx->GetCatalog())); + std::shared_ptr<FileSystem> specific_fs = ctx->GetSpecificFileSystem(); + // A catalog commit names its table; the file system comes from the table's own credentials. + if (specific_fs == nullptr && ctx->GetCatalog() != nullptr && !ctx->GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + // A commit commits to the root path and branch the table was loaded with, not the context's. + PAIMON_ASSIGN_OR_RAISE(std::optional<Identifier> catalog_identifier, + CatalogUtils::BranchIdentifier(ctx->GetIdentifier(), CurrentBranch())); + // The id a snapshot records the catalog's current schema by, read through the loader below + // rather than trusted from a schema this commit will not re-read. + FileStoreCommitImpl::SchemaIdLoader schema_id_loader; + if (ctx->GetCatalog() != nullptr) { + if (!catalog_identifier) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + std::shared_ptr<Catalog> catalog = ctx->GetCatalog(); + Identifier identifier = catalog_identifier.value(); + schema_id_loader = [catalog, identifier]() -> Result<int64_t> { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> schema, + catalog->LoadTableSchema(identifier)); + std::shared_ptr<DataSchema> data_schema = std::dynamic_pointer_cast<DataSchema>(schema); + if (data_schema == nullptr) { + return Status::Invalid( + "the catalog did not hand back a data table schema, which has the id a " + "snapshot records"); + } + return data_schema->Id(); + }; + } + const std::shared_ptr<TableSchema>& schema = store_->GetSchema(); + if (schema == nullptr) { + return Status::Invalid( + fmt::format("not found latest schema in branch {}", CurrentBranch())); + } + std::shared_ptr<arrow::Schema> arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(schema->Fields()); + PAIMON_RETURN_NOT_OK(CheckOptionsAgreeWithTable(*schema, ctx->GetOptions())); + CoreOptions options = store_->GetOptions(); + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateRealtime(*schema, options)); + assert(options.GetFileSystem()); + assert(options.GetFileFormat()); + PAIMON_RETURN_NOT_OK(FileStoreCommitImpl::ValidateCommitOptions(options)); + PAIMON_ASSIGN_OR_RAISE(bool is_object_store, FileSystem::IsObjectStore(root_path_)); + // Catalog commits do not require atomic rename on the object store. The inte-test marker is + // not a table option, so it is read from the context the commit was handed. + if (is_object_store && !ctx->UseRESTCatalogCommit() && + AsVersionManaged(ctx->GetCatalog()) == nullptr && + ctx->GetOptions().find("enable-object-store-commit-in-inte-test") == + ctx->GetOptions().end()) { + return Status::NotImplemented( + "commit operation does not support object store file system for now"); + } + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr<BinaryRowPartitionComputer> partition_computer, + store_->NewPartitionComputer(arrow_schema, options, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<FileStorePathFactory> path_factory, + store_->NewPathFactory(arrow_schema, options, options.GetFileFormat()->Identifier(), + ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<SnapshotManager> snapshot_manager, + store_->NewSnapshotManager(options, ctx->GetCatalog(), catalog_identifier)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<ManifestList> manifest_list, + store_->NewManifestList(options, path_factory, ctx->GetMemoryPool())); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Schema> partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<ManifestFile> manifest_file, + store_->NewManifestFile(options, path_factory, partition_schema, ctx->GetMemoryPool())); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr<IndexManifestFile> index_manifest_file, + store_->NewIndexManifestFile(options, path_factory, ctx->GetMemoryPool())); + + auto expire_snapshots = std::make_shared<ExpireSnapshots>( + snapshot_manager, path_factory, manifest_list, manifest_file, index_manifest_file, + options.GetFileSystem(), options.GetExpireConfig(), options.RealtimeEnabled(), + ctx->GetExecutor()); + + CommitScanner::ScanSupplier scan_supplier = + NewCommitScanSupplier(snapshot_manager, manifest_list, manifest_file, arrow_schema, options, + ctx->GetExecutor(), ctx->GetMemoryPool()); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SnapshotCommit> snapshot_commit, + SnapshotCommit::Create(ctx->GetCatalog(), catalog_identifier, + ctx->GetTableId(), ctx->UseRESTCatalogCommit(), + options.GetFileSystem(), snapshot_manager)); + + return std::make_unique<FileStoreCommitImpl>( + ctx->GetMemoryPool(), ctx->GetExecutor(), arrow_schema, root_path_, ctx->GetCommitUser(), + options, path_factory, std::move(partition_computer), snapshot_manager, snapshot_commit, + ctx->IgnoreEmptyCommit(), ctx->AppendCommitCheckConflict(), schema, + std::move(schema_id_loader), manifest_file, manifest_list, index_manifest_file, + expire_snapshots, GetSchemaManager(), std::move(scan_supplier)); +} + +Result<std::unique_ptr<FileStoreCommit>> FileStoreCommit::Create( + std::unique_ptr<CommitContext> ctx) { + PAIMON_RETURN_NOT_OK(ContextUtils::CheckOperatorContext(ctx, "commit")); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Table> table, FileStoreTableFactory::Create(*ctx)); + return table->NewCommit(std::move(ctx)); +} Review Comment: the `XxxOp::Create` impl is not in xxx_op.cpp ########## src/paimon/core/table/file_store_table_factory.cpp: ########## @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include <map> +#include <memory> +#include <optional> +#include <string> +#include <utility> + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/format/format_table.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, +/// read, write and commit entry points below all dispatch on the same question, so one place +/// answers it and they cannot disagree. Null is also the answer when no schema was found under +/// the path at all: what to say about a table that is not there is the managed path's to decide. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. +Result<std::shared_ptr<FormatTable>> TryLoadFormatTable( + const std::shared_ptr<FileSystem>& file_system, const std::string& table_path, + const std::string& branch, const std::map<std::string, std::string>& options, + const std::shared_ptr<TableSchema>& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr<FormatTable>(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr<FormatTable>(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast<DataSchema>(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on +/// the product serves them without merging again. A caller holding a table already changes it +/// through `Copy` itself, and an option that contradicts the table that way is refused. +Result<std::shared_ptr<Table>> WithContextOptions( + const std::shared_ptr<Table>& table, const std::shared_ptr<TableSchema>& schema, + const std::map<std::string, std::string>& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map<std::string, std::string>& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to do. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + +/// The tail every `Create` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed +/// over as it is - see `TryLoadFormatTable`. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from its +/// schema's own access configuration rather than being pinned to the file system this load +/// happened to read the schema through. +Result<std::shared_ptr<Table>> TableOfLoadedSchema( + const std::shared_ptr<FileSystem>& file_system, + const std::shared_ptr<FileSystem>& explicit_file_system, const std::string& path, + const std::string& branch, const std::map<std::string, std::string>& options, + const std::shared_ptr<SchemaManager>& schema_manager, + const std::shared_ptr<TableSchema>& latest_schema, const std::optional<Identifier>& identifier, + const std::string& uuid = "", const std::map<std::string, std::string>& fs_scheme_map = {}) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr<Table>(std::move(format_table)); + } + // The branch the schema was read on is folded into the object name. + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // The manager was built over the file system the context's own options resolve, while the + // store resolves the schema's: a schema carrying access configuration names a backend the + // context's options never did, and a manager kept through that disagreement would read the + // table's metadata through the wrong one. The schema is in hand already, so dropping the + // manager costs its cache alone. A context that supplied a file system wins on both sides and + // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. + std::shared_ptr<SchemaManager> manager = schema_manager; + if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && + (!fs_scheme_map.empty() || + !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { + manager = nullptr; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStoreTable> table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); + return WithContextOptions(table, latest_schema, options); +} + +/// The managed table a context names behind a path: the schema is read through a manager of the +/// context's branch - handed over instead when the caller already read it - and what the schema +/// says is there decides the table `TableOfLoadedSchema` hands back. +/// +/// @param schema_json The schema of the table, when the caller already read it, or null when it +/// has to be read here. It is honoured on the main branch only; a branch publishes its own +/// under `branch/branch-<name>`, which is where the read below goes. +/// @param explicit_file_system The file system the context supplied, which the managed table the +/// schema describes is opened through - see `TableOfLoadedSchema`. +Result<std::shared_ptr<Table>> ManagedTableOf( + const std::shared_ptr<FileSystem>& file_system, + const std::shared_ptr<FileSystem>& explicit_file_system, const std::string& path, + const std::string& branch, const std::map<std::string, std::string>& options, + const std::optional<std::string>& schema_json, const std::optional<Identifier>& identifier, + const std::string& uuid = "", const std::map<std::string, std::string>& fs_scheme_map = {}) { + auto schema_manager = std::make_shared<SchemaManager>(file_system, path, branch); + std::shared_ptr<TableSchema> latest_schema; + if (branch == BranchManager::DEFAULT_MAIN_BRANCH && schema_json) { + // Handing the schema over only saves reading it. A context always names a table path and + // this library writes the metadata under it, so `schema` and `branch` stay metadata. + PAIMON_ASSIGN_OR_RAISE(latest_schema, TableSchema::CreateFromJson(schema_json.value())); + } else { + PAIMON_ASSIGN_OR_RAISE(std::optional<std::shared_ptr<TableSchema>> latest, + schema_manager->Latest()); + if (latest) { + latest_schema = *latest; + } + } + return TableOfLoadedSchema(file_system, explicit_file_system, path, branch, options, + schema_manager, latest_schema, identifier, uuid, fs_scheme_map); +} + +/// The schema a catalog may hold for the table, read from it on the main branch only and null +/// otherwise: a branch publishes its own under `branch/branch-<name>`, which is where +/// `ManagedTableOf` reads instead, so a catalog answering for no branch schema does not fail the +/// `kind` operation to one. `kind` also names the operator the messages speak of. +Result<std::optional<std::string>> CatalogMainBranchSchemaJson( + const std::shared_ptr<Catalog>& catalog, const std::optional<Identifier>& catalog_identifier, + const std::string& branch, const char* kind) { + if (catalog == nullptr) { + return std::optional<std::string>(); + } + if (!catalog_identifier) { + return Status::Invalid(fmt::format("a catalog {} requires a table identifier", kind)); + } + if (BranchManager::IsMainBranch(branch)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Schema> schema, + catalog->LoadTableSchema(catalog_identifier.value())); + PAIMON_ASSIGN_OR_RAISE(std::string schema_json, schema->GetJsonSchema()); + return std::optional<std::string>(std::move(schema_json)); + } + return std::optional<std::string>(); +} + +} // namespace + +/// The table a scan context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const ScanContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr<Table>(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), {})); + PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table, + SystemTableLoader::LoadFromPath(tmp_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr<Table>(std::move(system_table)); + } + // One `TableScan` interface serves a format table too, the way Java Paimon serves both + // through one `ReadBuilder`; a format table is planned by listing directories instead. + const std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + return ManagedTableOf(tmp_options.GetFileSystem(), ctx.GetSpecificFileSystem(), ctx.GetPath(), + branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier()); +} + +/// The table a read context names: the format table the builder was handed, a system table behind +/// a `T$system` path, or the table the schema under the context's path says is there. +Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const ReadContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr<Table>(ctx.GetFormatTable()); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_core_options, + CoreOptions::FromMap(ctx.GetOptions(), ctx.GetSpecificFileSystem(), + ctx.GetFileSystemSchemeToIdentifierMap())); + PAIMON_ASSIGN_OR_RAISE(std::optional<SystemTablePath> system_table_path, + SystemTableLoader::TryParsePath(ctx.GetPath())); + if (system_table_path) { + if (PredicateUtils::ContainsFullTextMatch(ctx.GetPredicate())) { + return Status::NotImplemented( + "FullTextMatch predicate execution is not implemented yet"); + } + if (ctx.HasVectorOrFullTextSearch()) { + return Status::NotImplemented( + "VectorSearch or FullTextSearch is not supported for system tables"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<SystemTable> system_table, + SystemTableLoader::LoadFromPath(tmp_core_options.GetFileSystem(), + ctx.GetPath(), ctx.GetOptions())); + return std::shared_ptr<Table>(std::move(system_table)); + } + + // A format table has no manifest and a file another engine wrote carries no field ids, so how + // a row is put back together is its own; how a file is opened is not - both paths go through + // `DataFileReaderFactory`. + const std::string branch = ctx.GetBranch(); + // The scheme map the schema was read through, so the table's data resolves custom schemes the + // same way its metadata does. + return ManagedTableOf(tmp_core_options.GetFileSystem(), ctx.GetSpecificFileSystem(), + ctx.GetPath(), branch, ctx.GetOptions(), ctx.GetSpecificTableSchema(), + ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a write context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const WriteContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr<Table>(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + std::shared_ptr<FileSystem> specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetFileSystemSchemeToIdentifierMap().empty() && + ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog write requires a table identifier"); + } + // A catalog issuing per-table temporary credentials only hands them out through + // GetTableFileSystem, so the write uses this table's. + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs, + ctx.GetFileSystemSchemeToIdentifierMap())); + std::string branch = ctx.GetBranch(); + PAIMON_ASSIGN_OR_RAISE(std::optional<Identifier> catalog_identifier, + CatalogUtils::BranchIdentifier(ctx.GetIdentifier(), branch)); + // Only the main branch may have a current schema the catalog alone holds: a branch publishes + // its own under `branch/branch-<name>`, which is where the load below reads it instead. + PAIMON_ASSIGN_OR_RAISE( + std::optional<std::string> catalog_table_schema, + CatalogMainBranchSchemaJson(ctx.GetCatalog(), catalog_identifier, branch, "write")); + return ManagedTableOf(tmp_options.GetFileSystem(), specific_fs, ctx.GetRootPath(), branch, + ctx.GetOptions(), catalog_table_schema, ctx.GetIdentifier(), /*uuid=*/"", + ctx.GetFileSystemSchemeToIdentifierMap()); +} + +/// The table a commit context names: the format table the builder was handed, or the table the +/// schema under the context's root path says is there. +Result<std::shared_ptr<Table>> FileStoreTableFactory::Create(const CommitContext& ctx) { + if (ctx.GetFormatTable() != nullptr) { + return std::shared_ptr<Table>(ctx.GetFormatTable()); + } + PAIMON_RETURN_NOT_OK(CheckVersionManagementImplemented(ctx.GetCatalog())); + // Use this table's catalog credentials unless the caller supplied a file system: a catalog + // issuing per-table temporary ones only hands them out through GetTableFileSystem. + std::shared_ptr<FileSystem> specific_fs = ctx.GetSpecificFileSystem(); + if (specific_fs == nullptr && ctx.GetCatalog() != nullptr) { + if (!ctx.GetIdentifier()) { + return Status::Invalid("a catalog commit requires a table identifier"); + } + PAIMON_ASSIGN_OR_RAISE(specific_fs, + ctx.GetCatalog()->GetTableFileSystem(ctx.GetIdentifier().value())); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, + CoreOptions::FromMap(ctx.GetOptions(), specific_fs)); + const std::string& root_path = ctx.GetRootPath(); + // The branch the context was built for, already validated by `CommitContextBuilder`. + PAIMON_ASSIGN_OR_RAISE(std::string branch, BranchManager::ResolveBranch( Review Comment: why different with WriteContext's Create? One use `std::string branch = ctx.GetBranch();`, the other one use `BranchManager::ResolveBranch` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
