lszskye commented on code in PR #447: URL: https://github.com/apache/paimon-cpp/pull/447#discussion_r4386019913
########## src/paimon/core/table/file_store_table_factory.cpp: ########## @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/file_store_table_factory.h" + +#include <map> +#include <memory> +#include <optional> +#include <string> +#include <utility> + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/catalog/table.h" +#include "paimon/commit_context.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/catalog/version_managed_catalog.h" +#include "paimon/core/core_options.h" +#include "paimon/core/options/table_type.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/file_store.h" +#include "paimon/core/table/file_store_table.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/fs/file_system.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/result.h" +#include "paimon/scan_context.h" +#include "paimon/schema/schema.h" +#include "paimon/status.h" +#include "paimon/table/format/format_table.h" +#include "paimon/write_context.h" + +namespace paimon { + +namespace { + +/// Recognises a format table behind `table_path`, or returns null when it is not one. The scan, +/// read, write and commit entry points below all dispatch on the same question, so one place +/// answers it and they cannot disagree. Null is also the answer when no schema was found under +/// the path at all: what to say about a table that is not there is the managed path's to decide. +/// +/// The table type is read from the schema, never from `options`: `type` is structural, and one +/// call's options must not decide what kind of table this is. +Result<std::shared_ptr<FormatTable>> TryLoadFormatTable( + const std::shared_ptr<FileSystem>& file_system, const std::string& table_path, + const std::string& branch, const std::map<std::string, std::string>& options, + const std::shared_ptr<TableSchema>& table_schema) { + if (table_schema == nullptr) { + return std::shared_ptr<FormatTable>(); + } + // From the schema alone: see `FormatTable::Create()`. + PAIMON_ASSIGN_OR_RAISE(bool is_format_table, + TableTypeDefine::IsFormatTable(table_schema->Options())); + if (!is_format_table) { + return std::shared_ptr<FormatTable>(); + } + // A format table has no branches, so it is refused rather than served from the table path + // under another branch's name. See `SchemaValidation::ValidateFormatTableSchema()`. + if (BranchManager::NormalizeBranch(branch) != BranchManager::DEFAULT_MAIN_BRANCH) { + return Status::Invalid( + fmt::format("a format table has no branches, so {} cannot be read on branch '{}'", + table_path, branch)); + } + Identifier identifier = CatalogUtils::IdentifierFromPath(table_path, branch); + return FormatTable::Create(file_system, table_path, identifier, + checked_pointer_cast<DataSchema>(table_schema), + /*location_carries_paimon_metadata=*/true, options); +} + +/// The options a context carries, merged into the table the load hands back: a table is the +/// product of the load, so the options speak there - the way `Copy` merges them - and `NewXxx` on +/// the product serves them without merging again. A caller holding a table already changes it +/// through `Copy` itself, and an option that contradicts the table that way is refused. +Result<std::shared_ptr<Table>> WithContextOptions( + const std::shared_ptr<Table>& table, const std::shared_ptr<TableSchema>& schema, + const std::map<std::string, std::string>& options) { + if (table == nullptr || schema == nullptr || options.empty()) { + return table; + } + const std::map<std::string, std::string>& schema_options = schema->Options(); + for (const auto& [key, value] : options) { + auto schema_option = schema_options.find(key); + // A key the schema does not carry, or carries differently: the merge has something to do. + if (schema_option == schema_options.end() || schema_option->second != value) { + return table->Copy(options); + } + } + return table; +} + +/// The tail every `Create` ends with: the schema one of them read says what the path names. A +/// format table is handed back as the `FormatTable` it is, everything else as the `FileStoreTable` +/// a managed table is, carrying the schema for its `NewXxx` to build from. A null schema is handed +/// over as it is - see `TryLoadFormatTable`. +/// +/// @param file_system The file system the schema above was read through, which a format table +/// is served by - it has no schema options of its own to name another one. +/// @param explicit_file_system The file system the context supplied, which a managed table is +/// opened through when there is one. A null one leaves the table to resolve from its +/// schema's own access configuration rather than being pinned to the file system this load +/// happened to read the schema through. +Result<std::shared_ptr<Table>> TableOfLoadedSchema( + const std::shared_ptr<FileSystem>& file_system, + const std::shared_ptr<FileSystem>& explicit_file_system, const std::string& path, + const std::string& branch, const std::map<std::string, std::string>& options, + const std::shared_ptr<SchemaManager>& schema_manager, + const std::shared_ptr<TableSchema>& latest_schema, const std::optional<Identifier>& identifier, + const std::string& uuid = "", const std::map<std::string, std::string>& fs_scheme_map = {}) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FormatTable> format_table, + TryLoadFormatTable(file_system, path, branch, options, latest_schema)); + if (format_table != nullptr) { + return std::shared_ptr<Table>(std::move(format_table)); + } + // The branch the schema was read on is folded into the object name. + Identifier table_identifier = + identifier.value_or(CatalogUtils::IdentifierFromPath(path, branch)); + // The manager was built over the file system the context's own options resolve, while the + // store resolves the schema's: a schema carrying access configuration names a backend the + // context's options never did, and a manager kept through that disagreement would read the + // table's metadata through the wrong one. The schema is in hand already, so dropping the + // manager costs its cache alone. A context that supplied a file system wins on both sides and + // keeps it; a scheme map only the store's resolution takes part in is the same disagreement. + std::shared_ptr<SchemaManager> manager = schema_manager; + if (manager != nullptr && latest_schema != nullptr && explicit_file_system == nullptr && + (!fs_scheme_map.empty() || + !FileStore::FileSystemResolutionAgrees(latest_schema->Options(), options))) { + manager = nullptr; + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<FileStoreTable> table, + FileStoreTable::Create(explicit_file_system, path, latest_schema, + table_identifier, uuid, branch, manager, + /*snapshot_manager=*/nullptr, fs_scheme_map)); + return WithContextOptions(table, latest_schema, options); Review Comment: The current sequence is: 1. Create a FileStoreTable using the original schema. 2. FileStore::Create immediately calls CoreOptions::FromMap with the schema options. 3. Only after that succeeds are the context options merged through WithContextOptions → Copy. The problem is that CoreOptions::FromMap does more than store configuration strings: it also loads file format plugins. For example, the schema’s file.format may refer to a plugin unavailable in the current process, while the context overrides it with an available format and the selected data files are otherwise readable: - Previously, the options were merged first, and the overridden format was then resolved. - Now, resolving the original format fails before the override has a chance to take effect. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
