lxy-9602 commented on code in PR #545:
URL: https://github.com/apache/paimon-cpp/pull/545#discussion_r4533623241
##########
src/paimon/common/fs/external_path_provider_test.cpp:
##########
@@ -78,7 +81,8 @@ TEST(ExternalPathProviderTest,
TestGetNextExternalDataPathConcurrently) {
std::string relative_bucket_path = "p0=1/p1=0/bucket-0";
ASSERT_OK_AND_ASSIGN(std::unique_ptr<ExternalPathProvider> provider,
- ExternalPathProvider::Create(external_table_paths,
relative_bucket_path));
+ ExternalPathProvider::Create(external_table_paths,
relative_bucket_path,
+ /*weights=*/{}));
const std::set<std::string> expected_data_paths = {
Review Comment:
Please add a unit test for `ExternalPathProvider` covering a scenario with
weights.
##########
src/paimon/core/global_index/global_index_file_manager.h:
##########
@@ -154,21 +151,21 @@ class GlobalIndexFileManager : public
GlobalIndexFileReader,
}
Result<std::vector<CheckpointFile>> ListCheckpointFiles() const {
- std::vector<BasicFileStatus> file_statuses;
- PAIMON_RETURN_NOT_OK(
- fs_->ListDir(checkpoint_path_factory_->GetDirectoryPath(),
&file_statuses));
std::vector<CheckpointFile> checkpoint_files;
- for (const BasicFileStatus& file_status : file_statuses) {
- if (file_status.IsDir()) {
- continue;
- }
- std::string file_name = PathUtil::GetName(file_status.GetPath());
- std::optional<int64_t> id =
checkpoint_path_factory_->GetCheckpointId(file_name);
- if (!id) {
- continue;
+ for (const auto& directory :
checkpoint_path_factory_->GetDirectoryPaths()) {
+ std::vector<BasicFileStatus> file_statuses;
+ PAIMON_RETURN_NOT_OK(fs_->ListDir(directory, &file_statuses));
+ for (const BasicFileStatus& file_status : file_statuses) {
+ if (file_status.IsDir()) {
+ continue;
+ }
+ std::string file_name =
PathUtil::GetName(file_status.GetPath());
+ std::optional<int64_t> id =
checkpoint_path_factory_->GetCheckpointId(file_name);
+ if (id) {
+ checkpoint_files.push_back(
+ CheckpointFile{id.value(),
PathUtil::JoinPath(directory, file_name)});
Review Comment:
Is `PathUtil::JoinPath(directory, file_name)` the same as
`file_status.GetPath()`?
##########
include/paimon/defs.h:
##########
@@ -495,8 +495,15 @@ struct PAIMON_EXPORT Options {
/// written, multiple elements separated by commas.
static const char DATA_FILE_EXTERNAL_PATHS[];
/// "data-file.external-paths.strategy" - The strategy of selecting an
external path when
- /// writing data. Values can be: "none", "specific-fs", "round-robin".
Default value is "none".
+ /// writing data. Values: "none", "specific-fs", "round-robin",
"weighted". Default is "none".
static const char DATA_FILE_EXTERNAL_PATHS_STRATEGY[];
Review Comment:
The Java weighting strategy value is `weight-robin`, not `weighted`.
##########
src/paimon/core/global_index/global_index_write_task.cpp:
##########
@@ -87,13 +87,16 @@ Result<std::shared_ptr<FileStorePathFactory>>
CreateFileStorePathFactory(
auto all_arrow_schema =
DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields());
PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> external_paths,
core_options.CreateExternalPaths());
- PAIMON_ASSIGN_OR_RAISE(std::optional<std::string>
global_index_external_path,
+ PAIMON_ASSIGN_OR_RAISE(std::vector<int32_t> external_path_weights,
+ core_options.GetExternalPathWeights());
+ PAIMON_ASSIGN_OR_RAISE(std::vector<std::string> global_index_external_path,
core_options.CreateGlobalIndexExternalPath());
return FileStorePathFactory::Create(
table_path, all_arrow_schema, table_schema->PartitionKeys(),
core_options.GetPartitionDefaultName(),
core_options.GetFileFormat()->Identifier(),
core_options.DataFilePrefix(),
core_options.LegacyPartitionNameEnabled(), external_paths,
- global_index_external_path, core_options.IndexFileInDataFileDir(),
pool);
+ global_index_external_path, core_options.IndexFileInDataFileDir(),
+ /*external_path_weights=*/external_path_weights, pool);
Review Comment:
This seems like it doesn’t need a comment—the variable name
`external_path_weights` is pretty clear.
##########
src/paimon/core/core_options.cpp:
##########
@@ -1940,18 +1965,24 @@ std::optional<std::string>
CoreOptions::GetGlobalIndexExternalPath() const {
return impl_->global_index_external_path;
}
-Result<std::optional<std::string>>
CoreOptions::CreateGlobalIndexExternalPath() const {
+Result<std::vector<std::string>> CoreOptions::CreateGlobalIndexExternalPath()
const {
+ std::vector<std::string> external_paths;
std::optional<std::string> global_index_external_path =
GetGlobalIndexExternalPath();
- if (global_index_external_path == std::nullopt ||
global_index_external_path->empty()) {
- return std::optional<std::string>();
+ if (!global_index_external_path || global_index_external_path->empty()) {
+ return external_paths;
Review Comment:
Rename the parameter and the function to
`global_index_external_paths/CreateGlobalIndexExternalPaths`.
##########
src/paimon/core/utils/file_store_path_factory.h:
##########
@@ -102,8 +102,8 @@ class FileStorePathFactory : public
std::enable_shared_from_this<FileStorePathFa
return external_paths_;
}
- const std::optional<std::string>& GetGlobalIndexExternalPath() const {
- return global_index_external_path_;
+ const std::vector<std::string>& GetGlobalIndexExternalPath() const {
+ return global_index_external_paths_;
}
Review Comment:
Consider use `GetGlobalIndexExternalPaths`
##########
src/paimon/core/index/index_path_factory.h:
##########
@@ -31,6 +31,9 @@ class IndexPathFactory {
virtual ~IndexPathFactory() = default;
virtual std::string NewPath() const = 0;
+ virtual std::string NewPathFromName(const std::string& file_name) const {
+ return ToPath(file_name);
+ }
virtual std::string ToPath(const std::string& file_name) const = 0;
Review Comment:
Should we remove the `ToPath` function from `IndexPathFactory`? It looks
like `ToPath` doesn’t do directory selection. If production code misuses it,
the configuration may not take effect—or at least it should return `not
implemented`.
##########
src/paimon/core/utils/file_store_path_factory.cpp:
##########
@@ -195,21 +200,36 @@ std::unique_ptr<IndexPathFactory>
FileStorePathFactory::CreateGlobalIndexFileFac
}
return
PathUtil::JoinPath(factory_->IndexPath(factory_->RootPath()), file->FileName());
}
- std::string ToPath(const std::string& file_name) const override {
- const auto& external_path = factory_->GetGlobalIndexExternalPath();
- if (external_path) {
- return PathUtil::JoinPath(external_path.value(), file_name);
+ std::string NewPathFromName(const std::string& file_name) const
override {
+ if (provider_) {
+ return provider_->GetNextExternalDataPath(file_name);
}
- return
PathUtil::JoinPath(factory_->IndexPath(factory_->RootPath()), file_name);
+ return ToPath(file_name);
+ }
+ std::string ToPath(const std::string& file_name) const override {
+ return factory_->ToIndexFilePath(file_name);
Review Comment:
The current `ToPath` semantics differ from Java. In Java, it uses the
configured external index directory / `fileName`; if it’s not configured, it
falls back to `table/index/fileName`. But the current implementation always
returns `table/index/fileName`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]