pratham76 commented on code in PR #13646:
URL: https://github.com/apache/gluten/pull/13646#discussion_r4584793576
##########
cpp/velox/tests/GlutenS3FileSystemTest.cc:
##########
@@ -68,5 +68,39 @@ TEST(GlutenS3FileSystemTest, extractsAsyncUploadConfigs) {
EXPECT_EQ(hiveConfig->get<std::string>(prefix + "upload-threads"), "32");
}
+TEST(GlutenS3FileSystemTest, extractsPerBucketConfigs) {
+ using Keys = filesystems::S3Config::Keys;
+ auto config =
std::make_shared<velox::config::ConfigBase>(std::unordered_map<std::string,
std::string>{
+ {"spark.hadoop.fs.s3a.bucket.plain-bucket.endpoint",
"http://plain:9000"},
+ {"spark.hadoop.fs.s3a.bucket.my.data.bucket.endpoint",
"http://dotted:9000"},
+ {"spark.hadoop.fs.s3a.bucket.my.data.bucket.access.key",
"dotted-access"},
+ {"spark.hadoop.fs.s3a.bucket.my.data.bucket.endpoint.region",
"us-west-2"},
+ {"spark.hadoop.fs.s3a.bucket.my.data.bucket.iam.role.session.name",
"dotted-session"},
+ {"spark.hadoop.fs.s3a.bucket.my.data.bucket.unknown.option",
"ignored"}});
+
+ const auto hiveConfig = createHiveConnectorConfig(config,
FileSystemType::kS3);
+
+ auto bucketValue = [&](Keys key, std::string_view bucket) {
+ return
hiveConfig->get<std::string>(filesystems::S3Config::bucketConfigKey(key,
bucket));
+ };
+ EXPECT_EQ(bucketValue(Keys::kEndpoint, "plain-bucket"), "http://plain:9000");
+ EXPECT_EQ(bucketValue(Keys::kEndpoint, "my.data.bucket"),
"http://dotted:9000");
+ EXPECT_EQ(bucketValue(Keys::kAccessKey, "my.data.bucket"), "dotted-access");
+ EXPECT_EQ(bucketValue(Keys::kEndpointRegion, "my.data.bucket"), "us-west-2");
+ EXPECT_EQ(bucketValue(Keys::kIamRoleSessionName, "my.data.bucket"),
"dotted-session");
+
+ // Nothing is registered under a bucket name truncated at the first dot.
+ EXPECT_FALSE(bucketValue(Keys::kEndpoint, "my").has_value());
+
+ // Only the five recognized options above produce per-bucket Velox configs.
+ size_t numBucketConfigs = 0;
+ for (const auto& [key, value] : hiveConfig->rawConfigs()) {
+ if (key.rfind(filesystems::S3Config::kS3BucketPrefix, 0) == 0) {
+ ++numBucketConfigs;
+ }
+ }
+ EXPECT_EQ(numBucketConfigs, 5);
Review Comment:
Thanks. The count only depends on the six inputs in this test, not on the
size of `sparkSuffixes`, so adding new recognized options won't change it. It's
intentionally the check that the unknown option produces no `s3.bucket.*` entry
at all, under any bucket name.
##########
cpp/velox/utils/ConfigExtractor.cc:
##########
@@ -84,14 +84,25 @@ void getS3HiveConfig(
{S3Config::Keys::kIMDSEnabled, std::make_pair("aws.imds.enabled",
"true")},
};
- // get Velox S3 config key from Spark Suffix.
- auto getVeloxKey = [&](std::string_view suffix) {
+ // Split "<bucket>.<suffix>" into the bucket name and the Velox S3 config
key. Bucket names may contain dots,
+ // so match a known Spark suffix at the end instead of splitting at the
first dot. The longest matching suffix
+ // wins, e.g. "endpoint.region" over "endpoint".
+ auto parseBucketConfig = [&](std::string_view bucketAndSuffix) {
+ std::optional<std::pair<S3Config::Keys, std::string_view>> result;
+ size_t matchedSuffixSize = 0;
for (const auto& [key, value] : sparkSuffixes) {
- if (value.first == suffix) {
- return std::optional<S3Config::Keys>(key);
+ const std::string_view suffix = value.first;
+ // At least one character for the bucket name plus the separating dot.
+ if (bucketAndSuffix.size() < suffix.size() + 2 || suffix.size() <=
matchedSuffixSize) {
+ continue;
+ }
+ const auto dotPos = bucketAndSuffix.size() - suffix.size() - 1;
+ if (bucketAndSuffix[dotPos] == '.' && bucketAndSuffix.substr(dotPos + 1)
== suffix) {
+ result = std::make_pair(key, bucketAndSuffix.substr(0, dotPos));
+ matchedSuffixSize = suffix.size();
}
}
- return std::optional<S3Config::Keys>(std::nullopt);
+ return result;
};
Review Comment:
Thanks for the suggestion. `sparkSuffixes` is a 15-entry local map scanned
only for the few per-bucket keys present, so the cost is negligible. Keeping
the match inline avoids maintaining a second sorted copy of the table. I'd
prefer to keep the change minimal, but I'm happy to refactor if reviewers
prefer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]