This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fd2805f554 [GLUTEN-7420][VL] Fix GCS configuration (#7421)
fd2805f554 is described below
commit fd2805f55442fc91ef60f51aa8690e62cd448a25
Author: Deepak Majeti <[email protected]>
AuthorDate: Wed Oct 16 01:34:04 2024 -0400
[GLUTEN-7420][VL] Fix GCS configuration (#7421)
---
cpp/velox/utils/ConfigExtractor.cc | 9 ++-------
.../src/main/scala/org/apache/gluten/GlutenConfig.scala | 12 ++++++++++++
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index 7c0e180a54..707197eb05 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -19,9 +19,6 @@
#include "ConfigExtractor.h"
#include <stdexcept>
-#ifdef ENABLE_GCS
-#include <fstream>
-#endif
#include "utils/Exception.h"
#include "velox/connectors/hive/HiveConfig.h"
@@ -166,10 +163,8 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
getHiveConfig(
auto gsAuthServiceAccountJsonKeyfile =
conf->get<std::string>("spark.hadoop.fs.gs.auth.service.account.json.keyfile");
if (gsAuthServiceAccountJsonKeyfile.hasValue()) {
- auto stream = std::ifstream(gsAuthServiceAccountJsonKeyfile.value());
- stream.exceptions(std::ios::badbit);
- std::string gsAuthServiceAccountJson =
std::string(std::istreambuf_iterator<char>(stream.rdbuf()), {});
-
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSCredentialsPath]
= gsAuthServiceAccountJson;
+
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSCredentialsPath] =
+ gsAuthServiceAccountJsonKeyfile.value();
} else {
LOG(WARNING) << "STARTUP: conf spark.hadoop.fs.gs.auth.type is set to
SERVICE_ACCOUNT_JSON_KEYFILE, "
"however conf
spark.hadoop.fs.gs.auth.service.account.json.keyfile is not set";
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index aa54f28507..5ca6a1597d 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -792,10 +792,22 @@ object GlutenConfig {
.filter(_._1.startsWith(HADOOP_PREFIX + S3A_PREFIX))
.foreach(entry => nativeConfMap.put(entry._1, entry._2))
+ // handle ABFS config
conf
.filter(_._1.startsWith(SPARK_ABFS_ACCOUNT_KEY))
.foreach(entry => nativeConfMap.put(entry._1, entry._2))
+ // handle GCS config
+ if (conf.contains(SPARK_GCS_AUTH_TYPE)) {
+ nativeConfMap.put(SPARK_GCS_AUTH_TYPE, conf(SPARK_GCS_AUTH_TYPE))
+ }
+
+ if (conf.contains(SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE)) {
+ nativeConfMap.put(
+ SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE,
+ conf(SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE))
+ }
+
// return
nativeConfMap
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
