This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 29bba2989 [VL] Daily Update Velox Version(2024_08_15) (#6851)
29bba2989 is described below
commit 29bba29893077092a01a539209357339086e4f43
Author: Jin Chengcheng <[email protected]>
AuthorDate: Fri Aug 16 07:26:26 2024 +0800
[VL] Daily Update Velox Version(2024_08_15) (#6851)
---
cpp/velox/compute/VeloxBackend.cc | 16 +++++++++-------
cpp/velox/compute/VeloxBackend.h | 6 +++---
cpp/velox/compute/VeloxRuntime.cc | 5 +++--
cpp/velox/compute/VeloxRuntime.h | 2 +-
cpp/velox/compute/WholeStageResultIterator.cc | 13 +++++++------
cpp/velox/compute/WholeStageResultIterator.h | 6 +++---
cpp/velox/jni/JniFileSystem.cc | 10 ++++++----
cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h | 3 ++-
cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h | 3 ++-
cpp/velox/operators/writer/VeloxParquetDatasourceS3.h | 3 ++-
cpp/velox/substrait/SubstraitToVeloxPlan.cc | 3 ++-
cpp/velox/utils/ConfigExtractor.cc | 16 +++++++++-------
cpp/velox/utils/ConfigExtractor.h | 5 +++--
cpp/velox/utils/HdfsUtils.cc | 2 +-
cpp/velox/utils/HdfsUtils.h | 4 ++--
ep/build-velox/src/get_velox.sh | 2 +-
16 files changed, 56 insertions(+), 43 deletions(-)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 2dad6adf2..cba573ade 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -63,13 +63,14 @@ gluten::Runtime* veloxRuntimeFactory(
} // namespace
void VeloxBackend::init(const std::unordered_map<std::string, std::string>&
conf) {
- backendConf_ = std::make_shared<facebook::velox::core::MemConfig>(conf);
+ backendConf_ =
+
std::make_shared<facebook::velox::config::ConfigBase>(std::unordered_map<std::string,
std::string>(conf));
// Register Velox runtime factory
gluten::Runtime::registerFactory(gluten::kVeloxRuntimeKind,
veloxRuntimeFactory);
if (backendConf_->get<bool>(kDebugModeEnabled, false)) {
- LOG(INFO) << "VeloxBackend config:" << printConfig(backendConf_->values());
+ LOG(INFO) << "VeloxBackend config:" <<
printConfig(backendConf_->rawConfigs());
}
// Init glog and log level.
@@ -77,7 +78,7 @@ void VeloxBackend::init(const std::unordered_map<std::string,
std::string>& conf
FLAGS_v = backendConf_->get<uint32_t>(kGlogVerboseLevel,
kGlogVerboseLevelDefault);
FLAGS_minloglevel = backendConf_->get<uint32_t>(kGlogSeverityLevel,
kGlogSeverityLevelDefault);
} else {
- if (backendConf_->isValueExists(kGlogVerboseLevel)) {
+ if (backendConf_->valueExists(kGlogVerboseLevel)) {
FLAGS_v = backendConf_->get<uint32_t>(kGlogVerboseLevel,
kGlogVerboseLevelDefault);
} else {
FLAGS_v = kGlogVerboseLevelMaximum;
@@ -187,15 +188,15 @@ void VeloxBackend::initCache() {
void VeloxBackend::initConnector() {
// The configs below are used at process level.
- std::unordered_map<std::string, std::string> connectorConfMap =
backendConf_->values();
+ std::unordered_map<std::string, std::string> connectorConfMap =
backendConf_->rawConfigs();
auto hiveConf = getHiveConfig(backendConf_);
- for (auto& [k, v] : hiveConf->valuesCopy()) {
+ for (auto& [k, v] : hiveConf->rawConfigsCopy()) {
connectorConfMap[k] = v;
}
#ifdef ENABLE_ABFS
- const auto& confValue = backendConf_->values();
+ const auto& confValue = backendConf_->rawConfigs();
for (auto& [k, v] : confValue) {
if (k.find("fs.azure.account.key") == 0) {
connectorConfMap[k] = v;
@@ -205,6 +206,7 @@ void VeloxBackend::initConnector() {
}
}
#endif
+
connectorConfMap[velox::connector::hive::HiveConfig::kEnableFileHandleCache]
=
backendConf_->get<bool>(kVeloxFileHandleCacheEnabled,
kVeloxFileHandleCacheEnabledDefault) ? "true" : "false";
@@ -233,7 +235,7 @@ void VeloxBackend::initConnector() {
}
velox::connector::registerConnector(std::make_shared<velox::connector::hive::HiveConnector>(
kHiveConnectorId,
-
std::make_shared<facebook::velox::core::MemConfig>(std::move(connectorConfMap)),
+
std::make_shared<facebook::velox::config::ConfigBase>(std::move(connectorConfMap)),
ioExecutor_.get()));
}
diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h
index e8298eeed..df0442855 100644
--- a/cpp/velox/compute/VeloxBackend.h
+++ b/cpp/velox/compute/VeloxBackend.h
@@ -25,9 +25,9 @@
#include <filesystem>
#include "velox/common/caching/AsyncDataCache.h"
+#include "velox/common/config/Config.h"
#include "velox/common/memory/MemoryPool.h"
#include "velox/common/memory/MmapAllocator.h"
-#include "velox/core/Config.h"
namespace gluten {
/// As a static instance in per executor, initialized at executor startup.
@@ -53,7 +53,7 @@ class VeloxBackend {
facebook::velox::cache::AsyncDataCache* getAsyncDataCache() const;
- std::shared_ptr<facebook::velox::Config> getBackendConf() const {
+ std::shared_ptr<facebook::velox::config::ConfigBase> getBackendConf() const {
return backendConf_;
}
@@ -92,7 +92,7 @@ class VeloxBackend {
std::string cachePathPrefix_;
std::string cacheFilePrefix_;
- std::shared_ptr<facebook::velox::Config> backendConf_;
+ std::shared_ptr<facebook::velox::config::ConfigBase> backendConf_;
};
} // namespace gluten
diff --git a/cpp/velox/compute/VeloxRuntime.cc
b/cpp/velox/compute/VeloxRuntime.cc
index cdce781bd..996fcb850 100644
--- a/cpp/velox/compute/VeloxRuntime.cc
+++ b/cpp/velox/compute/VeloxRuntime.cc
@@ -62,7 +62,8 @@ VeloxRuntime::VeloxRuntime(
: Runtime(std::make_shared<VeloxMemoryManager>(std::move(listener)),
confMap) {
// Refresh session config.
vmm_ = dynamic_cast<VeloxMemoryManager*>(memoryManager_.get());
- veloxCfg_ = std::make_shared<facebook::velox::core::MemConfig>(confMap_);
+ veloxCfg_ =
+
std::make_shared<facebook::velox::config::ConfigBase>(std::unordered_map<std::string,
std::string>(confMap_));
debugModeEnabled_ = veloxCfg_->get<bool>(kDebugModeEnabled, false);
FLAGS_minloglevel = veloxCfg_->get<uint32_t>(kGlogSeverityLevel,
FLAGS_minloglevel);
FLAGS_v = veloxCfg_->get<uint32_t>(kGlogVerboseLevel, FLAGS_v);
@@ -270,7 +271,7 @@ std::unique_ptr<ColumnarBatchSerializer>
VeloxRuntime::createColumnarBatchSerial
}
void VeloxRuntime::dumpConf(const std::string& path) {
- const auto& backendConfMap = VeloxBackend::get()->getBackendConf()->values();
+ const auto& backendConfMap =
VeloxBackend::get()->getBackendConf()->rawConfigs();
auto allConfMap = backendConfMap;
for (const auto& pair : confMap_) {
diff --git a/cpp/velox/compute/VeloxRuntime.h b/cpp/velox/compute/VeloxRuntime.h
index 952a103ed..3460677d9 100644
--- a/cpp/velox/compute/VeloxRuntime.h
+++ b/cpp/velox/compute/VeloxRuntime.h
@@ -101,7 +101,7 @@ class VeloxRuntime final : public Runtime {
private:
VeloxMemoryManager* vmm_;
std::shared_ptr<const facebook::velox::core::PlanNode> veloxPlan_;
- std::shared_ptr<facebook::velox::Config> veloxCfg_;
+ std::shared_ptr<facebook::velox::config::ConfigBase> veloxCfg_;
bool debugModeEnabled_{false};
std::unordered_map<int32_t, std::shared_ptr<ColumnarBatch>>
emptySchemaBatchLoopUp_;
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index eb700c648..b9c790001 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -62,7 +62,8 @@ WholeStageResultIterator::WholeStageResultIterator(
const std::unordered_map<std::string, std::string>& confMap,
const SparkTaskInfo& taskInfo)
: memoryManager_(memoryManager),
- veloxCfg_(std::make_shared<facebook::velox::core::MemConfig>(confMap)),
+ veloxCfg_(
+
std::make_shared<facebook::velox::config::ConfigBase>(std::unordered_map<std::string,
std::string>(confMap))),
taskInfo_(taskInfo),
veloxPlan_(planNode),
scanNodeIds_(scanNodeIds),
@@ -175,7 +176,7 @@ WholeStageResultIterator::WholeStageResultIterator(
}
std::shared_ptr<velox::core::QueryCtx>
WholeStageResultIterator::createNewVeloxQueryCtx() {
- std::unordered_map<std::string, std::shared_ptr<velox::Config>>
connectorConfigs;
+ std::unordered_map<std::string, std::shared_ptr<velox::config::ConfigBase>>
connectorConfigs;
connectorConfigs[kHiveConnectorId] = createConnectorConfig();
std::shared_ptr<velox::core::QueryCtx> ctx = velox::core::QueryCtx::create(
@@ -437,10 +438,10 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
// Find offheap size from Spark confs. If found, set the max memory usage of
partial aggregation.
// FIXME this uses process-wise off-heap memory which is not for task
try {
- if (veloxCfg_->isValueExists(kDefaultSessionTimezone)) {
+ if (veloxCfg_->valueExists(kDefaultSessionTimezone)) {
configs[velox::core::QueryConfig::kSessionTimezone] =
veloxCfg_->get<std::string>(kDefaultSessionTimezone, "");
}
- if (veloxCfg_->isValueExists(kSessionTimezone)) {
+ if (veloxCfg_->valueExists(kSessionTimezone)) {
configs[velox::core::QueryConfig::kSessionTimezone] =
veloxCfg_->get<std::string>(kSessionTimezone, "");
}
// Adjust timestamp according to the above configured session timezone.
@@ -519,7 +520,7 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
return configs;
}
-std::shared_ptr<velox::Config>
WholeStageResultIterator::createConnectorConfig() {
+std::shared_ptr<velox::config::ConfigBase>
WholeStageResultIterator::createConnectorConfig() {
// The configs below are used at session level.
std::unordered_map<std::string, std::string> configs = {};
// The semantics of reading as lower case is opposite with case-sensitive.
@@ -532,7 +533,7 @@ std::shared_ptr<velox::Config>
WholeStageResultIterator::createConnectorConfig()
std::to_string(veloxCfg_->get<int32_t>(kMaxPartitions, 10000));
configs[velox::connector::hive::HiveConfig::kIgnoreMissingFilesSession] =
std::to_string(veloxCfg_->get<bool>(kIgnoreMissingFiles, false));
- return std::make_shared<velox::core::MemConfig>(configs);
+ return std::make_shared<velox::config::ConfigBase>(std::move(configs));
}
} // namespace gluten
diff --git a/cpp/velox/compute/WholeStageResultIterator.h
b/cpp/velox/compute/WholeStageResultIterator.h
index 5e661f404..371ec0c14 100644
--- a/cpp/velox/compute/WholeStageResultIterator.h
+++ b/cpp/velox/compute/WholeStageResultIterator.h
@@ -23,8 +23,8 @@
#include "substrait/SubstraitToVeloxPlan.h"
#include "substrait/plan.pb.h"
#include "utils/metrics.h"
+#include "velox/common/config/Config.h"
#include "velox/connectors/hive/iceberg/IcebergSplit.h"
-#include "velox/core/Config.h"
#include "velox/core/PlanNode.h"
#include "velox/exec/Task.h"
@@ -80,7 +80,7 @@ class WholeStageResultIterator : public ColumnarBatchIterator
{
std::vector<facebook::velox::core::PlanNodeId>& nodeIds);
/// Create connector config.
- std::shared_ptr<facebook::velox::Config> createConnectorConfig();
+ std::shared_ptr<facebook::velox::config::ConfigBase> createConnectorConfig();
/// Construct partition columns.
void constructPartitionColumns(
@@ -103,7 +103,7 @@ class WholeStageResultIterator : public
ColumnarBatchIterator {
VeloxMemoryManager* memoryManager_;
/// Config, task and plan.
- std::shared_ptr<Config> veloxCfg_;
+ std::shared_ptr<config::ConfigBase> veloxCfg_;
const SparkTaskInfo taskInfo_;
std::shared_ptr<facebook::velox::exec::Task> task_;
std::shared_ptr<const facebook::velox::core::PlanNode> veloxPlan_;
diff --git a/cpp/velox/jni/JniFileSystem.cc b/cpp/velox/jni/JniFileSystem.cc
index 8bf791c23..55c1dc734 100644
--- a/cpp/velox/jni/JniFileSystem.cc
+++ b/cpp/velox/jni/JniFileSystem.cc
@@ -261,7 +261,8 @@ class FileSystemWrapper : public
facebook::velox::filesystems::FileSystem {
class JniFileSystem : public facebook::velox::filesystems::FileSystem {
public:
- explicit JniFileSystem(jobject obj, std::shared_ptr<const
facebook::velox::Config> config) : FileSystem(config) {
+ explicit JniFileSystem(jobject obj, std::shared_ptr<const
facebook::velox::config::ConfigBase> config)
+ : FileSystem(config) {
JNIEnv* env;
attachCurrentThreadAsDaemonOrThrow(vm, &env);
obj_ = env->NewGlobalRef(obj);
@@ -374,9 +375,10 @@ class JniFileSystem : public
facebook::velox::filesystems::FileSystem {
return [](std::string_view filePath) { return filePath.find(kJniFsScheme)
== 0; };
}
- static std::function<std::shared_ptr<FileSystem>(std::shared_ptr<const
facebook::velox::Config>, std::string_view)>
+ static std::function<
+ std::shared_ptr<FileSystem>(std::shared_ptr<const
facebook::velox::config::ConfigBase>, std::string_view)>
fileSystemGenerator() {
- return [](std::shared_ptr<const facebook::velox::Config> properties,
std::string_view filePath) {
+ return [](std::shared_ptr<const facebook::velox::config::ConfigBase>
properties, std::string_view filePath) {
JNIEnv* env;
attachCurrentThreadAsDaemonOrThrow(vm, &env);
jobject obj = env->CallStaticObjectMethod(jniFileSystemClass,
jniGetFileSystem);
@@ -455,7 +457,7 @@ void gluten::registerJolFileSystem(uint64_t maxFileSize) {
auto fileSystemGenerator =
[maxFileSize](
- std::shared_ptr<const facebook::velox::Config> properties,
+ std::shared_ptr<const facebook::velox::config::ConfigBase>
properties,
std::string_view filePath) ->
std::shared_ptr<facebook::velox::filesystems::FileSystem> {
// select JNI file if there is enough space
if (JniFileSystem::isCapableForNewFile(maxFileSize)) {
diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h
b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h
index 82e8f794c..b542e42ba 100644
--- a/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h
+++ b/cpp/velox/operators/writer/VeloxParquetDatasourceABFS.h
@@ -43,7 +43,8 @@ class VeloxParquetDatasourceABFS final : public
VeloxParquetDatasource {
: VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {}
void initSink(const std::unordered_map<std::string, std::string>&
sparkConfs) override {
- auto hiveConf =
getHiveConfig(std::make_shared<facebook::velox::core::MemConfig>(sparkConfs));
+ auto hiveConf =
getHiveConfig(std::make_shared<facebook::velox::config::ConfigBase>(
+ std::unordered_map<std::string, std::string>(sparkConfs)));
auto fileSystem = filesystems::getFileSystem(filePath_, hiveConf);
auto* abfsFileSystem =
dynamic_cast<filesystems::abfs::AbfsFileSystem*>(fileSystem.get());
sink_ = std::make_unique<dwio::common::WriteFileSink>(
diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h
b/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h
index 7722c8e51..19e9e3560 100644
--- a/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h
+++ b/cpp/velox/operators/writer/VeloxParquetDatasourceHDFS.h
@@ -43,7 +43,8 @@ class VeloxParquetDatasourceHDFS final : public
VeloxParquetDatasource {
: VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {}
void initSink(const std::unordered_map<std::string, std::string>&
sparkConfs) override {
- auto hiveConf =
getHiveConfig(std::make_shared<facebook::velox::core::MemConfig>(sparkConfs));
+ auto hiveConf =
getHiveConfig(std::make_shared<facebook::velox::config::ConfigBase>(
+ std::unordered_map<std::string, std::string>(sparkConfs)));
sink_ = dwio::common::FileSink::create(filePath_, {.connectorProperties =
hiveConf, .pool = sinkPool_.get()});
}
};
diff --git a/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h
b/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h
index 3231a8a1e..8219fe42a 100644
--- a/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h
+++ b/cpp/velox/operators/writer/VeloxParquetDatasourceS3.h
@@ -43,7 +43,8 @@ class VeloxParquetDatasourceS3 final : public
VeloxParquetDatasource {
: VeloxParquetDatasource(filePath, veloxPool, sinkPool, schema) {}
void initSink(const std::unordered_map<std::string, std::string>&
sparkConfs) override {
- auto hiveConf =
getHiveConfig(std::make_shared<facebook::velox::core::MemConfig>(sparkConfs));
+ auto hiveConf =
getHiveConfig(std::make_shared<facebook::velox::config::ConfigBase>(
+ std::unordered_map<std::string, std::string>(sparkConfs)));
sink_ = dwio::common::FileSink::create(filePath_, {.connectorProperties =
hiveConf, .pool = sinkPool_.get()});
}
};
diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
index d7de84119..1604c15e3 100644
--- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc
+++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc
@@ -1148,7 +1148,8 @@ core::PlanNodePtr
SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait::
std::vector<TypePtr> veloxTypeList;
std::vector<ColumnType> columnTypes;
// Convert field names into lower case when not case-sensitive.
- std::unique_ptr<facebook::velox::Config> veloxCfg =
std::make_unique<facebook::velox::core::MemConfig>(confMap_);
+ std::unique_ptr<facebook::velox::config::ConfigBase> veloxCfg =
+
std::make_unique<facebook::velox::config::ConfigBase>(std::unordered_map<std::string,
std::string>(confMap_));
bool asLowerCase = !veloxCfg->get<bool>(kCaseSensitive, false);
if (readRel.has_base_schema()) {
const auto& baseSchema = readRel.base_schema();
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index 816166351..4189df758 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -59,7 +59,8 @@ std::string getConfigValue(
return got->second;
}
-std::shared_ptr<facebook::velox::core::MemConfig>
getHiveConfig(std::shared_ptr<facebook::velox::Config> conf) {
+std::shared_ptr<facebook::velox::config::ConfigBase> getHiveConfig(
+ std::shared_ptr<facebook::velox::config::ConfigBase> conf) {
std::unordered_map<std::string, std::string> hiveConfMap;
#ifdef ENABLE_S3
@@ -125,7 +126,7 @@ std::shared_ptr<facebook::velox::core::MemConfig>
getHiveConfig(std::shared_ptr<
#ifdef ENABLE_GCS
//
https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md#api-client-configuration
- auto gsStorageRootUrl = conf->get("spark.hadoop.fs.gs.storage.root.url");
+ auto gsStorageRootUrl =
conf->get<std::string>("spark.hadoop.fs.gs.storage.root.url");
if (gsStorageRootUrl.hasValue()) {
std::string url = gsStorageRootUrl.value();
std::string gcsScheme;
@@ -146,23 +147,24 @@ std::shared_ptr<facebook::velox::core::MemConfig>
getHiveConfig(std::shared_ptr<
//
https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md#http-transport-configuration
//
https://cloud.google.com/cpp/docs/reference/storage/latest/classgoogle_1_1cloud_1_1storage_1_1LimitedErrorCountRetryPolicy
- auto gsMaxRetryCount = conf->get("spark.hadoop.fs.gs.http.max.retry");
+ auto gsMaxRetryCount =
conf->get<std::string>("spark.hadoop.fs.gs.http.max.retry");
if (gsMaxRetryCount.hasValue()) {
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSMaxRetryCount] =
gsMaxRetryCount.value();
}
//
https://cloud.google.com/cpp/docs/reference/storage/latest/classgoogle_1_1cloud_1_1storage_1_1LimitedTimeRetryPolicy
- auto gsMaxRetryTime = conf->get("spark.hadoop.fs.gs.http.max.retry-time");
+ auto gsMaxRetryTime =
conf->get<std::string>("spark.hadoop.fs.gs.http.max.retry-time");
if (gsMaxRetryTime.hasValue()) {
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kGCSMaxRetryTime] =
gsMaxRetryTime.value();
}
//
https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/CONFIGURATION.md#authentication
- auto gsAuthType = conf->get("spark.hadoop.fs.gs.auth.type");
+ auto gsAuthType = conf->get<std::string>("spark.hadoop.fs.gs.auth.type");
if (gsAuthType.hasValue()) {
std::string type = gsAuthType.value();
if (type == "SERVICE_ACCOUNT_JSON_KEYFILE") {
- auto gsAuthServiceAccountJsonKeyfile =
conf->get("spark.hadoop.fs.gs.auth.service.account.json.keyfile");
+ auto gsAuthServiceAccountJsonKeyfile =
+
conf->get<std::string>("spark.hadoop.fs.gs.auth.service.account.json.keyfile");
if (gsAuthServiceAccountJsonKeyfile.hasValue()) {
auto stream = std::ifstream(gsAuthServiceAccountJsonKeyfile.value());
stream.exceptions(std::ios::badbit);
@@ -180,7 +182,7 @@ std::shared_ptr<facebook::velox::core::MemConfig>
getHiveConfig(std::shared_ptr<
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kEnableFileHandleCache]
=
conf->get<bool>(kVeloxFileHandleCacheEnabled,
kVeloxFileHandleCacheEnabledDefault) ? "true" : "false";
- return
std::make_shared<facebook::velox::core::MemConfig>(std::move(hiveConfMap));
+ return
std::make_shared<facebook::velox::config::ConfigBase>(std::move(hiveConfMap));
}
} // namespace gluten
diff --git a/cpp/velox/utils/ConfigExtractor.h
b/cpp/velox/utils/ConfigExtractor.h
index c5f662c95..4cbfdf991 100644
--- a/cpp/velox/utils/ConfigExtractor.h
+++ b/cpp/velox/utils/ConfigExtractor.h
@@ -24,7 +24,7 @@
#include <unordered_map>
#include "config/GlutenConfig.h"
-#include "velox/core/Config.h"
+#include "velox/common/config/Config.h"
namespace gluten {
@@ -33,6 +33,7 @@ std::string getConfigValue(
const std::string& key,
const std::optional<std::string>& fallbackValue);
-std::shared_ptr<facebook::velox::core::MemConfig>
getHiveConfig(std::shared_ptr<facebook::velox::Config> conf);
+std::shared_ptr<facebook::velox::config::ConfigBase> getHiveConfig(
+ std::shared_ptr<facebook::velox::config::ConfigBase> conf);
} // namespace gluten
diff --git a/cpp/velox/utils/HdfsUtils.cc b/cpp/velox/utils/HdfsUtils.cc
index a912c04ee..1bc326d4c 100644
--- a/cpp/velox/utils/HdfsUtils.cc
+++ b/cpp/velox/utils/HdfsUtils.cc
@@ -36,7 +36,7 @@ struct Credential {
};
} // namespace
-void updateHdfsTokens(const facebook::velox::Config* veloxCfg) {
+void updateHdfsTokens(const facebook::velox::config::ConfigBase* veloxCfg) {
static std::mutex mtx;
std::lock_guard lock{mtx};
diff --git a/cpp/velox/utils/HdfsUtils.h b/cpp/velox/utils/HdfsUtils.h
index cd017f250..2e07d7ddf 100644
--- a/cpp/velox/utils/HdfsUtils.h
+++ b/cpp/velox/utils/HdfsUtils.h
@@ -15,8 +15,8 @@
* limitations under the License.
*/
-#include <velox/core/Config.h>
+#include <velox/common/config/Config.h>
#include <memory>
namespace gluten {
-void updateHdfsTokens(const facebook::velox::Config* veloxCfg);
+void updateHdfsTokens(const facebook::velox::config::ConfigBase* veloxCfg);
}
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 9af55aae2..b51233983 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_08_14
+VELOX_BRANCH=2024_08_15
VELOX_HOME=""
OS=`uname -s`
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
