This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 56c5a24c1 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240526)
(#5870)
56c5a24c1 is described below
commit 56c5a24c1bc2886e2cae697711bcda016d0c0cca
Author: Kyligence Git <[email protected]>
AuthorDate: Sun May 26 05:35:08 2024 -0500
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240526) (#5870)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240526)
* Fix build due to https://github.com/ClickHouse/ClickHouse/pull/59767
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 ++--
.../local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp | 8 +++++---
.../local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h | 2 +-
.../Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp | 3 +--
cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp | 6 ++----
.../local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp | 7 ++-----
.../local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp | 6 +++---
7 files changed, 16 insertions(+), 20 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 1bc0454a1..775667afb 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20240524
-CH_COMMIT=b1a2a14bdd2
\ No newline at end of file
+CH_BRANCH=rebase_ch/20240526
+CH_COMMIT=ff17e067fac
\ No newline at end of file
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
index 3a844a91f..60b82ec84 100644
--- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
+++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
@@ -17,7 +17,7 @@
#include "GlutenHDFSObjectStorage.h"
#if USE_HDFS
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
using namespace DB;
namespace local_engine
{
@@ -35,8 +35,10 @@ std::unique_ptr<ReadBufferFromFileBase>
GlutenHDFSObjectStorage::readObject( ///
DB::ObjectStorageKey
local_engine::GlutenHDFSObjectStorage::generateObjectKeyForPath(const
std::string & path) const
{
- return DB::ObjectStorageKey::createAsAbsolute(hdfs_root_path + path);
+ initializeHDFSFS();
+ /// what ever data_source_description.description value is, consider that
key as relative key
+ chassert(data_directory.starts_with("/"));
+ return ObjectStorageKey::createAsRelative(fs::path(url_without_path) /
data_directory.substr(1) / path);
}
}
#endif
-
diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
index 1efa441c2..a532c98cb 100644
--- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
+++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
@@ -33,7 +33,7 @@ public:
const String & hdfs_root_path_,
SettingsPtr settings_,
const Poco::Util::AbstractConfiguration & config_)
- : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_),
config(config_)
+ : HDFSObjectStorage(hdfs_root_path_, std::move(settings_), config_, /*
lazy_initialize */true), config(config_)
{
}
std::unique_ptr<DB::ReadBufferFromFileBase> readObject( /// NOLINT
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
index 8f2008029..800b51f93 100644
---
a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
+++
b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
@@ -75,7 +75,7 @@ void registerGlutenS3ObjectStorage(ObjectStorageFactory &
factory)
auto uri = getS3URI(config, config_prefix, context);
auto s3_capabilities = getCapabilitiesFromConfig(config,
config_prefix);
auto settings = getSettings(config, config_prefix, context);
- auto client = getClient(config, config_prefix, context, *settings);
+ auto client = getClient(config, config_prefix, context, *settings,
true);
auto key_generator =
createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key);
auto object_storage = std::make_shared<S3ObjectStorage>(
@@ -110,7 +110,6 @@ void registerGlutenHDFSObjectStorage(ObjectStorageFactory &
factory)
std::unique_ptr<HDFSObjectStorageSettings> settings =
std::make_unique<HDFSObjectStorageSettings>(
config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 *
1024),
- config.getInt(config_prefix + ".objects_chunk_size_to_delete",
1000),
context->getSettingsRef().hdfs_replication
);
return std::make_unique<GlutenHDFSObjectStorage>(uri,
std::move(settings), config);
diff --git a/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp
b/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp
index 02f9cd9f2..3dc205c39 100644
--- a/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp
+++ b/cpp-ch/local-engine/Storages/Output/WriteBufferBuilder.cpp
@@ -17,13 +17,11 @@
#include <memory>
#include <IO/WriteSettings.h>
#include <Interpreters/Cache/FileCache.h>
-#include <Interpreters/Cache/FileCacheSettings.h>
#include <Interpreters/Context_fwd.h>
-#include <Storages/HDFS/HDFSCommon.h>
-#include <Storages/HDFS/WriteBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/WriteBufferFromHDFS.h>
#include <Storages/Output/WriteBufferBuilder.h>
#include <hdfs/hdfs.h>
-#include <Poco/Logger.h>
#include <Poco/URI.h>
#include <Common/CHUtil.h>
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
index d7cbcaa5f..038f280b0 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
@@ -14,13 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-#include "Common/CHUtil.h"
#include "ExcelTextFormatFile.h"
-
+#include <Common/CHUtil.h>
#include <memory>
#include <string>
-#include <utility>
#include <Columns/ColumnNullable.h>
#include <DataTypes/DataTypeDecimalBase.h>
@@ -28,9 +26,8 @@
#include <DataTypes/Serializations/SerializationNullable.h>
#include <Formats/FormatSettings.h>
#include <IO/PeekableReadBuffer.h>
-#include <IO/SeekableReadBuffer.h>
#include <Processors/Formats/IRowInputFormat.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
#include <Storages/Serializations/ExcelDecimalSerialization.h>
#include <Storages/Serializations/ExcelSerialization.h>
#include <Storages/Serializations/ExcelStringReader.h>
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
index 21640fe49..d54ff985e 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
@@ -35,9 +35,9 @@
#include <Interpreters/Cache/FileCacheFactory.h>
#include <Interpreters/Cache/FileCacheSettings.h>
#include <Interpreters/Context_fwd.h>
-#include <Storages/HDFS/AsynchronousReadBufferFromHDFS.h>
-#include <Storages/HDFS/HDFSCommon.h>
-#include <Storages/HDFS/ReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h>
+#include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
#include <Storages/StorageS3Settings.h>
#include <Storages/SubstraitSource/ReadBufferBuilder.h>
#include <Storages/SubstraitSource/SubstraitFileSource.h>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
