This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cb0ad2024 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240803)
(#6700)
cb0ad2024 is described below
commit cb0ad202442c698fb06611ffebfb113764c21948
Author: Kyligence Git <[email protected]>
AuthorDate: Sat Aug 3 08:49:04 2024 -0500
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20240803) (#6700)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20240803)
* fix build due to https://github.com/ClickHouse/ClickHouse/pull/65751
* Fix build due to https://github.com/apache/incubator-gluten/pull/6558
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 ++--
.../Disks/ObjectStorages/CompactObjectStorageDiskTransaction.cpp | 4 ++--
cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp | 2 +-
cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h | 2 +-
cpp-ch/local-engine/Storages/Mergetree/MetaDataHelper.cpp | 2 +-
cpp-ch/local-engine/tests/benchmark_local_engine.cpp | 2 +-
6 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index dd638c974..d73e34fb4 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,4 +1,4 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20240801
-CH_COMMIT=768e193bd4d
+CH_BRANCH=rebase_ch/20240803
+CH_COMMIT=b3cfc38f967
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/CompactObjectStorageDiskTransaction.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/CompactObjectStorageDiskTransaction.cpp
index 66c447010..3a4d15687 100644
---
a/cpp-ch/local-engine/Disks/ObjectStorages/CompactObjectStorageDiskTransaction.cpp
+++
b/cpp-ch/local-engine/Disks/ObjectStorages/CompactObjectStorageDiskTransaction.cpp
@@ -60,8 +60,8 @@ void CompactObjectStorageDiskTransaction::commit()
std::filesystem::path meta_path = std::filesystem::path(prefix_path) /
"meta.bin";
auto object_storage = disk.getObjectStorage();
- auto data_key = object_storage->generateObjectKeyForPath(data_path);
- auto meta_key = object_storage->generateObjectKeyForPath(meta_path);
+ auto data_key = object_storage->generateObjectKeyForPath(data_path,
std::nullopt);
+ auto meta_key = object_storage->generateObjectKeyForPath(meta_path,
std::nullopt);
disk.createDirectories(prefix_path);
auto data_write_buffer =
object_storage->writeObject(DB::StoredObject(data_key.serialize(), data_path),
DB::WriteMode::Rewrite);
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
index cab87d66d..38db8a2df 100644
--- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
+++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.cpp
@@ -33,7 +33,7 @@ std::unique_ptr<ReadBufferFromFileBase>
GlutenHDFSObjectStorage::readObject( ///
return std::make_unique<ReadBufferFromHDFS>(hdfs_uri, hdfs_path, config,
HDFSObjectStorage::patchSettings(read_settings));
}
-DB::ObjectStorageKey
local_engine::GlutenHDFSObjectStorage::generateObjectKeyForPath(const
std::string & path) const
+DB::ObjectStorageKey
local_engine::GlutenHDFSObjectStorage::generateObjectKeyForPath(const
std::string & path, const std::optional<std::string> & key_prefix) const
{
initializeHDFSFS();
/// what ever data_source_description.description value is, consider that
key as relative key
diff --git a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
index da37e1d78..82f3003eb 100644
--- a/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
+++ b/cpp-ch/local-engine/Disks/ObjectStorages/GlutenHDFSObjectStorage.h
@@ -41,7 +41,7 @@ public:
const DB::ReadSettings & read_settings = DB::ReadSettings{},
std::optional<size_t> read_hint = {},
std::optional<size_t> file_size = {}) const override;
- DB::ObjectStorageKey generateObjectKeyForPath(const std::string & path)
const override;
+ DB::ObjectStorageKey generateObjectKeyForPath(const std::string & path,
const std::optional<std::string> & key_prefix) const override;
hdfsFS getHDFSFS() const { return hdfs_fs.get(); }
};
#endif
diff --git a/cpp-ch/local-engine/Storages/Mergetree/MetaDataHelper.cpp
b/cpp-ch/local-engine/Storages/Mergetree/MetaDataHelper.cpp
index ebc01544f..bcb130259 100644
--- a/cpp-ch/local-engine/Storages/Mergetree/MetaDataHelper.cpp
+++ b/cpp-ch/local-engine/Storages/Mergetree/MetaDataHelper.cpp
@@ -105,7 +105,7 @@ void restoreMetaData(CustomStorageMergeTreePtr & storage,
const MergeTreeTable &
return;
else
metadata_disk->createDirectories(part_path);
- auto key =
s3->generateObjectKeyForPath(metadata_file_path.generic_string());
+ auto key =
s3->generateObjectKeyForPath(metadata_file_path.generic_string(), std::nullopt);
StoredObject metadata_object(key.serialize());
auto part_metadata =
extractPartMetaData(*s3->readObject(metadata_object));
for (const auto & item : part_metadata)
diff --git a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
index 3402d5026..fba8ef732 100644
--- a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
@@ -35,7 +35,7 @@
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
#include <QueryPipeline/QueryPipelineBuilder.h>
#include <Shuffle/ShuffleReader.h>
-#include <Shuffle/ShuffleSplitter.h>
+
#include <Storages/CustomMergeTreeSink.h>
#include <Storages/CustomStorageMergeTree.h>
#include <Storages/MergeTree/MergeTreeData.h>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
