This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 9a19204377 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241017)
(#7567)
9a19204377 is described below
commit 9a19204377a68c3aebfad17fe9f1993f244d75ec
Author: Kyligence Git <[email protected]>
AuthorDate: Thu Oct 17 02:19:52 2024 -0500
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20241017) (#7567)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241017)
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/68146
* Fix Build due to rebasing failed with
https://github.com/ClickHouse/ClickHouse/pull/69179
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 ++--
.../Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp | 14 +++++++-------
.../Disks/ObjectStorages/MetadataStorageFromRocksDB.h | 6 +++---
.../MetadataStorageFromRocksDBTransactionOperations.cpp | 8 +++++++-
.../MetadataStorageFromRocksDBTransactionOperations.h | 1 +
cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp | 12 ++++++------
.../Storages/MergeTree/SparkStorageMergeTree.cpp | 2 +-
7 files changed, 27 insertions(+), 20 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 63308f5a51..65feb5e57c 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20241016
-CH_COMMIT=c3a1c1b8457
+CH_BRANCH=rebase_ch/20241017
+CH_COMMIT=afe3026c6ce
\ No newline at end of file
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp
index aba4265641..cebc34853e 100644
--- a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp
+++ b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.cpp
@@ -80,21 +80,21 @@ DB::MetadataStorageType
MetadataStorageFromRocksDB::getType() const
return DB::MetadataStorageType::None;
}
-bool MetadataStorageFromRocksDB::exists(const std::string & path) const
+bool MetadataStorageFromRocksDB::existsFileOrDirectory(const std::string &
path) const
{
return exist(getRocksDB(), path);
}
-bool MetadataStorageFromRocksDB::isFile(const std::string & path) const
+bool MetadataStorageFromRocksDB::existsFile(const std::string & path) const
{
- auto data = getData(getRocksDB(), path);
- return data != RocksDBCreateDirectoryOperation::DIR_DATA;
+ std::string data;
+ return tryGetData(getRocksDB(), path, &data) && data !=
RocksDBCreateDirectoryOperation::DIR_DATA;
}
-bool MetadataStorageFromRocksDB::isDirectory(const std::string & path) const
+bool MetadataStorageFromRocksDB::existsDirectory(const std::string & path)
const
{
- auto data = getData(getRocksDB(), path);
- return data == RocksDBCreateDirectoryOperation::DIR_DATA;
+ std::string data;
+ return tryGetData(getRocksDB(), path, &data) && data ==
RocksDBCreateDirectoryOperation::DIR_DATA;
}
uint64_t MetadataStorageFromRocksDB::getFileSize(const std::string & path)
const
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h
b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h
index 1f53a1efee..66a9ca4999 100644
--- a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h
+++ b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDB.h
@@ -39,9 +39,9 @@ public:
DB::MetadataTransactionPtr createTransaction() override;
const std::string & getPath() const override;
DB::MetadataStorageType getType() const override;
- bool exists(const std::string & path) const override;
- bool isFile(const std::string & path) const override;
- bool isDirectory(const std::string & path) const override;
+ bool existsFile(const std::string & path) const override;
+ bool existsDirectory(const std::string & path) const override;
+ bool existsFileOrDirectory(const std::string & path) const override;
uint64_t getFileSize(const std::string & path) const override;
Poco::Timestamp getLastModified(const std::string & path) const override;
bool supportsChmod() const override;
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp
index 34214594bb..0219dd2629 100644
---
a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp
+++
b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.cpp
@@ -17,6 +17,7 @@
#include <config.h>
#if USE_ROCKSDB
#include "MetadataStorageFromRocksDBTransactionOperations.h"
+
#include <ranges>
namespace DB
@@ -39,7 +40,12 @@ void throwRockDBErrorNotOk(const rocksdb::Status & status)
bool exist(rocksdb::DB & db, const std::string & path)
{
std::string data;
- auto status = db.Get({}, path, &data);
+ return tryGetData(db, path, &data);
+}
+
+bool tryGetData(rocksdb::DB & db, const std::string & path, std::string *
value)
+{
+ auto status = db.Get({}, path, value);
if (status.IsNotFound())
return false;
throwRockDBErrorNotOk(status);
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h
b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h
index 4a246cf7b1..a6c344d842 100644
---
a/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h
+++
b/cpp-ch/local-engine/Disks/ObjectStorages/MetadataStorageFromRocksDBTransactionOperations.h
@@ -25,6 +25,7 @@ namespace local_engine
{
void throwRockDBErrorNotOk(const rocksdb::Status & status);
bool exist(rocksdb::DB & db, const std::string & path);
+bool tryGetData(rocksdb::DB & db, const std::string & path, std::string*
value);
String getData(rocksdb::DB & db, const std::string & path);
std::vector<String> listKeys(rocksdb::DB & db, const std::string & path);
diff --git a/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp
b/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp
index 19a07b06f3..a904fd0876 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp
+++ b/cpp-ch/local-engine/Storages/MergeTree/MetaDataHelper.cpp
@@ -90,7 +90,7 @@ void restoreMetaData<ROCKSDB>(const SparkStorageMergeTreePtr
& storage, const Me
for (const auto & part : mergeTreeTable.getPartNames())
{
auto part_path = table_path / part;
- if (!metadata_storage->exists(part_path))
+ if (!metadata_storage->existsDirectory(part_path))
not_exists_part.emplace(part);
}
@@ -104,7 +104,7 @@ void restoreMetaData<ROCKSDB>(const
SparkStorageMergeTreePtr & storage, const Me
auto s3 = data_disk->getObjectStorage();
auto transaction = metadata_storage->createTransaction();
- if (!metadata_storage->exists(table_path))
+ if (!metadata_storage->existsDirectory(table_path))
transaction->createDirectoryRecursive(table_path.generic_string());
for (const auto & part : not_exists_part)
@@ -112,7 +112,7 @@ void restoreMetaData<ROCKSDB>(const
SparkStorageMergeTreePtr & storage, const Me
auto part_path = table_path / part;
auto metadata_file_path = part_path / METADATA_FILE_NAME;
- if (metadata_storage->exists(part_path))
+ if (metadata_storage->existsDirectory(part_path))
return;
else
transaction->createDirectoryRecursive(part_path);
@@ -143,7 +143,7 @@ void restoreMetaData<LOCAL>(
for (const auto & part : mergeTreeTable.getPartNames())
{
auto part_path = table_path / part;
- if (!metadata_disk->exists(part_path))
+ if (!metadata_disk->existsDirectory(part_path))
not_exists_part.emplace(part);
}
@@ -166,7 +166,7 @@ void restoreMetaData<LOCAL>(
not_exists_part.size());
auto s3 = data_disk->getObjectStorage();
- if (!metadata_disk->exists(table_path))
+ if (!metadata_disk->existsDirectory(table_path))
metadata_disk->createDirectories(table_path.generic_string());
for (const auto & part : not_exists_part)
@@ -176,7 +176,7 @@ void restoreMetaData<LOCAL>(
auto part_path = table_path / part;
auto metadata_file_path = part_path / METADATA_FILE_NAME;
- if (metadata_disk->exists(part_path))
+ if (metadata_disk->existsDirectory(part_path))
return;
else
metadata_disk->createDirectories(part_path);
diff --git a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
index 6b17388113..d85efd03b8 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
+++ b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
@@ -165,7 +165,7 @@ void
SparkStorageMergeTree::prefetchMetaDataFile(std::unordered_set<std::string>
read_settings.remote_fs_method = RemoteFSReadMethod::read;
for (const auto & meta_path : meta_paths)
{
- if (!disk->exists(meta_path))
+ if (!disk->existsDirectory(meta_path))
continue;
auto in = disk->readFile(meta_path, read_settings);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
