This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 71a22da95a [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250115)
(#8537)
71a22da95a is described below
commit 71a22da95adc1fed239c6b292281ba0251d28780
Author: Kyligence Git <[email protected]>
AuthorDate: Wed Jan 15 03:52:33 2025 -0600
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20250115) (#8537)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250115)
* Fix build due to https://github.com/ClickHouse/ClickHouse/pull/73294
* Fix UT due to https://github.com/ClickHouse/ClickHouse/pull/70102
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 ++--
cpp-ch/local-engine/Common/CHUtil.cpp | 16 ++++++++++++++--
.../Storages/MergeTree/MergeSparkMergeTreeTask.cpp | 4 ++--
3 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 79ca4eb69b..45287fab51 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20250110
-CH_COMMIT=eafc5ef70b3
+CH_BRANCH=rebase_ch/20250115
+CH_COMMIT=8e0d5eaf0fc
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp
b/cpp-ch/local-engine/Common/CHUtil.cpp
index ace3a515a2..b2dfa3cf44 100644
--- a/cpp-ch/local-engine/Common/CHUtil.cpp
+++ b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -77,6 +77,10 @@ namespace ServerSetting
extern const ServerSettingsString primary_index_cache_policy;
extern const ServerSettingsUInt64 primary_index_cache_size;
extern const ServerSettingsDouble primary_index_cache_size_ratio;
+extern const ServerSettingsString skipping_index_cache_policy;
+extern const ServerSettingsUInt64 skipping_index_cache_size;
+extern const ServerSettingsUInt64 skipping_index_cache_max_entries;
+extern const ServerSettingsDouble skipping_index_cache_size_ratio;
}
namespace Setting
{
@@ -762,11 +766,11 @@ void
BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config)
global_context->setConfig(config);
auto tmp_path = config->getString("tmp_path",
PathConfig::DEFAULT_TEMP_FILE_PATH);
- if(config->getBool(PathConfig::USE_CURRENT_DIRECTORY_AS_TMP, false))
+ if (config->getBool(PathConfig::USE_CURRENT_DIRECTORY_AS_TMP, false))
{
char buffer[PATH_MAX];
if (getcwd(buffer, sizeof(buffer)) != nullptr)
- tmp_path = std::string(buffer) + tmp_path;
+ tmp_path = std::string(buffer) + tmp_path;
};
global_context->setTemporaryStoragePath(tmp_path, 0);
@@ -805,6 +809,14 @@ void
BackendInitializerUtil::initContexts(DB::Context::ConfigurationPtr config)
double index_mark_cache_size_ratio =
config->getDouble("index_mark_cache_size_ratio",
DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO);
global_context->setIndexMarkCache(index_mark_cache_policy,
index_mark_cache_size, index_mark_cache_size_ratio);
+ String skipping_index_cache_policy =
server_settings[ServerSetting::skipping_index_cache_policy];
+ size_t skipping_index_cache_size =
server_settings[ServerSetting::skipping_index_cache_size];
+ size_t skipping_index_cache_max_entries =
server_settings[ServerSetting::skipping_index_cache_max_entries];
+ double skipping_index_cache_size_ratio =
server_settings[ServerSetting::skipping_index_cache_size_ratio];
+ LOG_INFO(log, "Skipping index cache size to {}",
formatReadableSizeWithBinarySuffix(skipping_index_cache_size));
+ global_context->setSkippingIndexCache(
+ skipping_index_cache_policy, skipping_index_cache_size,
skipping_index_cache_max_entries, skipping_index_cache_size_ratio);
+
size_t mmap_cache_size = config->getUInt64("mmap_cache_size",
DEFAULT_MMAP_CACHE_MAX_SIZE);
global_context->setMMappedFileCache(mmap_cache_size);
diff --git a/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp
b/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp
index ee6930e4de..cb0b3f2054 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp
+++ b/cpp-ch/local-engine/Storages/MergeTree/MergeSparkMergeTreeTask.cpp
@@ -17,6 +17,7 @@
#include "MergeSparkMergeTreeTask.h"
#include <Interpreters/TransactionLog.h>
+#include <Storages/MergeTree/Compaction/CompactionStatistics.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
#include <Storages/MergeTree/SparkStorageMergeTree.h>
@@ -149,8 +150,7 @@ void MergeSparkMergeTreeTask::prepare()
task_context,
//merge_mutate_entry->tagger->reserved_space,
storage.tryReserveSpace(
-
MergeTreeDataMergerMutator::estimateNeededDiskSpace(future_part->parts),
- future_part->parts[0]->getDataPartStorage()),
+ CompactionStatistics::estimateNeededDiskSpace(future_part->parts),
future_part->parts[0]->getDataPartStorage()),
deduplicate,
deduplicate_by_columns,
cleanup,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
