This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6ec90374c2 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241018)
(#7588)
6ec90374c2 is described below
commit 6ec90374c2ffdac2db1bad35871c970378bffef7
Author: Kyligence Git <[email protected]>
AuthorDate: Fri Oct 18 02:18:34 2024 -0500
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20241018) (#7588)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241018)
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/69862
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 ++--
cpp-ch/local-engine/Common/CHUtil.cpp | 3 +++
.../Storages/SubstraitSource/ParquetFormatFile.cpp | 19 +++++++++++++++++--
cpp-ch/local-engine/tests/benchmark_parquet_read.cpp | 4 ++--
cpp-ch/local-engine/tests/benchmark_spark_row.cpp | 2 +-
cpp-ch/local-engine/tests/gtest_parquet_read.cpp | 2 +-
6 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 65feb5e57c..565b9c2bc4 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20241017
-CH_COMMIT=afe3026c6ce
\ No newline at end of file
+CH_BRANCH=rebase_ch/20241018
+CH_COMMIT=6f7cbecdc1f
\ No newline at end of file
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp
b/cpp-ch/local-engine/Common/CHUtil.cpp
index 44d86168d6..369dceb61a 100644
--- a/cpp-ch/local-engine/Common/CHUtil.cpp
+++ b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -709,6 +709,9 @@ void BackendInitializerUtil::initSettings(const
std::map<std::string, std::strin
settings.set(MERGETREE_INSERT_WITHOUT_LOCAL_STORAGE, false);
settings.set(DECIMAL_OPERATIONS_ALLOW_PREC_LOSS, true);
settings.set("remote_filesystem_read_prefetch", false);
+ settings.set("max_parsing_threads", 1);
+ settings.set("max_download_threads", 1);
+ settings.set("input_format_parquet_enable_row_group_prefetch", false);
for (const auto & [key, value] : spark_conf_map)
{
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
index f557df5b27..fdd7574a06 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
@@ -22,6 +22,7 @@
#include <numeric>
#include <utility>
+#include <Core/Settings.h>
#include <DataTypes/DataTypeNullable.h>
#include <Formats/FormatFactory.h>
#include <Formats/FormatSettings.h>
@@ -30,13 +31,18 @@
#include <Processors/Formats/Impl/ArrowColumnToCHColumn.h>
#include <Processors/Formats/Impl/ParquetBlockInputFormat.h>
#include <Storages/Parquet/VectorizedParquetRecordReader.h>
-#include <Storages/SubstraitSource/SubstraitFileSourceStep.h>
#include <parquet/arrow/reader.h>
#include <parquet/metadata.h>
#include <Common/Exception.h>
namespace DB
{
+namespace Setting
+{
+extern const SettingsMaxThreads max_download_threads;
+extern const SettingsMaxThreads max_parsing_threads;
+}
+
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
@@ -86,10 +92,19 @@ FormatFile::InputFormatPtr
ParquetFormatFile::createInputFormat(const DB::Block
std::ranges::set_difference(total_row_group_indices,
required_row_group_indices, std::back_inserter(skip_row_group_indices));
format_settings.parquet.skip_row_groups =
std::unordered_set<int>(skip_row_group_indices.begin(),
skip_row_group_indices.end());
+
+ const DB::Settings & settings = context->getSettingsRef();
+
if (use_pageindex_reader && pageindex_reader_support(header))
res->input =
std::make_shared<VectorizedParquetBlockInputFormat>(*(res->read_buffer),
header, format_settings);
else
- res->input =
std::make_shared<DB::ParquetBlockInputFormat>(*(res->read_buffer), header,
format_settings, 1, 8192);
+ res->input = std::make_shared<DB::ParquetBlockInputFormat>(
+ *(res->read_buffer),
+ header,
+ format_settings,
+ settings[DB::Setting::max_parsing_threads],
+ settings[DB::Setting::max_download_threads],
+ 8192);
return res;
}
diff --git a/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
b/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
index 8ed7c10840..0cb9df3c7e 100644
--- a/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
@@ -69,7 +69,7 @@ void BM_ColumnIndexRead_Old(benchmark::State & state)
for (auto _ : state)
{
auto in = std::make_unique<ReadBufferFromFile>(file);
- auto format = std::make_shared<ParquetBlockInputFormat>(*in, header,
format_settings, 1, 8192);
+ auto format = std::make_shared<ParquetBlockInputFormat>(*in, header,
format_settings, 1, 1, 8192);
auto pipeline = QueryPipeline(std::move(format));
auto reader = std::make_unique<PullingPipelineExecutor>(pipeline);
while (reader->pull(res))
@@ -93,7 +93,7 @@ void BM_ParquetReadDate32(benchmark::State & state)
for (auto _ : state)
{
auto in = std::make_unique<ReadBufferFromFile>(file);
- auto format = std::make_shared<ParquetBlockInputFormat>(*in, header,
format_settings, 1, 8192);
+ auto format = std::make_shared<ParquetBlockInputFormat>(*in, header,
format_settings, 1, 1, 8192);
auto pipeline = QueryPipeline(std::move(format));
auto reader = std::make_unique<PullingPipelineExecutor>(pipeline);
while (reader->pull(res))
diff --git a/cpp-ch/local-engine/tests/benchmark_spark_row.cpp
b/cpp-ch/local-engine/tests/benchmark_spark_row.cpp
index 28e11a7bad..b8592a0d15 100644
--- a/cpp-ch/local-engine/tests/benchmark_spark_row.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_spark_row.cpp
@@ -56,7 +56,7 @@ static void readParquetFile(const Block & header, const
String & file, Block & b
{
auto in = std::make_unique<ReadBufferFromFile>(file);
FormatSettings format_settings;
- auto format = std::make_shared<ParquetBlockInputFormat>(*in, header,
format_settings, 1, 8192);
+ auto format = std::make_shared<ParquetBlockInputFormat>(*in, header,
format_settings, 1, 1, 8192);
auto pipeline = QueryPipeline(std::move(format));
auto reader = std::make_unique<PullingPipelineExecutor>(pipeline);
while (reader->pull(block))
diff --git a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
index 1cbe34319d..3421d7f78b 100644
--- a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
+++ b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
@@ -109,7 +109,7 @@ void readData(const String & path, const std::map<String,
Field> & fields)
InputFormatPtr format;
if constexpr (std::is_same_v<InputFormat, DB::ParquetBlockInputFormat>)
- format = std::make_shared<InputFormat>(in, header, settings, 1, 8192);
+ format = std::make_shared<InputFormat>(in, header, settings, 1, 1,
8192);
else
format = std::make_shared<InputFormat>(in, header, settings);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
