This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ca528c577a [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241203)
(#8125)
ca528c577a is described below
commit ca528c577a4ce8f9fa91678de219ac9353b308a5
Author: Kyligence Git <[email protected]>
AuthorDate: Tue Dec 3 04:54:35 2024 -0600
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20241203) (#8125)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241203)
* Fix build due to https://github.com/ClickHouse/ClickHouse/pull/72651
* fix in clang19
* Fix UT due to https://github.com/ClickHouse/ClickHouse/pull/72651
* remove last '/' for HDFS_METADATA_PATH
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
.../execution/GlutenClickHouseWholeStageTransformerSuite.scala | 4 ++--
.../mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala | 10 +++++-----
cpp-ch/clickhouse.version | 4 ++--
.../Functions/SparkFunctionDecimalBinaryArithmetic.cpp | 2 +-
.../local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp | 1 +
.../local-engine/Storages/MergeTree/StorageMergeTreeFactory.h | 7 +++----
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
index 6ed106e2c7..d7b277edfa 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala
@@ -42,14 +42,14 @@ class GlutenClickHouseWholeStageTransformerSuite extends
WholeStageTransformerSu
}
val SPARK_DIR_NAME: String = sparkVersion.replace(".", "-")
- val S3_METADATA_PATH = s"/tmp/metadata/s3/$SPARK_DIR_NAME/"
+ val S3_METADATA_PATH = s"/tmp/metadata/s3/$SPARK_DIR_NAME"
val S3_CACHE_PATH = s"/tmp/s3_cache/$SPARK_DIR_NAME/"
val S3_ENDPOINT = "s3://127.0.0.1:9000/"
val MINIO_ENDPOINT: String = S3_ENDPOINT.replace("s3", "http")
val BUCKET_NAME: String = SPARK_DIR_NAME
val WHOLE_PATH: String = MINIO_ENDPOINT + BUCKET_NAME + "/"
- val HDFS_METADATA_PATH = s"/tmp/metadata/hdfs/$SPARK_DIR_NAME/"
+ val HDFS_METADATA_PATH = s"/tmp/metadata/hdfs/$SPARK_DIR_NAME"
val HDFS_CACHE_PATH = s"/tmp/hdfs_cache/$SPARK_DIR_NAME/"
val HDFS_URL_ENDPOINT = "hdfs://127.0.0.1:8020"
val HDFS_URL = s"$HDFS_URL_ENDPOINT/$SPARK_DIR_NAME"
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala
index 14a8c68bdb..a01c708ada 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala
@@ -134,7 +134,7 @@ class GlutenClickHouseMergeTreeCacheDataSuite
| aaa='ccc')""".stripMargin)
.collect()
assertResult(true)(res(0).getBoolean(0))
- val metaPath = new File(HDFS_METADATA_PATH +
s"$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
+ val metaPath = new
File(s"$HDFS_METADATA_PATH/$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
assertResult(true)(metaPath.exists() && metaPath.isDirectory)
assertResult(22)(metaPath.list().length)
assert(countFiles(dataPath) > initial_cache_files)
@@ -238,7 +238,7 @@ class GlutenClickHouseMergeTreeCacheDataSuite
| aaa='ccc')""".stripMargin)
.collect()
assertResult(true)(res(0).getBoolean(0))
- val metaPath = new File(HDFS_METADATA_PATH +
s"$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
+ val metaPath = new
File(s"$HDFS_METADATA_PATH/$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
assertResult(true)(metaPath.exists() && metaPath.isDirectory)
eventually(timeout(60.seconds), interval(2.seconds)) {
assertResult(22)(metaPath.list().length)
@@ -346,7 +346,7 @@ class GlutenClickHouseMergeTreeCacheDataSuite
| aaa='ccc')""".stripMargin)
.collect()
assertResult(true)(res(0).getBoolean(0))
- val metaPath = new File(HDFS_METADATA_PATH +
s"$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
+ val metaPath = new
File(s"$HDFS_METADATA_PATH/$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
assertResult(true)(metaPath.exists() && metaPath.isDirectory)
assertResult(22)(metaPath.list().length)
assert(countFiles(dataPath) > initial_cache_files)
@@ -439,7 +439,7 @@ class GlutenClickHouseMergeTreeCacheDataSuite
val dataPath = new File(HDFS_CACHE_PATH)
val initial_cache_files = countFiles(dataPath)
- val metaPath = new File(HDFS_METADATA_PATH +
s"$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
+ val metaPath = new
File(s"$HDFS_METADATA_PATH/$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
val res1 = spark.sql(s"cache data select * from
lineitem_mergetree_hdfs").collect()
assertResult(true)(res1(0).getBoolean(0))
assertResult(1)(metaPath.list().length)
@@ -539,7 +539,7 @@ class GlutenClickHouseMergeTreeCacheDataSuite
| aaa='ccc')""".stripMargin)
.collect()
assertResult(true)(res(0).getBoolean(0))
- val metaPath = new File(HDFS_METADATA_PATH +
s"$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
+ val metaPath = new
File(s"$HDFS_METADATA_PATH/$SPARK_DIR_NAME/test/lineitem_mergetree_hdfs")
assertResult(true)(metaPath.exists() && metaPath.isDirectory)
assertResult(22)(metaPath.list().length)
assert(countFiles(dataPath) > initial_cache_files)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 565220d786..c3fda4bd2f 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20241130
-CH_COMMIT=d5d38588bd3
+CH_BRANCH=rebase_ch/20241203
+CH_COMMIT=473be0b9c50
diff --git
a/cpp-ch/local-engine/Functions/SparkFunctionDecimalBinaryArithmetic.cpp
b/cpp-ch/local-engine/Functions/SparkFunctionDecimalBinaryArithmetic.cpp
index f89943fc7a..8b79bae90d 100644
--- a/cpp-ch/local-engine/Functions/SparkFunctionDecimalBinaryArithmetic.cpp
+++ b/cpp-ch/local-engine/Functions/SparkFunctionDecimalBinaryArithmetic.cpp
@@ -487,7 +487,7 @@ public:
right_generic,
removeNullable(arguments[2].type).get(),
[&](const auto & left, const auto & right, const auto & result) {
- return (res = SparkDecimalBinaryOperation<Operation,
Mode>::template executeDecimal(arguments, left, right, result))
+ return (res = SparkDecimalBinaryOperation<Operation,
Mode>::template executeDecimal<>(arguments, left, right, result))
!= nullptr;
});
diff --git a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
index 5669489f54..cbb40ae89d 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
+++ b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.cpp
@@ -502,6 +502,7 @@ MergeTreeDataWriter::TemporaryPart
SparkMergeTreeDataWriter::writeTempPart(
compression_codec,
index_granularity_ptr,
txn ? txn->tid : Tx::PrehistoricTID,
+ block.bytes(),
false,
false,
context->getWriteSettings());
diff --git a/cpp-ch/local-engine/Storages/MergeTree/StorageMergeTreeFactory.h
b/cpp-ch/local-engine/Storages/MergeTree/StorageMergeTreeFactory.h
index 2c76fe51ed..f6bc6cda4f 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/StorageMergeTreeFactory.h
+++ b/cpp-ch/local-engine/Storages/MergeTree/StorageMergeTreeFactory.h
@@ -30,8 +30,8 @@ class DataPartStorageHolder
{
public:
DataPartStorageHolder(const DataPartPtr& data_part, const
SparkStorageMergeTreePtr& storage)
- : data_part_(data_part),
- storage_(storage)
+ : storage_(storage),
+ data_part_(data_part)
{
}
@@ -48,12 +48,11 @@ public:
~DataPartStorageHolder()
{
storage_->removePartFromMemory(*data_part_);
- // std::cerr << fmt::format("clean part {}", data_part_->name) <<
std::endl;
}
private:
- DataPartPtr data_part_;
SparkStorageMergeTreePtr storage_;
+ DataPartPtr data_part_;
};
using DataPartStorageHolderPtr = std::shared_ptr<DataPartStorageHolder>;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
