This is an automated email from the ASF dual-hosted git repository.
liuneng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 046c13260 [CH] Fix CH CI #5493
046c13260 is described below
commit 046c13260e99f1481d77421d8e66c511ae6c9ef4
Author: Hongze Zhang <[email protected]>
AuthorDate: Tue Apr 23 14:10:01 2024 +0800
[CH] Fix CH CI #5493
There is a CI error related to #5482 but was not detected since CH CI was
broken at that time. The patch fixes the error..
---
.../scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala | 2 ++
.../main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala | 1 +
2 files changed, 3 insertions(+)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
index fcc3c333e..952812d68 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
@@ -27,6 +27,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.sql.execution.datasources.v1._
+import org.apache.spark.util.SparkDirectoryUtil
import org.apache.commons.lang3.StringUtils
@@ -43,6 +44,7 @@ class CHListenerApi extends ListenerApi with Logging {
override def onExecutorShutdown(): Unit = shutdown()
private def initialize(conf: SparkConf, isDriver: Boolean): Unit = {
+ SparkDirectoryUtil.init(conf)
val libPath = conf.get(GlutenConfig.GLUTEN_LIB_PATH, StringUtils.EMPTY)
if (StringUtils.isBlank(libPath)) {
throw new IllegalArgumentException(
diff --git
a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
index edecca56d..3a80e18bd 100644
---
a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
@@ -46,6 +46,7 @@ class CHColumnarShuffleWriter[K, V](
private val blockManager = SparkEnv.get.blockManager
private val localDirs = SparkDirectoryUtil
+ .get()
.namespace("ch-shuffle-write")
.mkChildDirs(UUID.randomUUID().toString)
.map(_.getAbsolutePath)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
