This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 75053f4b35 [GLUTEN][CH][UT] move ut in iceberg dir (#9425)
75053f4b35 is described below
commit 75053f4b35e5a0579129fd9f57722f7118752ef3
Author: jlf <[email protected]>
AuthorDate: Sun Apr 27 12:25:33 2025 +0800
[GLUTEN][CH][UT] move ut in iceberg dir (#9425)
---
backends-clickhouse/pom.xml | 4 +-
.../gluten/execution/iceberg/TestFlinkUpsert.java | 0
.../iceberg/TestPositionDeletesTableGluten.java | 0
.../iceberg/TestPositionDeletesTableGluten.java | 0
.../ClickHouseIcebergHiveTableSupport.scala | 0
.../ClickHouseIcebergHiveTableSupport.scala | 101 ---------------------
6 files changed, 2 insertions(+), 103 deletions(-)
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
index ef066fa5b1..14e3144f95 100644
--- a/backends-clickhouse/pom.xml
+++ b/backends-clickhouse/pom.xml
@@ -90,7 +90,7 @@
<configuration>
<sources>
<source>${project.basedir}/src-spark33/test/scala</source>
- <source>${project.basedir}/src-spark33/test/java</source>
+
<source>${project.basedir}/src-iceberg/test/java-spark33</source>
</sources>
</configuration>
</execution>
@@ -192,7 +192,7 @@
<configuration>
<sources>
<source>${project.basedir}/src-spark35/test/scala</source>
- <source>${project.basedir}/src-spark35/test/java</source>
+
<source>${project.basedir}/src-iceberg/test/java-spark35</source>
</sources>
</configuration>
</execution>
diff --git
a/backends-clickhouse/src-spark33/test/java/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java
b/backends-clickhouse/src-iceberg/test/java-spark33/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java
similarity index 100%
rename from
backends-clickhouse/src-spark33/test/java/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java
rename to
backends-clickhouse/src-iceberg/test/java-spark33/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java
diff --git
a/backends-clickhouse/src-spark33/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
b/backends-clickhouse/src-iceberg/test/java-spark33/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
similarity index 100%
rename from
backends-clickhouse/src-spark33/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
rename to
backends-clickhouse/src-iceberg/test/java-spark33/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
diff --git
a/backends-clickhouse/src-spark35/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
b/backends-clickhouse/src-iceberg/test/java-spark35/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
similarity index 100%
rename from
backends-clickhouse/src-spark35/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
rename to
backends-clickhouse/src-iceberg/test/java-spark35/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
diff --git
a/backends-clickhouse/src-spark33/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
similarity index 100%
rename from
backends-clickhouse/src-spark33/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
rename to
backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
diff --git
a/backends-clickhouse/src-spark35/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
b/backends-clickhouse/src-spark35/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
deleted file mode 100644
index c9e6a8ee5e..0000000000
---
a/backends-clickhouse/src-spark35/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gluten.execution.iceberg
-
-import com.google.common.base.Strings
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.SparkSession
-import
org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig
-
-class ClickHouseIcebergHiveTableSupport {
-
- private val sparkConf: SparkConf = new SparkConf()
-
- private var _hiveSpark: SparkSession = _
-
- def spark: SparkSession = _hiveSpark
-
- def initSparkConf(url: String, catalog: String, path: String): SparkConf = {
- import org.apache.gluten.backendsapi.clickhouse.CHConfig._
-
- sparkConf
- .set("spark.plugins", "org.apache.gluten.GlutenPlugin")
- .set("spark.memory.offHeap.enabled", "true")
- .set("spark.memory.offHeap.size", "536870912")
- .set("spark.sql.catalogImplementation", "hive")
- .set("spark.sql.adaptive.enabled", "true")
- .set("spark.sql.files.maxPartitionBytes", "1g")
- .set("spark.serializer", "org.apache.spark.serializer.JavaSerializer")
- .set("spark.sql.shuffle.partitions", "5")
- .set("spark.sql.adaptive.enabled", "false")
- .set("spark.sql.files.minPartitionNum", "1")
- .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1")
- .set("spark.gluten.sql.columnar.iterator", "true")
- .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true")
- .set("spark.gluten.sql.enable.native.validation", "false")
- .set("spark.gluten.sql.parquet.maxmin.index", "true")
- .set("spark.hive.exec.dynamic.partition.mode", "nonstrict")
- .set("spark.gluten.supported.hive.udfs", "my_add")
- .set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
- .set("spark.sql.adaptive.enabled", "true")
- .set("spark.sql.shuffle.partitions", "2")
- .set("spark.memory.offHeap.size", "2g")
- .set("spark.unsafe.exceptionOnMemoryLeak", "true")
- .set("spark.sql.autoBroadcastJoinThreshold", "-1")
- .setCHConfig("use_local_format", true)
- .set("spark.sql.extensions",
- "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
- .set("spark.sql.catalog.spark_catalog",
"org.apache.iceberg.spark.SparkSessionCatalog")
- .set("spark.sql.catalog.spark_catalog.type", "hive")
- .setMaster("local[*]")
- if (!Strings.isNullOrEmpty(url)) {
- sparkConf.set("spark.hadoop.hive.metastore.uris", url)
- }
- if (!Strings.isNullOrEmpty(catalog)) {
- sparkConf.set("spark.sql.catalog." + catalog,
"org.apache.iceberg.spark.SparkCatalog")
- .set("spark.sql.catalog." + catalog + ".type", "hive")
- }
- if (!Strings.isNullOrEmpty(path)) {
- sparkConf.set("spark.sql.warehouse.dir", path)
- }
- sparkConf
- }
-
- def initializeSession(): Unit = {
- if (_hiveSpark == null) {
- _hiveSpark =
- SparkSession
- .builder()
- .config(sparkConf)
- .enableHiveSupport()
- .getOrCreate()
- }
- }
-
- def clean(): Unit = {
- try {
- if (_hiveSpark != null) {
- _hiveSpark.stop()
- _hiveSpark = null
- }
- } finally {
- SparkSession.clearActiveSession()
- SparkSession.clearDefaultSession()
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
