This is an automated email from the ASF dual-hosted git repository.
richox pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 92af4e0c [AURON #1480] Add auron-iceberg project #1480 (#1486)
92af4e0c is described below
commit 92af4e0ca7bfc155b16e352e628558cb172f5a0d
Author: guixiaowen <[email protected]>
AuthorDate: Tue Oct 28 14:16:49 2025 +0800
[AURON #1480] Add auron-iceberg project #1480 (#1486)
* [AURON #1480]add auron-iceberg project #1480
* [AURON #1480]add auron-iceberg project #1480 #1486
* [AURON #1480]add auron-iceberg project #1480
---------
Co-authored-by: guihuawen <[email protected]>
---
auron-build.sh | 34 +++++++++-
dev/reformat | 2 +-
pom.xml | 16 +++++
thirdparty/auron-iceberg/pom.xml | 75 ++++++++++++++++++++++
.../iceberg/AuronIcebergIntegrationSuite.scala | 34 ++++++++++
.../auron/iceberg/BaseAuronIcebergSuite.scala | 34 ++++++++++
6 files changed, 192 insertions(+), 3 deletions(-)
diff --git a/auron-build.sh b/auron-build.sh
index 054a3b62..87a74487 100755
--- a/auron-build.sh
+++ b/auron-build.sh
@@ -37,6 +37,7 @@ SUPPORTED_CELEBORN_VERSIONS=("0.5" "0.6")
SUPPORTED_UNIFFLE_VERSIONS=("0.10")
SUPPORTED_PAIMON_VERSIONS=("1.2")
SUPPORTED_FLINK_VERSIONS=("1.18")
+SUPPORTED_ICEBERG_VERSIONS=("1.9")
# -----------------------------------------------------------------------------
# Function: print_help
@@ -60,6 +61,8 @@ print_help() {
IFS=','; echo " --celeborn <VERSION> Specify Celeborn version (e.g.
${SUPPORTED_CELEBORN_VERSIONS[*]})"; unset IFS
IFS=','; echo " --uniffle <VERSION> Specify Uniffle version (e.g.
${SUPPORTED_UNIFFLE_VERSIONS[*]})"; unset IFS
IFS=','; echo " --paimon <VERSION> Specify Paimon version (e.g.
${SUPPORTED_PAIMON_VERSIONS[*]})"; unset IFS
+ IFS=','; echo " --iceberg <VERSION> Specify Iceberg version (e.g.
${SUPPORTED_ICEBERG_VERSIONS[*]})"; unset IFS
+
echo " -h, --help Show this help message"
echo
echo "Examples:"
@@ -72,7 +75,8 @@ print_help() {
"--scalaver ${SUPPORTED_SCALA_VERSIONS[*]: -1}" \
"--celeborn ${SUPPORTED_CELEBORN_VERSIONS[*]: -1}" \
"--uniffle ${SUPPORTED_UNIFFLE_VERSIONS[*]: -1}" \
- "--paimon ${SUPPORTED_PAIMON_VERSIONS[*]: -1}"
+ "--paimon ${SUPPORTED_PAIMON_VERSIONS[*]: -1}" \
+ "--iceberg ${SUPPORTED_ICEBERG_VERSIONS[*]: -1}"
exit 0
}
@@ -126,6 +130,7 @@ SCALA_VER=""
CELEBORN_VER=""
UNIFFLE_VER=""
PAIMON_VER=""
+ICEBERG_VER=""
# -----------------------------------------------------------------------------
# Section: Argument Parsing
@@ -247,6 +252,27 @@ while [[ $# -gt 0 ]]; do
exit 1
fi
;;
+ --iceberg)
+ if [[ -n "$2" && "$2" != -* ]]; then
+ ICEBERG_VER="$2"
+ if ! check_supported_version "$ICEBERG_VER"
"${SUPPORTED_ICEBERG_VERSIONS[@]}"; then
+ print_invalid_option_error Iceberg "$ICEBERG_VER"
"${SUPPORTED_ICEBERG_VERSIONS[@]}"
+ fi
+ if [ -z "$SPARK_VER" ]; then
+ echo "ERROR: Building iceberg requires spark at the same
time, and only Spark versions 3.4 or 3.5 are supported."
+ exit 1
+ fi
+ if [ "$SPARK_VER" != "3.4" ] && [ "$SPARK_VER" != "3.5" ]; then
+ echo "ERROR: Building iceberg requires spark versions are
3.4 or 3.5."
+ exit 1
+ fi
+ shift 2
+ else
+ IFS=','; echo "ERROR: Missing argument for --iceberg," \
+ "specify one of: ${SUPPORTED_ICEBERG_VERSIONS[*]}" >&2; unset
IFS
+ exit 1
+ fi
+ ;;
--flinkver)
if [[ -n "$2" && "$2" != -* ]]; then
FLINK_VER="$2"
@@ -351,6 +377,9 @@ fi
if [[ -n "$FLINK_VER" ]]; then
BUILD_ARGS+=("-Pflink-$FLINK_VER")
fi
+if [[ -n "$ICEBERG_VER" ]]; then
+ BUILD_ARGS+=("-Piceberg-$ICEBERG_VER")
+fi
MVN_ARGS=("${CLEAN_ARGS[@]}" "${BUILD_ARGS[@]}")
@@ -376,6 +405,7 @@ RUST_VERSION=$(rustc --version | awk '{print $2}')
echo "uniffle.version=${UNIFFLE_VER}"
echo "paimon.version=${PAIMON_VER}"
echo "flink.version=${FLINK_VER}"
+ echo "iceberg.version=${ICEBERG_VER}"
echo "build.timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
} > "$BUILD_INFO_FILE"
@@ -419,4 +449,4 @@ if [[ "$USE_DOCKER" == true ]]; then
else
echo "[INFO] Compiling locally with maven args: $MVN_CMD ${MVN_ARGS[@]} $@"
"$MVN_CMD" "${MVN_ARGS[@]}" "$@"
-fi
+fi
\ No newline at end of file
diff --git a/dev/reformat b/dev/reformat
index e1793c8a..90a25dd6 100755
--- a/dev/reformat
+++ b/dev/reformat
@@ -44,7 +44,7 @@ function run_maven() {
sparkver=spark-3.5
for celebornver in celeborn-0.5 celeborn-0.6
do
- run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10
-Ppaimon,paimon-1.2 -Pflink,flink-1.18
+ run_maven -P"${sparkver}" -Pceleborn,"${celebornver}" -Puniffle,uniffle-0.10
-Ppaimon,paimon-1.2 -Pflink,flink-1.18 -Piceberg,iceberg-1.9
done
diff --git a/pom.xml b/pom.xml
index 7c295bb5..775f6176 100644
--- a/pom.xml
+++ b/pom.xml
@@ -568,6 +568,7 @@
<shimPkg>spark-extension-shims-spark</shimPkg>
<scalaTestVersion>3.0.8</scalaTestVersion>
<sparkVersion>3.0.3</sparkVersion>
+ <shortSparkVersion>3.0</shortSparkVersion>
</properties>
</profile>
@@ -578,6 +579,7 @@
<shimPkg>spark-extension-shims-spark</shimPkg>
<scalaTestVersion>3.2.9</scalaTestVersion>
<sparkVersion>3.1.3</sparkVersion>
+ <shortSparkVersion>3.1</shortSparkVersion>
</properties>
</profile>
@@ -588,6 +590,7 @@
<shimPkg>spark-extension-shims-spark</shimPkg>
<scalaTestVersion>3.2.9</scalaTestVersion>
<sparkVersion>3.2.4</sparkVersion>
+ <shortSparkVersion>3.2</shortSparkVersion>
</properties>
</profile>
@@ -598,6 +601,7 @@
<shimPkg>spark-extension-shims-spark</shimPkg>
<scalaTestVersion>3.2.9</scalaTestVersion>
<sparkVersion>3.3.4</sparkVersion>
+ <shortSparkVersion>3.3</shortSparkVersion>
</properties>
</profile>
@@ -608,6 +612,7 @@
<shimPkg>spark-extension-shims-spark</shimPkg>
<scalaTestVersion>3.2.9</scalaTestVersion>
<sparkVersion>3.4.4</sparkVersion>
+ <shortSparkVersion>3.4</shortSparkVersion>
</properties>
</profile>
@@ -618,6 +623,7 @@
<shimPkg>spark-extension-shims-spark</shimPkg>
<scalaTestVersion>3.2.9</scalaTestVersion>
<sparkVersion>3.5.7</sparkVersion>
+ <shortSparkVersion>3.5</shortSparkVersion>
</properties>
</profile>
@@ -792,5 +798,15 @@
<flink.version>1.18.1</flink.version>
</properties>
</profile>
+
+ <profile>
+ <id>iceberg-1.9</id>
+ <modules>
+ <module>thirdparty/auron-iceberg</module>
+ </modules>
+ <properties>
+ <icebergVersion>1.9.2</icebergVersion>
+ </properties>
+ </profile>
</profiles>
</project>
diff --git a/thirdparty/auron-iceberg/pom.xml b/thirdparty/auron-iceberg/pom.xml
new file mode 100644
index 00000000..44b247ae
--- /dev/null
+++ b/thirdparty/auron-iceberg/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.auron</groupId>
+ <artifactId>auron-parent_${scalaVersion}</artifactId>
+ <version>${project.version}</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>auron-iceberg_${scalaVersion}</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Auron Iceberg ${icebergVersion}</name>
+ <description>Apache Auron Iceberg ${icebergVersion}
${scalaVersion}</description>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+
<artifactId>iceberg-spark-runtime-${shortSparkVersion}_${scalaVersion}</artifactId>
+ <version>${icebergVersion}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-catalyst_${scalaVersion}</artifactId>
+ <version>${sparkVersion}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-hive_${scalaVersion}</artifactId>
+ <version>${sparkVersion}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scalaVersion}</artifactId>
+ <version>${sparkVersion}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scalaVersion}</artifactId>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-catalyst_${scalaVersion}</artifactId>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scalaVersion}</artifactId>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git
a/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
new file mode 100644
index 00000000..d16d47b4
--- /dev/null
+++
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.iceberg
+
+import org.apache.spark.sql.Row
+
+class AuronIcebergIntegrationSuite
+ extends org.apache.spark.sql.QueryTest
+ with BaseAuronIcebergSuite {
+
+ test("test iceberg integrate ") {
+ withTable("local.db.t1") {
+ sql(
+ "create table local.db.t1 using iceberg PARTITIONED BY (part) as
select 1 as c1, 2 as c2, 'test test' as part")
+ val df = sql("select * from local.db.t1")
+ checkAnswer(df, Seq(Row(1, 2, "test test")))
+ }
+ }
+
+}
diff --git
a/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/BaseAuronIcebergSuite.scala
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/BaseAuronIcebergSuite.scala
new file mode 100644
index 00000000..64e772ad
--- /dev/null
+++
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/BaseAuronIcebergSuite.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.iceberg
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+trait BaseAuronIcebergSuite extends SharedSparkSession {
+
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(
+ "spark.sql.extensions",
+ "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
+ .set("spark.sql.catalog.spark_catalog",
"org.apache.iceberg.spark.SparkSessionCatalog")
+ .set("spark.sql.catalog.local", "org.apache.iceberg.spark.SparkCatalog")
+ .set("spark.sql.catalog.local.type", "hadoop")
+ .set("spark.sql.catalog.local.warehouse", "iceberg_warehouse")
+ }
+}
