This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 82abc21ae [spark] Support scala 2.13 (#4302)
82abc21ae is described below
commit 82abc21ae4847a7e800cb8db6375a49da21cc75c
Author: Zouxxyy <[email protected]>
AuthorDate: Sat Oct 12 15:02:50 2024 +0800
[spark] Support scala 2.13 (#4302)
---
...se-flink-jdk11.yml => utitcase-flink-jdk11.yml} | 0
.../{unitcase-jdk11.yml => utitcase-jdk11.yml} | 9 +++-
.../{utitcase.yml => utitcase-spark-scala2.13.yml} | 15 ++++---
.../workflows/{utitcase.yml => utitcase-spark.yml} | 13 ++++--
.github/workflows/utitcase.yml | 9 +++-
paimon-codegen/pom.xml | 10 +++--
paimon-e2e-tests/pom.xml | 2 +-
paimon-flink/paimon-flink-1.15/pom.xml | 4 +-
paimon-flink/paimon-flink-1.16/pom.xml | 2 +-
paimon-flink/paimon-flink-1.17/pom.xml | 2 +-
paimon-flink/paimon-flink-1.18/pom.xml | 2 +-
paimon-flink/paimon-flink-1.19/pom.xml | 2 +-
paimon-flink/paimon-flink-cdc/pom.xml | 4 +-
paimon-flink/paimon-flink-common/pom.xml | 4 +-
paimon-hive/paimon-hive-connector-2.3/pom.xml | 2 +-
paimon-hive/paimon-hive-connector-3.1/pom.xml | 2 +-
paimon-hive/paimon-hive-connector-common/pom.xml | 4 +-
paimon-spark/paimon-spark-3.2/pom.xml | 48 ++++++++--------------
paimon-spark/paimon-spark-3.3/pom.xml | 48 ++++++++--------------
paimon-spark/paimon-spark-3.4/pom.xml | 48 ++++++++--------------
paimon-spark/paimon-spark-3.5/pom.xml | 24 ++++++++---
paimon-spark/paimon-spark-common/pom.xml | 46 +++++++++------------
.../org/apache/paimon/spark/PaimonStatistics.scala | 4 +-
.../scala/org/apache/paimon/spark/ScanHelper.scala | 2 +-
.../catalyst/analysis/PaimonMergeIntoBase.scala | 2 +-
.../catalyst/analysis/PaimonUpdateTable.scala | 2 +-
.../optimizer/EvalSubqueriesForDeleteTable.scala | 5 ++-
.../MergePaimonScalarSubqueriesBase.scala | 2 +-
.../commands/DeleteFromPaimonTableCommand.scala | 2 +-
.../spark/commands/MergeIntoPaimonTable.scala | 2 +-
.../commands/PaimonAnalyzeTableColumnCommand.scala | 6 +--
.../paimon/spark/commands/PaimonCommand.scala | 4 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 4 +-
.../paimon/spark/commands/SparkDataFileMeta.scala | 2 +-
.../spark/orphan/SparkOrphanFilesClean.scala | 4 +-
.../spark/statistics/StatisticsHelperBase.scala | 4 +-
.../org/apache/paimon/spark/util/OptionUtils.scala | 1 +
.../extensions/PaimonSqlExtensionsAstBuilder.scala | 6 +--
.../apache/paimon/spark/SparkInternalRowTest.java | 10 ++++-
.../paimon/spark/sql/AnalyzeTableTestBase.scala | 14 +++----
pom.xml | 23 +++++++++--
41 files changed, 208 insertions(+), 191 deletions(-)
diff --git a/.github/workflows/unitcase-flink-jdk11.yml
b/.github/workflows/utitcase-flink-jdk11.yml
similarity index 100%
rename from .github/workflows/unitcase-flink-jdk11.yml
rename to .github/workflows/utitcase-flink-jdk11.yml
diff --git a/.github/workflows/unitcase-jdk11.yml
b/.github/workflows/utitcase-jdk11.yml
similarity index 80%
rename from .github/workflows/unitcase-jdk11.yml
rename to .github/workflows/utitcase-jdk11.yml
index 1baed87f9..f1d7c25cb 100644
--- a/.github/workflows/unitcase-jdk11.yml
+++ b/.github/workflows/utitcase-jdk11.yml
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-name: UTCase and ITCase Non Flink on JDK 11
+name: UTCase and ITCase Others on JDK 11
on:
issue_comment:
@@ -52,6 +52,11 @@ jobs:
. .github/workflows/utils.sh
jvm_timezone=$(random_timezone)
echo "JVM timezone is set to $jvm_timezone"
- mvn -T 1C -B clean install -pl
'!paimon-e2e-tests,!org.apache.paimon:paimon-hive-connector-3.1'
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
+
test_modules="!paimon-e2e-tests,!org.apache.paimon:paimon-hive-connector-3.1,"
+ for suffix in 3.5 3.4 3.3 3.2 common; do
+ test_modules+="!org.apache.paimon:paimon-spark-${suffix},"
+ done
+ test_modules="${test_modules%,}"
+ mvn -T 1C -B clean install -pl "${test_modules}"
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
env:
MAVEN_OPTS: -Xmx4096m
\ No newline at end of file
diff --git a/.github/workflows/utitcase.yml
b/.github/workflows/utitcase-spark-scala2.13.yml
similarity index 78%
copy from .github/workflows/utitcase.yml
copy to .github/workflows/utitcase-spark-scala2.13.yml
index 431b44332..05ee066c9 100644
--- a/.github/workflows/utitcase.yml
+++ b/.github/workflows/utitcase-spark-scala2.13.yml
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-name: UTCase and ITCase Non Flink
+name: UTCase and ITCase Spark on Scala 2.13
on:
push:
@@ -44,15 +44,20 @@ jobs:
with:
java-version: ${{ env.JDK_VERSION }}
distribution: 'adopt'
- - name: Build
- run: mvn -T 1C -B clean install -DskipTests
- - name: Test
+ - name: Build Spark
+ run: mvn -T 1C -B clean install -DskipTests -Pscala-2.13
+ - name: Test Spark
timeout-minutes: 60
run: |
# run tests with random timezone to find out timezone related bugs
. .github/workflows/utils.sh
jvm_timezone=$(random_timezone)
echo "JVM timezone is set to $jvm_timezone"
- mvn -T 1C -B clean install -pl '!paimon-e2e-tests'
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
+ test_modules=""
+ for suffix in common 3.5 3.4 3.3 3.2; do
+ test_modules+="org.apache.paimon:paimon-spark-${suffix},"
+ done
+ test_modules="${test_modules%,}"
+ mvn -T 1C -B test -pl "${test_modules}"
-Duser.timezone=$jvm_timezone -Pscala-2.13
env:
MAVEN_OPTS: -Xmx4096m
\ No newline at end of file
diff --git a/.github/workflows/utitcase.yml
b/.github/workflows/utitcase-spark.yml
similarity index 82%
copy from .github/workflows/utitcase.yml
copy to .github/workflows/utitcase-spark.yml
index 431b44332..0561b3857 100644
--- a/.github/workflows/utitcase.yml
+++ b/.github/workflows/utitcase-spark.yml
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-name: UTCase and ITCase Non Flink
+name: UTCase and ITCase Spark
on:
push:
@@ -44,15 +44,20 @@ jobs:
with:
java-version: ${{ env.JDK_VERSION }}
distribution: 'adopt'
- - name: Build
+ - name: Build Spark
run: mvn -T 1C -B clean install -DskipTests
- - name: Test
+ - name: Test Spark
timeout-minutes: 60
run: |
# run tests with random timezone to find out timezone related bugs
. .github/workflows/utils.sh
jvm_timezone=$(random_timezone)
echo "JVM timezone is set to $jvm_timezone"
- mvn -T 1C -B clean install -pl '!paimon-e2e-tests'
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
+ test_modules=""
+ for suffix in common 3.5 3.4 3.3 3.2; do
+ test_modules+="org.apache.paimon:paimon-spark-${suffix},"
+ done
+ test_modules="${test_modules%,}"
+ mvn -T 1C -B test -pl "${test_modules}" -Duser.timezone=$jvm_timezone
env:
MAVEN_OPTS: -Xmx4096m
\ No newline at end of file
diff --git a/.github/workflows/utitcase.yml b/.github/workflows/utitcase.yml
index 431b44332..7963e7c21 100644
--- a/.github/workflows/utitcase.yml
+++ b/.github/workflows/utitcase.yml
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-name: UTCase and ITCase Non Flink
+name: UTCase and ITCase Others
on:
push:
@@ -53,6 +53,11 @@ jobs:
. .github/workflows/utils.sh
jvm_timezone=$(random_timezone)
echo "JVM timezone is set to $jvm_timezone"
- mvn -T 1C -B clean install -pl '!paimon-e2e-tests'
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
+ test_modules="!paimon-e2e-tests,"
+ for suffix in 3.5 3.4 3.3 3.2 common; do
+ test_modules+="!org.apache.paimon:paimon-spark-${suffix},"
+ done
+ test_modules="${test_modules%,}"
+ mvn -T 1C -B clean install -pl "${test_modules}"
-Pskip-paimon-flink-tests -Duser.timezone=$jvm_timezone
env:
MAVEN_OPTS: -Xmx4096m
\ No newline at end of file
diff --git a/paimon-codegen/pom.xml b/paimon-codegen/pom.xml
index 8a43f3909..4a40d487d 100644
--- a/paimon-codegen/pom.xml
+++ b/paimon-codegen/pom.xml
@@ -41,19 +41,19 @@ under the License.
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
+ <version>${codegen.scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
- <version>${scala.version}</version>
+ <version>${codegen.scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
- <version>${scala.version}</version>
+ <version>${codegen.scala.version}</version>
</dependency>
</dependencies>
@@ -86,6 +86,10 @@ under the License.
</goals>
</execution>
</executions>
+
+ <configuration>
+ <scalaVersion>${codegen.scala.version}</scalaVersion>
+ </configuration>
</plugin>
<plugin>
diff --git a/paimon-e2e-tests/pom.xml b/paimon-e2e-tests/pom.xml
index 7ca13535e..abe9ca896 100644
--- a/paimon-e2e-tests/pom.xml
+++ b/paimon-e2e-tests/pom.xml
@@ -34,7 +34,7 @@ under the License.
<properties>
<flink.shaded.hadoop.version>2.8.3-10.0</flink.shaded.hadoop.version>
<flink.cdc.version>3.1.1</flink.cdc.version>
-
<flink.sql.connector.hive>flink-sql-connector-hive-2.3.10_${scala.binary.version}</flink.sql.connector.hive>
+
<flink.sql.connector.hive>flink-sql-connector-hive-2.3.10_${flink.scala.binary.version}</flink.sql.connector.hive>
</properties>
<dependencies>
diff --git a/paimon-flink/paimon-flink-1.15/pom.xml
b/paimon-flink/paimon-flink-1.15/pom.xml
index bfc6ec53c..21c179226 100644
--- a/paimon-flink/paimon-flink-1.15/pom.xml
+++ b/paimon-flink/paimon-flink-1.15/pom.xml
@@ -87,14 +87,14 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<type>test-jar</type>
<scope>test</scope>
diff --git a/paimon-flink/paimon-flink-1.16/pom.xml
b/paimon-flink/paimon-flink-1.16/pom.xml
index 0edfc9e20..c5a334d6f 100644
--- a/paimon-flink/paimon-flink-1.16/pom.xml
+++ b/paimon-flink/paimon-flink-1.16/pom.xml
@@ -90,7 +90,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-flink/paimon-flink-1.17/pom.xml
b/paimon-flink/paimon-flink-1.17/pom.xml
index 112ecc4c7..06cc4f330 100644
--- a/paimon-flink/paimon-flink-1.17/pom.xml
+++ b/paimon-flink/paimon-flink-1.17/pom.xml
@@ -97,7 +97,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-flink/paimon-flink-1.18/pom.xml
b/paimon-flink/paimon-flink-1.18/pom.xml
index 5aaf42eb4..add1915a5 100644
--- a/paimon-flink/paimon-flink-1.18/pom.xml
+++ b/paimon-flink/paimon-flink-1.18/pom.xml
@@ -90,7 +90,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-flink/paimon-flink-1.19/pom.xml
b/paimon-flink/paimon-flink-1.19/pom.xml
index ab8c2540e..371ba54d1 100644
--- a/paimon-flink/paimon-flink-1.19/pom.xml
+++ b/paimon-flink/paimon-flink-1.19/pom.xml
@@ -97,7 +97,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-flink/paimon-flink-cdc/pom.xml
b/paimon-flink/paimon-flink-cdc/pom.xml
index 48f0d5a13..7f5bb3e78 100644
--- a/paimon-flink/paimon-flink-cdc/pom.xml
+++ b/paimon-flink/paimon-flink-cdc/pom.xml
@@ -185,14 +185,14 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
<type>test-jar</type>
diff --git a/paimon-flink/paimon-flink-common/pom.xml
b/paimon-flink/paimon-flink-common/pom.xml
index ef0f7fe17..3463aa9d9 100644
--- a/paimon-flink/paimon-flink-common/pom.xml
+++ b/paimon-flink/paimon-flink-common/pom.xml
@@ -122,7 +122,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
<type>test-jar</type>
@@ -130,7 +130,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-hive/paimon-hive-connector-2.3/pom.xml
b/paimon-hive/paimon-hive-connector-2.3/pom.xml
index e61e493d3..22f93ed0e 100644
--- a/paimon-hive/paimon-hive-connector-2.3/pom.xml
+++ b/paimon-hive/paimon-hive-connector-2.3/pom.xml
@@ -81,7 +81,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-hive/paimon-hive-connector-3.1/pom.xml
b/paimon-hive/paimon-hive-connector-3.1/pom.xml
index 3e08196a7..70afb0e4e 100644
--- a/paimon-hive/paimon-hive-connector-3.1/pom.xml
+++ b/paimon-hive/paimon-hive-connector-3.1/pom.xml
@@ -88,7 +88,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-hive/paimon-hive-connector-common/pom.xml
b/paimon-hive/paimon-hive-connector-common/pom.xml
index 0f7da151b..f22b73dea 100644
--- a/paimon-hive/paimon-hive-connector-common/pom.xml
+++ b/paimon-hive/paimon-hive-connector-common/pom.xml
@@ -111,14 +111,14 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+
<artifactId>flink-connector-hive_${flink.scala.binary.version}</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
diff --git a/paimon-spark/paimon-spark-3.2/pom.xml
b/paimon-spark/paimon-spark-3.2/pom.xml
index 24e5198e9..b12f4ba86 100644
--- a/paimon-spark/paimon-spark-3.2/pom.xml
+++ b/paimon-spark/paimon-spark-3.2/pom.xml
@@ -36,6 +36,18 @@ under the License.
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-bundle</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-spark-common</artifactId>
@@ -60,7 +72,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -97,7 +109,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -141,7 +153,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -154,23 +166,11 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-column</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -187,15 +187,11 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -208,10 +204,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -223,7 +215,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
<exclusions>
@@ -235,10 +227,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
diff --git a/paimon-spark/paimon-spark-3.3/pom.xml
b/paimon-spark/paimon-spark-3.3/pom.xml
index 3950ff9f8..b89ec3eb4 100644
--- a/paimon-spark/paimon-spark-3.3/pom.xml
+++ b/paimon-spark/paimon-spark-3.3/pom.xml
@@ -36,6 +36,18 @@ under the License.
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-bundle</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-spark-common</artifactId>
@@ -60,7 +72,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -93,7 +105,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -137,7 +149,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -150,23 +162,11 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-column</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -183,15 +183,11 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -204,10 +200,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -218,7 +210,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
<exclusions>
@@ -230,10 +222,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
diff --git a/paimon-spark/paimon-spark-3.4/pom.xml
b/paimon-spark/paimon-spark-3.4/pom.xml
index 000acdc26..c7bf21785 100644
--- a/paimon-spark/paimon-spark-3.4/pom.xml
+++ b/paimon-spark/paimon-spark-3.4/pom.xml
@@ -36,6 +36,18 @@ under the License.
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-bundle</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-spark-common</artifactId>
@@ -60,7 +72,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -93,7 +105,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -137,7 +149,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -150,23 +162,11 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-column</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -183,15 +183,11 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -204,10 +200,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -218,7 +210,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
<exclusions>
@@ -230,10 +222,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
diff --git a/paimon-spark/paimon-spark-3.5/pom.xml
b/paimon-spark/paimon-spark-3.5/pom.xml
index 0ba453068..08d218100 100644
--- a/paimon-spark/paimon-spark-3.5/pom.xml
+++ b/paimon-spark/paimon-spark-3.5/pom.xml
@@ -36,6 +36,18 @@ under the License.
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-bundle</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-spark-common</artifactId>
@@ -60,7 +72,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -89,7 +101,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -133,7 +145,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -158,7 +170,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -183,7 +195,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -210,7 +222,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
<exclusions>
diff --git a/paimon-spark/paimon-spark-common/pom.xml
b/paimon-spark/paimon-spark-common/pom.xml
index e9ca8b0e1..36139e283 100644
--- a/paimon-spark/paimon-spark-common/pom.xml
+++ b/paimon-spark/paimon-spark-common/pom.xml
@@ -38,6 +38,18 @@ under the License.
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-bundle</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
@@ -56,14 +68,14 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-avro_2.12</artifactId>
+ <artifactId>spark-avro_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -96,7 +108,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -133,7 +145,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
<exclusions>
@@ -145,10 +157,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -158,7 +166,7 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -171,19 +179,11 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-column</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -200,10 +200,6 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j2-impl</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -212,7 +208,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
@@ -225,10 +221,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
index 15a62f266..28af4ac0a 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonStatistics.scala
@@ -129,12 +129,12 @@ object PaimonColumnStats {
def apply(v1ColStats: ColumnStat): PaimonColumnStats = {
import PaimonImplicits._
PaimonColumnStats(
- if (v1ColStats.nullCount.isDefined)
OptionalLong.of(v1ColStats.nullCount.get.longValue())
+ if (v1ColStats.nullCount.isDefined)
OptionalLong.of(v1ColStats.nullCount.get.longValue)
else OptionalLong.empty(),
v1ColStats.min,
v1ColStats.max,
if (v1ColStats.distinctCount.isDefined)
- OptionalLong.of(v1ColStats.distinctCount.get.longValue())
+ OptionalLong.of(v1ColStats.distinctCount.get.longValue)
else OptionalLong.empty(),
if (v1ColStats.avgLen.isDefined)
OptionalLong.of(v1ColStats.avgLen.get.longValue())
else OptionalLong.empty(),
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
index a5d4f10ba..b5b56ba1d 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/ScanHelper.scala
@@ -76,7 +76,7 @@ trait ScanHelper extends Logging {
def closeDataSplit(): Unit = {
if (currentSplit.nonEmpty && currentDataFiles.nonEmpty) {
val newSplit =
- copyDataSplit(currentSplit.get, currentDataFiles,
currentDeletionFiles)
+ copyDataSplit(currentSplit.get, currentDataFiles.toSeq,
currentDeletionFiles.toSeq)
currentSplits += newSplit
}
currentDataFiles.clear()
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala
index ba6108395..f2530b50c 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonMergeIntoBase.scala
@@ -53,7 +53,7 @@ trait PaimonMergeIntoBase
merge.notMatchedActions.flatMap(_.condition).foreach(checkCondition)
val updateActions = merge.matchedActions.collect { case a:
UpdateAction => a }
- val primaryKeys = v2Table.getTable.primaryKeys().asScala
+ val primaryKeys = v2Table.getTable.primaryKeys().asScala.toSeq
if (primaryKeys.nonEmpty) {
checkUpdateActionValidity(
AttributeSet(targetOutput),
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala
index 123c67a2f..ad3912ddb 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonUpdateTable.scala
@@ -41,7 +41,7 @@ object PaimonUpdateTable
table.getTable match {
case paimonTable: FileStoreTable =>
- val primaryKeys = paimonTable.primaryKeys().asScala
+ val primaryKeys = paimonTable.primaryKeys().asScala.toSeq
if (!validUpdateAssignment(u.table.outputSet, primaryKeys,
assignments)) {
throw new RuntimeException("Can't update the primary key
column.")
}
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala
index 5d264370a..4cf9284f9 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/EvalSubqueriesForDeleteTable.scala
@@ -49,7 +49,10 @@ object EvalSubqueriesForDeleteTable extends
Rule[LogicalPlan] with ExpressionHel
plan.transformDown {
case d @ DeleteFromPaimonTableCommand(_, table, condition)
if SubqueryExpression.hasSubquery(condition) &&
- isPredicatePartitionColumnsOnly(condition,
table.partitionKeys().asScala, resolver) =>
+ isPredicatePartitionColumnsOnly(
+ condition,
+ table.partitionKeys().asScala.toSeq,
+ resolver) =>
try {
d.copy(condition = evalSubquery(condition))
} catch {
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala
index eca8c9cdf..a59c2a3fe 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/optimizer/MergePaimonScalarSubqueriesBase.scala
@@ -95,7 +95,7 @@ trait MergePaimonScalarSubqueriesBase extends
Rule[LogicalPlan] with PredicateHe
val newPlan = removeReferences(planWithReferences, cache)
val subqueryCTEs =
cache.filter(_.merged).map(_.plan.asInstanceOf[CTERelationDef])
if (subqueryCTEs.nonEmpty) {
- WithCTE(newPlan, subqueryCTEs)
+ WithCTE(newPlan, subqueryCTEs.toSeq)
} else {
newPlan
}
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
index 2b3888911..0087ffb74 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/DeleteFromPaimonTableCommand.scala
@@ -60,7 +60,7 @@ case class DeleteFromPaimonTableCommand(
} else {
val (partitionCondition, otherCondition) =
splitPruePartitionAndOtherPredicates(
condition,
- table.partitionKeys().asScala,
+ table.partitionKeys().asScala.toSeq,
sparkSession.sessionState.conf.resolver)
val partitionPredicate = if (partitionCondition.isEmpty) {
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala
index 51ae6e086..c64f20c68 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/MergeIntoPaimonTable.scala
@@ -243,7 +243,7 @@ case class MergeIntoPaimonTable(
val outputFields = mutable.ArrayBuffer(tableSchema.fields: _*)
outputFields += StructField(ROW_KIND_COL, ByteType)
outputFields ++= metadataCols.map(_.toStructField)
- val outputSchema = StructType(outputFields)
+ val outputSchema = StructType(outputFields.toSeq)
val joinedRowEncoder = EncoderUtils.encode(joinedPlan.schema)
val outputEncoder = EncoderUtils.encode(outputSchema).resolveAndBind()
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala
index 6b3d96813..19f73cb6c 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonAnalyzeTableColumnCommand.scala
@@ -25,8 +25,8 @@ import org.apache.paimon.stats.{ColStats, Statistics}
import org.apache.paimon.table.FileStoreTable
import org.apache.paimon.table.sink.BatchWriteBuilder
import org.apache.paimon.table.source.DataSplit
+import org.apache.paimon.utils.Preconditions.checkState
-import org.apache.parquet.Preconditions
import org.apache.spark.sql.{PaimonStatsUtils, Row, SparkSession}
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.ColumnStat
@@ -74,9 +74,7 @@ case class PaimonAnalyzeTableColumnCommand(
PaimonStatsUtils.computeColumnStats(sparkSession, relation, attributes)
val totalRecordCount = currentSnapshot.totalRecordCount()
- Preconditions.checkState(
- totalRecordCount >= mergedRecordCount,
- s"totalRecordCount: $totalRecordCount should be greater or equal than
mergedRecordCount: $mergedRecordCount.")
+ checkState(totalRecordCount >= mergedRecordCount)
val mergedRecordSize = totalSize * (mergedRecordCount.toDouble /
totalRecordCount).toLong
// convert to paimon stats
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala
index 62b8ee849..191d7a766 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonCommand.scala
@@ -101,7 +101,7 @@ trait PaimonCommand extends WithFileStoreTable with
ExpressionHelper with SQLCon
filter.foreach(snapshotReader.withFilter)
}
- snapshotReader.read().splits().asScala.collect { case s: DataSplit => s }
+ snapshotReader.read().splits().asScala.collect { case s: DataSplit => s
}.toSeq
}
protected def findTouchedFiles(
@@ -232,7 +232,7 @@ trait PaimonCommand extends WithFileStoreTable with
ExpressionHelper with SQLCon
.as[(String, Long)]
.groupByKey(_._1)
.mapGroups {
- case (filePath, iter) =>
+ (filePath, iter) =>
val dv = new BitmapDeletionVector()
while (iter.hasNext) {
dv.delete(iter.next()._2)
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
index 391ba2b87..7d56fe867 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala
@@ -250,7 +250,7 @@ case class PaimonSparkWriter(table: FileStoreTable) {
val serializedCommits = deletionVectors
.groupByKey(_.partitionAndBucket)
.mapGroups {
- case (_, iter: Iterator[SparkDeletionVectors]) =>
+ (_, iter: Iterator[SparkDeletionVectors]) =>
val indexHandler = table.store().newIndexFileHandler()
var dvIndexFileMaintainer: AppendDeletionFileMaintainer = null
while (iter.hasNext) {
@@ -397,7 +397,7 @@ case class PaimonSparkWriter(table: FileStoreTable) {
}
private def repartitionByPartitionsAndBucket(df: DataFrame): DataFrame = {
- val partitionCols = tableSchema.partitionKeys().asScala.map(col)
+ val partitionCols = tableSchema.partitionKeys().asScala.map(col).toSeq
df.repartition(partitionCols ++ Seq(col(BUCKET_COL)): _*)
}
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala
index b380d36c3..9c377b47c 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/SparkDataFileMeta.scala
@@ -58,7 +58,7 @@ object SparkDataFileMeta {
file,
dvFactory.create(file.fileName()))
}
- }
+ }.toSeq
def convertToDataSplits(
sparkDataFiles: Array[SparkDataFileMeta],
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala
index d79105e24..488d70e34 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/orphan/SparkOrphanFilesClean.scala
@@ -62,7 +62,7 @@ case class SparkOrphanFilesClean(
val maxBranchParallelism = Math.min(branches.size(), parallelism)
// find snapshots using branch and find manifests(manifest, index,
statistics) using snapshot
val usedManifestFiles = spark.sparkContext
- .parallelize(branches.asScala, maxBranchParallelism)
+ .parallelize(branches.asScala.toSeq, maxBranchParallelism)
.mapPartitions(_.flatMap {
branch => safelyGetAllSnapshots(branch).asScala.map(snapshot =>
(branch, snapshot.toJson))
})
@@ -114,7 +114,7 @@ case class SparkOrphanFilesClean(
.toDF("used_name")
// find candidate files which can be removed
- val fileDirs = listPaimonFileDirs.asScala.map(_.toUri.toString)
+ val fileDirs = listPaimonFileDirs.asScala.map(_.toUri.toString).toSeq
val maxFileDirsParallelism = Math.min(fileDirs.size, parallelism)
val candidates = spark.sparkContext
.parallelize(fileDirs, maxFileDirsParallelism)
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala
index 32fa48210..627c6a168 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/statistics/StatisticsHelperBase.scala
@@ -77,11 +77,11 @@ trait StatisticsHelperBase extends SQLConfHelper {
private def toV2Stats(v1Stats: logical.Statistics): Statistics = {
new Statistics() {
override def sizeInBytes(): OptionalLong = if (v1Stats.sizeInBytes !=
null)
- OptionalLong.of(v1Stats.sizeInBytes.longValue())
+ OptionalLong.of(v1Stats.sizeInBytes.longValue)
else OptionalLong.empty()
override def numRows(): OptionalLong = if (v1Stats.rowCount.isDefined)
- OptionalLong.of(v1Stats.rowCount.get.longValue())
+ OptionalLong.of(v1Stats.rowCount.get.longValue)
else OptionalLong.empty()
override def columnStats(): java.util.Map[NamedReference,
ColumnStatistics] = {
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
index af7ff7204..5b762ffb4 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
@@ -38,6 +38,7 @@ object OptionUtils extends SQLConfHelper {
case (key, value) =>
key.stripPrefix(PAIMON_OPTION_PREFIX) -> value
}
+ .toMap
.asJava)
mergedOptions.putAll(extraOptions)
mergedOptions
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
index 397424c63..3424129e9 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
@@ -57,8 +57,8 @@ class PaimonSqlExtensionsAstBuilder(delegate: ParserInterface)
/** Creates a [[PaimonCallStatement]] for a stored procedure call. */
override def visitCall(ctx: CallContext): PaimonCallStatement =
withOrigin(ctx) {
- val name = toSeq(ctx.multipartIdentifier.parts).map(_.getText)
- val args = toSeq(ctx.callArgument).map(typedVisit[PaimonCallArgument])
+ val name = ctx.multipartIdentifier.parts.asScala.map(_.getText).toSeq
+ val args =
ctx.callArgument.asScala.map(typedVisit[PaimonCallArgument]).toSeq
logical.PaimonCallStatement(name, args)
}
@@ -89,7 +89,7 @@ class PaimonSqlExtensionsAstBuilder(delegate: ParserInterface)
/** Returns a multi-part identifier as Seq[String]. */
override def visitMultipartIdentifier(ctx: MultipartIdentifierContext):
Seq[String] =
withOrigin(ctx) {
- ctx.parts.asScala.map(_.getText)
+ ctx.parts.asScala.map(_.getText).toSeq
}
private def toBuffer[T](list: java.util.List[T]) = list.asScala
diff --git
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java
index 9af886d83..b98213c0e 100644
---
a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java
+++
b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkInternalRowTest.java
@@ -104,8 +104,8 @@ public class SparkInternalRowTest {
+ "paimon,"
+ "22.2,"
+ "Map(key2 -> [2.4,3.5], key1 -> [1.2,2.3]),"
- + "WrappedArray(v1, v5),"
- + "WrappedArray(10, 30),"
+ + "[v1, v5],"
+ + "[10, 30],"
+ "true,"
+ "22,"
+ "356,"
@@ -129,6 +129,12 @@ public class SparkInternalRowTest {
private String sparkRowToString(org.apache.spark.sql.Row row) {
return JavaConverters.seqAsJavaList(row.toSeq()).stream()
+ .map(
+ x ->
+ (x instanceof scala.collection.Seq)
+ ? JavaConverters.seqAsJavaList(
+ (scala.collection.Seq<Object>)
x)
+ : x)
.map(Object::toString)
// Since the toString result of Spark's binary col is
unstable, replace it
.map(x -> x.startsWith("[B@") ? "[B@" : x)
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
index 1ccf4e38e..4aa73d026 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/AnalyzeTableTestBase.scala
@@ -403,7 +403,7 @@ abstract class AnalyzeTableTestBase extends
PaimonSparkTestBase {
spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS")
val stats = getScanStatistic("SELECT * FROM T")
- Assertions.assertEquals(2L, stats.rowCount.get.longValue())
+ Assertions.assertEquals(2L, stats.rowCount.get.longValue)
}
test("Paimon analyze: spark use col stats") {
@@ -418,7 +418,7 @@ abstract class AnalyzeTableTestBase extends
PaimonSparkTestBase {
spark.sql(s"ANALYZE TABLE T COMPUTE STATISTICS FOR ALL COLUMNS")
val stats = getScanStatistic("SELECT * FROM T")
- Assertions.assertEquals(2L, stats.rowCount.get.longValue())
+ Assertions.assertEquals(2L, stats.rowCount.get.longValue)
Assertions.assertEquals(if (gteqSpark3_4) 4 else 0,
stats.attributeStats.size)
}
@@ -437,19 +437,19 @@ abstract class AnalyzeTableTestBase extends
PaimonSparkTestBase {
var sql = "SELECT * FROM T WHERE pt < 1"
Assertions.assertEquals(
if (gteqSpark3_4) 0L else 4L,
- getScanStatistic(sql).rowCount.get.longValue())
+ getScanStatistic(sql).rowCount.get.longValue)
checkAnswer(spark.sql(sql), Nil)
// partition push down hit and select without it
sql = "SELECT id FROM T WHERE pt < 1"
Assertions.assertEquals(
if (gteqSpark3_4) 0L else 4L,
- getScanStatistic(sql).rowCount.get.longValue())
+ getScanStatistic(sql).rowCount.get.longValue)
checkAnswer(spark.sql(sql), Nil)
// partition push down not hit
sql = "SELECT * FROM T WHERE id < 1"
- Assertions.assertEquals(4L, getScanStatistic(sql).rowCount.get.longValue())
+ Assertions.assertEquals(4L, getScanStatistic(sql).rowCount.get.longValue)
checkAnswer(spark.sql(sql), Nil)
}
@@ -468,10 +468,10 @@ abstract class AnalyzeTableTestBase extends
PaimonSparkTestBase {
// For col type such as char, varchar that don't have min and max,
filter estimation on stats has no effect.
var sqlText = "SELECT * FROM T WHERE pt < '1'"
- Assertions.assertEquals(4L,
getScanStatistic(sqlText).rowCount.get.longValue())
+ Assertions.assertEquals(4L,
getScanStatistic(sqlText).rowCount.get.longValue)
sqlText = "SELECT id FROM T WHERE pt < '1'"
- Assertions.assertEquals(4L,
getScanStatistic(sqlText).rowCount.get.longValue())
+ Assertions.assertEquals(4L,
getScanStatistic(sqlText).rowCount.get.longValue)
}
})
}
diff --git a/pom.xml b/pom.xml
index 1f0d47860..28952f46d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -80,8 +80,12 @@ under the License.
<paimon.shade.netty.version>4.1.100.Final</paimon.shade.netty.version>
<antlr4.version>4.9.3</antlr4.version>
<hadoop.version>2.8.5</hadoop.version>
- <scala.version>2.12.15</scala.version>
<scala.binary.version>2.12</scala.binary.version>
+ <flink.scala.binary.version>2.12</flink.scala.binary.version>
+ <scala212.version>2.12.15</scala212.version>
+ <scala213.version>2.13.14</scala213.version>
+ <scala.version>${scala212.version}</scala.version>
+ <codegen.scala.version>${scala212.version}</codegen.scala.version>
<snappy.version>1.1.8.4</snappy.version>
<airlift.version>0.27</airlift.version>
<lz4.version>1.8.0</lz4.version>
@@ -338,6 +342,19 @@ under the License.
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>scala-2.13</id>
+ <properties>
+ <scala.binary.version>2.13</scala.binary.version>
+ <scala.version>${scala213.version}</scala.version>
+ </properties>
+ <activation>
+ <property>
+ <name>scala-2.13</name>
+ </property>
+ </activation>
+ </profile>
</profiles>
<build>
@@ -615,10 +632,10 @@ under the License.
<rules>
<bannedDependencies>
<excludes>
-
<exclude>org.apache.flink:flink-table-planner_${scala.binary.version}</exclude>
+
<exclude>org.apache.flink:flink-table-planner_${flink.scala.binary.version}</exclude>
</excludes>
<includes>
-
<include>org.apache.flink:flink-table-planner_${scala.binary.version}:*:*:test</include>
+
<include>org.apache.flink:flink-table-planner_${flink.scala.binary.version}:*:*:test</include>
</includes>
<message>
Direct dependencies on
flink-table-planner are not allowed.
