This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 8f59bb6 [refractor](connector) Refactor module structure for release
(#247)
8f59bb6 is described below
commit 8f59bb67954c93d27e2e87a99aaa54fad2625539
Author: gnehil <[email protected]>
AuthorDate: Tue Dec 31 14:20:09 2024 +0800
[refractor](connector) Refactor module structure for release (#247)
---
.github/workflows/build-extension.yml | 28 ++-----
.github/workflows/run-itcase-12.yml | 4 +-
.github/workflows/run-itcase-20.yml | 4 +-
spark-doris-connector/build.sh | 51 ++++++------
spark-doris-connector/pom.xml | 50 ++++++++----
.../spark-doris-connector-base/pom.xml | 3 +-
.../apache/doris/spark/load/CopyIntoLoader.scala | 4 +-
.../apache/doris/spark/util/SchemaConvertors.scala | 2 +-
.../spark-doris-connector-dist/pom.xml | 93 ----------------------
.../spark-doris-connector-it/pom.xml | 8 +-
.../apache/doris/spark/sql/DorisReaderITCase.scala | 5 +-
.../spark-doris-connector-shims/pom.xml | 52 ------------
.../spark/catalog/DorisTableCatalogBase.scala | 28 -------
.../doris/spark/sql/sources/DorisDataSource.scala | 24 ------
.../spark-doris-connector-shims_spark-3.1/pom.xml | 49 ------------
.../spark/catalog/DorisTableCatalogBase.scala | 28 -------
.../doris/spark/sql/sources/DorisDataSource.scala | 20 -----
.../spark/catalog/DorisTableCatalogBase.scala | 28 -------
.../doris/spark/sql/sources/DorisDataSource.scala | 20 -----
.../spark/catalog/DorisTableCatalogBase.scala | 26 ------
.../doris/spark/sql/sources/DorisDataSource.scala | 20 -----
.../spark/catalog/DorisTableCatalogBase.scala | 26 ------
.../doris/spark/sql/sources/DorisDataSource.scala | 20 -----
.../spark-doris-connector-shims_spark-3.5/pom.xml | 48 -----------
.../spark/catalog/DorisTableCatalogBase.scala | 26 ------
.../apache/doris/spark/read/DorisScanBuilder.scala | 24 ------
.../doris/spark/sql/sources/DorisDataSource.scala | 20 -----
.../pom.xml | 16 ++--
.../doris/spark/catalog/DorisTableBase.scala} | 12 +--
.../spark/catalog/DorisTableCatalogBase.scala} | 14 ++--
.../spark/catalog/DorisTableProviderBase.scala} | 13 +--
.../doris/spark/read/DorisPartitionReader.scala | 0
.../spark/read/DorisPartitionReaderFactory.scala | 0
.../org/apache/doris/spark/read/DorisScan.scala | 0
.../doris/spark/read/DorisScanBuilderBase.scala} | 2 +-
.../org/apache/doris/spark/read/ScanMode.java | 0
.../spark/sql/sources/DorisSourceProvider.scala | 0
.../apache/doris/spark/write/DorisDataWriter.scala | 0
.../doris/spark/write/DorisDataWriterFactory.scala | 0
.../org/apache/doris/spark/write/DorisWrite.scala | 0
.../doris/spark/write/DorisWriteBuilder.scala | 0
.../pom.xml | 17 ++--
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../apache/doris/spark/catalog/DorisTable.scala} | 22 +++--
.../doris/spark/catalog/DorisTableCatalog.scala} | 10 ++-
.../apache/doris/spark/read/DorisScanBuilder.scala | 2 +-
.../doris/spark/sql/sources/DorisDataSource.scala} | 18 ++---
.../pom.xml | 17 ++--
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../apache/doris/spark/catalog/DorisTable.scala} | 22 +++--
.../doris/spark/catalog/DorisTableCatalog.scala} | 10 ++-
.../apache/doris/spark/read/DorisScanBuilder.scala | 2 +-
.../doris/spark/sql/sources/DorisDataSource.scala} | 18 ++---
.../pom.xml | 17 ++--
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../apache/doris/spark/catalog/DorisTable.scala} | 21 +++--
.../doris/spark/catalog/DorisTableCatalog.scala} | 10 ++-
.../apache/doris/spark/read/DorisScanBuilder.scala | 2 +-
.../doris/spark/sql/sources/DorisDataSource.scala} | 17 ++--
.../pom.xml | 17 ++--
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../apache/doris/spark/catalog/DorisTable.scala} | 21 +++--
.../doris/spark/catalog/DorisTableCatalog.scala} | 10 ++-
.../apache/doris/spark/read/DorisScanBuilder.scala | 3 +-
.../doris/spark/sql/sources/DorisDataSource.scala} | 17 ++--
.../pom.xml | 19 +++--
...org.apache.spark.sql.sources.DataSourceRegister | 2 +-
.../apache/doris/spark/catalog/DorisTable.scala} | 21 +++--
.../doris/spark/catalog/DorisTableCatalog.scala} | 10 ++-
.../apache/doris/spark/read/DorisScanBuilder.scala | 2 +-
.../doris/spark/sql/sources/DorisDataSource.scala} | 17 ++--
71 files changed, 292 insertions(+), 828 deletions(-)
diff --git a/.github/workflows/build-extension.yml
b/.github/workflows/build-extension.yml
index 7d4acbd..707e311 100644
--- a/.github/workflows/build-extension.yml
+++ b/.github/workflows/build-extension.yml
@@ -44,42 +44,28 @@ jobs:
- name: Build spark connector 2.4 2.11
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-2 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT}
-Pspark-2.4_2.11 -pl spark-doris-connector-spark-2 -am
- name: Build spark connector 2.4 2.12
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Dscala.version=2.12.18 \
- -Dscala.major.version=2.12 \
- -Pspark-2 -pl spark-doris-connector-dist -am
-
- - name: Build spark connector 3.0
- run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.0 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT}
-Pspark-2.4_2.12 -pl spark-doris-connector-spark-2 -am
- name: Build spark connector 3.1
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.1 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.1
-pl spark-doris-connector-spark-3.1 -am
- name: Build spark connector 3.2
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.2 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.2
-pl spark-doris-connector-spark-3.2 -am
- name: Build spark connector 3.3
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.3 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.3
-pl spark-doris-connector-spark-3.3 -am
- name: Build spark connector 3.4
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.4 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.4
-pl spark-doris-connector-spark-3.4 -am
- name: Build spark connector 3.5
run: |
- cd spark-doris-connector && mvn clean install ${MVN_OPT} \
- -Pspark-3,spark-3.5 -pl spark-doris-connector-dist -am
+ cd spark-doris-connector && mvn clean install ${MVN_OPT} -Pspark-3.5
-pl spark-doris-connector-spark-3.5 -am
diff --git a/.github/workflows/run-itcase-12.yml
b/.github/workflows/run-itcase-12.yml
index 559f17d..fddcda7 100644
--- a/.github/workflows/run-itcase-12.yml
+++ b/.github/workflows/run-itcase-12.yml
@@ -40,8 +40,8 @@ jobs:
- name: Run ITCases for spark 2
run: |
- cd spark-doris-connector && mvn clean test -pl
spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase"
-Dimage="adamlee489/doris:1.2.7.1_x86"
+ cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11
-pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase"
-Dimage="adamlee489/doris:1.2.7.1_x86"
- name: Run ITCases for spark 3
run: |
- cd spark-doris-connector && mvn clean test -pl
spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false
-Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
+ cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl
spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase"
-Dimage="adamlee489/doris:1.2.7.1_x86"
diff --git a/.github/workflows/run-itcase-20.yml
b/.github/workflows/run-itcase-20.yml
index 7af80ee..d16d810 100644
--- a/.github/workflows/run-itcase-20.yml
+++ b/.github/workflows/run-itcase-20.yml
@@ -40,9 +40,9 @@ jobs:
- name: Run ITCases for spark 2
run: |
- cd spark-doris-connector && mvn clean test -pl
spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase"
-Dimage="adamlee489/doris:2.0.3"
+ cd spark-doris-connector && mvn clean test -Pspark-2-it,spark-2.4_2.11
-pl spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase"
-Dimage="adamlee489/doris:2.0.3"
- name: Run ITCases for spark 3
run: |
- cd spark-doris-connector && mvn clean test -pl
spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false
-Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+ cd spark-doris-connector && mvn clean test -Pspark-3-it,spark-3.1 -pl
spark-doris-connector-it -am -DfailIfNoTests=false -Dtest="*ITCase"
-Dimage="adamlee489/doris:2.0.3"
\ No newline at end of file
diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
index 88ea305..2e953ce 100755
--- a/spark-doris-connector/build.sh
+++ b/spark-doris-connector/build.sh
@@ -142,32 +142,29 @@ selectScala() {
selectSpark() {
echo 'Spark-Doris-Connector supports multiple versions of spark. Which
version do you need ?'
- select spark in "2.4" "3.0" "3.1" "3.2" "3.3" "3.4" "3.5" "other"
+ select spark in "2.4" "3.1" "3.2" "3.3" "3.4" "3.5" "other"
do
case $spark in
"2.4")
return 1
;;
- "3.0")
- return 2
- ;;
"3.1")
- return 3
+ return 2
;;
"3.2")
- return 4
+ return 3
;;
"3.3")
- return 5
+ return 4
;;
"3.4")
- return 6
+ return 5
;;
"3.5")
- return 7
+ return 6
;;
"other")
- return 8
+ return 7
;;
esac
done
@@ -189,18 +186,16 @@ SparkVer=$?
if [ ${SparkVer} -eq 1 ]; then
SPARK_VERSION="2.4.8"
elif [ ${SparkVer} -eq 2 ]; then
- SPARK_VERSION="3.0.3"
-elif [ ${SparkVer} -eq 3 ]; then
SPARK_VERSION="3.1.3"
-elif [ ${SparkVer} -eq 4 ]; then
+elif [ ${SparkVer} -eq 3 ]; then
SPARK_VERSION="3.2.4"
-elif [ ${SparkVer} -eq 5 ]; then
+elif [ ${SparkVer} -eq 4 ]; then
SPARK_VERSION="3.3.4"
-elif [ ${SparkVer} -eq 6 ]; then
+elif [ ${SparkVer} -eq 5 ]; then
SPARK_VERSION="3.4.3"
-elif [ ${SparkVer} -eq 7 ]; then
+elif [ ${SparkVer} -eq 6 ]; then
SPARK_VERSION="3.5.3"
-elif [ ${SparkVer} -eq 8 ]; then
+elif [ ${SparkVer} -eq 7 ]; then
# shellcheck disable=SC2162
read -p 'Which spark version do you need? please input
:' ver
@@ -223,23 +218,23 @@ echo_g " scala version: ${SCALA_VERSION}, major version:
${SCALA_MAJOR_VERSION}"
echo_g " spark version: ${SPARK_VERSION}, major version:
${SPARK_MAJOR_VERSION}"
echo_g " build starting..."
-SPARK_PRIMARY_VERSION=0
-[ ${SPARK_MAJOR_VERSION} != 0 ] &&
SPARK_PRIMARY_VERSION=${SPARK_MAJOR_VERSION%.*}
+if [[ $SPARK_VERSION =~ ^3.* ]]; then
+ profile_name="spark-${SPARK_MAJOR_VERSION}"
+ module_suffix=${SPARK_MAJOR_VERSION}
+else
+ profile_name="spark-${SPARK_MAJOR_VERSION}_${SCALA_MAJOR_VERSION}"
+ module_suffix="2"
+fi
-${MVN_BIN} clean install \
- -Dspark.version=${SPARK_VERSION} \
- -Dscala.version=${SCALA_VERSION} \
- -Dspark.major.version=${SPARK_MAJOR_VERSION} \
- -Dscala.major.version=${SCALA_MAJOR_VERSION} \
- -Pspark-${SPARK_PRIMARY_VERSION} -pl spark-doris-connector-dist -am "$@"
+${MVN_BIN} clean install -P"${profile_name}" -am "$@"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
DIST_DIR=${DORIS_HOME}/dist
[ ! -d "$DIST_DIR" ] && mkdir "$DIST_DIR"
- dist_jar=$(ls "${ROOT}"/spark-doris-connector-dist/target | grep
"spark-doris-" | grep -v "sources.jar" | grep -v "original-")
- rm -rf "${DIST_DIR}"/spark-doris-connector-dist/"${dist_jar}"
- cp "${ROOT}"/spark-doris-connector-dist/target/"${dist_jar}" "$DIST_DIR"
+ dist_jar=$(ls
"${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target | grep
"spark-doris-" | grep -v "sources.jar" | grep -v "original-")
+ rm -rf
"${DIST_DIR}"/spark-doris-connector-spark-"${module_suffix}"/"${dist_jar}"
+ cp
"${ROOT}"/spark-doris-connector-spark-"${module_suffix}"/target/"${dist_jar}"
"$DIST_DIR"
echo_g "*****************************************************************"
echo_g "Successfully build Spark-Doris-Connector"
diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml
index 8191ed6..db96187 100644
--- a/spark-doris-connector/pom.xml
+++ b/spark-doris-connector/pom.xml
@@ -41,11 +41,14 @@
</licenses>
<modules>
<module>spark-doris-connector-base</module>
- <module>spark-doris-connector-shims</module>
- <module>spark-doris-connector-dist</module>
<module>spark-doris-connector-spark-2</module>
- <module>spark-doris-connector-spark-3</module>
<module>spark-doris-connector-it</module>
+ <module>spark-doris-connector-spark-3-base</module>
+ <module>spark-doris-connector-spark-3.1</module>
+ <module>spark-doris-connector-spark-3.2</module>
+ <module>spark-doris-connector-spark-3.3</module>
+ <module>spark-doris-connector-spark-3.4</module>
+ <module>spark-doris-connector-spark-3.5</module>
</modules>
<scm>
@@ -92,6 +95,7 @@
<jmockit.version>1.49</jmockit.version>
<fe_ut_parallel>1</fe_ut_parallel>
<argLine>-Xmx512m</argLine>
+ <maven.deploy.skip>true</maven.deploy.skip>
</properties>
<dependencyManagement>
@@ -108,20 +112,34 @@
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-spark-3</artifactId>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
-
<artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
+ <artifactId>spark-doris-connector-spark-3.1</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.doris</groupId>
-
<artifactId>spark-doris-connector-shims_spark-${spark.major.version}</artifactId>
+ <artifactId>spark-doris-connector-spark-3.2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.doris</groupId>
+ <artifactId>spark-doris-connector-spark-3.3</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.doris</groupId>
+ <artifactId>spark-doris-connector-spark-3.4</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.doris</groupId>
+ <artifactId>spark-doris-connector-spark-3.5</artifactId>
<version>${project.version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>thrift-service</artifactId>
@@ -764,7 +782,7 @@
<!-->not reuse forked jvm, so that each unit test
will run in separate jvm. to avoid singleton confict<-->
<reuseForks>false</reuseForks>
<argLine>
-
-javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar
@{argLine}
+
-javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar
--add-opens=java.base/java.nio=ALL-UNNAMED @{argLine}
</argLine>
</configuration>
</plugin>
@@ -773,22 +791,19 @@
</profile>
<profile>
- <id>spark-2</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
+ <id>spark-2.4_2.11</id>
<properties>
<spark.version>2.4.8</spark.version>
<spark.major.version>2.4</spark.major.version>
- <scala.version>2.11.10</scala.version>
+ <scala.version>2.11.12</scala.version>
<scala.major.version>2.11</scala.major.version>
</properties>
</profile>
<profile>
- <id>spark-3.0</id>
+ <id>spark-2.4_2.12</id>
<properties>
- <spark.version>3.0.0</spark.version>
- <spark.major.version>3.0</spark.major.version>
+ <spark.version>2.4.8</spark.version>
+ <spark.major.version>2.4</spark.major.version>
<scala.version>2.12.18</scala.version>
<scala.major.version>2.12</scala.major.version>
</properties>
@@ -813,6 +828,9 @@
</profile>
<profile>
<id>spark-3.3</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<properties>
<spark.version>3.3.0</spark.version>
<spark.major.version>3.3</spark.major.version>
diff --git a/spark-doris-connector/spark-doris-connector-base/pom.xml
b/spark-doris-connector/spark-doris-connector-base/pom.xml
index 4f2f0ef..0e26fc2 100644
--- a/spark-doris-connector/spark-doris-connector-base/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-base/pom.xml
@@ -35,6 +35,7 @@
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.deploy.skip>true</maven.deploy.skip>
</properties>
<dependencies>
@@ -296,4 +297,4 @@
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git
a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
index 3ae5ec6..03a914a 100644
---
a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
+++
b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/load/CopyIntoLoader.scala
@@ -38,7 +38,7 @@ import java.io.{ByteArrayOutputStream, IOException}
import java.nio.charset.StandardCharsets
import java.util.zip.GZIPOutputStream
import java.util.{Base64, Properties, UUID}
-import scala.collection.JavaConverters._
+import scala.collection.JavaConverters.{asJavaIteratorConverter,
mapAsJavaMapConverter, propertiesAsScalaMapConverter}
import scala.util.{Failure, Success, Try}
case class CopyIntoResponse(code: Int, msg: String, content: String)
@@ -234,7 +234,7 @@ class CopyIntoLoader(settings: SparkSettings, isStreaming:
Boolean) extends Load
}
props.remove("columns")
val properties = new Properties()
- properties.putAll(props.mapValues(_.toString).asJava)
+ props.foreach(p => properties.setProperty(p._1, p._2))
properties
}
diff --git
a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
index 275ad87..f85eb28 100644
---
a/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
+++
b/spark-doris-connector/spark-doris-connector-base/src/main/scala/org/apache/doris/spark/util/SchemaConvertors.scala
@@ -67,7 +67,7 @@ object SchemaConvertors {
def convertToSchema(tscanColumnDescs: Seq[TScanColumnDesc]): Schema = {
val schema = new Schema(tscanColumnDescs.length)
tscanColumnDescs.foreach(desc => {
- println(desc.getName + " " + desc.getType.name())
+ // println(desc.getName + " " + desc.getType.name())
schema.put(new Field(desc.getName, desc.getType.name, "", 0, 0, ""))
})
schema
diff --git a/spark-doris-connector/spark-doris-connector-dist/pom.xml
b/spark-doris-connector/spark-doris-connector-dist/pom.xml
deleted file mode 100644
index 5bda3b5..0000000
--- a/spark-doris-connector/spark-doris-connector-dist/pom.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector</artifactId>
- <version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
-
- <artifactId>spark-doris-connector-dist</artifactId>
- <packaging>jar</packaging>
-
- <properties>
- <maven.compiler.source>8</maven.compiler.source>
- <maven.compiler.target>8</maven.compiler.target>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <profiles>
- <profile>
- <id>spark-2</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-spark-2</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
- <profile>
- <id>spark-3</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-spark-3</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
- <build>
-
<finalName>spark-doris-connector-${spark.major.version}_${scala.major.version}-${revision}</finalName>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>flatten-maven-plugin</artifactId>
- <version>1.2.5</version>
- <configuration>
- <updatePomFile>true</updatePomFile>
- <flattenMode>resolveCiFriendliesOnly</flattenMode>
- </configuration>
- <executions>
- <execution>
- <id>flatten</id>
- <phase>process-resources</phase>
- <goals>
- <goal>flatten</goal>
- </goals>
- </execution>
- <execution>
- <id>flatten.clean</id>
- <phase>clean</phase>
- <goals>
- <goal>clean</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-it/pom.xml
b/spark-doris-connector/spark-doris-connector-it/pom.xml
index 761d5a1..9493c03 100644
--- a/spark-doris-connector/spark-doris-connector-it/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-it/pom.xml
@@ -56,10 +56,12 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.major.version}</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.major.version}</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
@@ -77,7 +79,7 @@
<profiles>
<profile>
- <id>spark-2</id>
+ <id>spark-2-it</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
@@ -91,11 +93,11 @@
</dependencies>
</profile>
<profile>
- <id>spark-3</id>
+ <id>spark-3-it</id>
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-spark-3</artifactId>
+ <artifactId>spark-doris-connector-spark-3.1</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
diff --git
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
index e2d0e08..a147a7d 100644
---
a/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
+++
b/spark-doris-connector/spark-doris-connector-it/src/test/java/org/apache/doris/spark/sql/DorisReaderITCase.scala
@@ -37,6 +37,7 @@ class DorisReaderITCase extends DorisTestBase {
val sparkConf: SparkConf = new
SparkConf().setMaster("local[*]").setAppName("rddSource")
val sc = new SparkContext(sparkConf)
+ // sc.setLogLevel("DEBUG")
val dorisSparkRDD = sc.dorisRDD(
tableIdentifier = Some(DATABASE + "." + TABLE_READ),
cfg = Some(Map(
@@ -121,7 +122,7 @@ class DorisReaderITCase extends DorisTestBase {
private def compareCollectResult(a1: Array[AnyRef], a2: Array[AnyRef]):
Boolean = {
if (a1.length == a2.length) {
- for (idx <- a1.indices) {
+ for (idx <- 0 until a1.length) {
if (!a1(idx).isInstanceOf[Array[AnyRef]] ||
!a2(idx).isInstanceOf[Array[AnyRef]]) {
return false
}
@@ -130,7 +131,7 @@ class DorisReaderITCase extends DorisTestBase {
if (arr1.length != arr2.length) {
return false
}
- for (idx2 <- arr1.indices) {
+ for (idx2 <- 0 until arr2.length) {
if (arr1(idx2) != arr2(idx2)) {
return false
}
diff --git a/spark-doris-connector/spark-doris-connector-shims/pom.xml
b/spark-doris-connector/spark-doris-connector-shims/pom.xml
deleted file mode 100644
index e0f5c0c..0000000
--- a/spark-doris-connector/spark-doris-connector-shims/pom.xml
+++ /dev/null
@@ -1,52 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector</artifactId>
- <version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <packaging>pom</packaging>
-
- <artifactId>spark-doris-connector-shims</artifactId>
-
- <modules>
- <module>spark-doris-connector-shims_spark-3.0</module>
- <module>spark-doris-connector-shims_spark-3.1</module>
- <module>spark-doris-connector-shims_spark-3.2</module>
- <module>spark-doris-connector-shims_spark-3.3</module>
- <module>spark-doris-connector-shims_spark-3.4</module>
- <module>spark-doris-connector-shims_spark-3.5</module>
- <module>spark-doris-connector-shims_spark-3-base</module>
- </modules>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.major.version}</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 0effe52..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- @throws[NoSuchNamespaceException]
- override def dropNamespace(namespace: Array[String]): Boolean = throw new
UnsupportedOperationException()
-
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index d137bd5..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-import org.apache.spark.sql.sources.DataSourceRegister
-
-abstract class DorisDataSource extends DataSourceRegister with
DorisSourceProvider with Serializable {
- override def shortName(): String = "doris"
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml
deleted file mode 100644
index 02675f9..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/pom.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
- <version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <packaging>jar</packaging>
-
- <artifactId>spark-doris-connector-shims_spark-3.1</artifactId>
-
- <properties>
- <spark.version>3.1.3</spark.version>
- <spark.major.version>3.1</spark.major.version>
- <scala.version>2.12.18</scala.version>
- <scala.major.version>2.12</scala.major.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 0effe52..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- @throws[NoSuchNamespaceException]
- override def dropNamespace(namespace: Array[String]): Boolean = throw new
UnsupportedOperationException()
-
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 782a63d..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait with
DorisSourceProvider with Serializable {}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 0effe52..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- @throws[NoSuchNamespaceException]
- override def dropNamespace(namespace: Array[String]): Boolean = throw new
UnsupportedOperationException()
-
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 782a63d..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait with
DorisSourceProvider with Serializable {}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 64e1f79..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean =
throw new UnsupportedOperationException()
-
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 782a63d..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait with
DorisSourceProvider with Serializable {}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 64e1f79..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean =
throw new UnsupportedOperationException()
-
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 1235f5b..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait {}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml
deleted file mode 100644
index 42d6d7b..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/pom.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
- <version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <packaging>jar</packaging>
-
- <artifactId>spark-doris-connector-shims_spark-3.5</artifactId>
-
- <properties>
- <spark.version>3.4.3</spark.version>
- <spark.major.version>3.4</spark.major.version>
- <scala.version>2.12.18</scala.version>
- <scala.major.version>2.12</scala.major.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
- </dependency>
- </dependencies>
-
-</project>
\ No newline at end of file
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
deleted file mode 100644
index 64e1f79..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.catalog
-
-import org.apache.spark.sql.connector.catalog.SupportsNamespaces
-
-abstract class DorisTableCatalogBase extends SupportsNamespaces {
-
- def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean =
throw new UnsupportedOperationException()
-
-}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
deleted file mode 100644
index 5450bbf..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.read
-
-import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
-import org.apache.spark.sql.types.StructType
-
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
b/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
deleted file mode 100644
index 7bec365..0000000
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.spark.sql.sources
-
-abstract class DorisDataSource extends DorisSourceRegisterTrait {}
\ No newline at end of file
diff --git a/spark-doris-connector/spark-doris-connector-spark-3/pom.xml
b/spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml
similarity index 79%
rename from spark-doris-connector/spark-doris-connector-spark-3/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml
index 4660d90..3d7499d 100644
--- a/spark-doris-connector/spark-doris-connector-spark-3/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3-base/pom.xml
@@ -26,22 +26,26 @@
<artifactId>spark-doris-connector</artifactId>
<version>${revision}</version>
</parent>
- <packaging>jar</packaging>
- <artifactId>spark-doris-connector-spark-3</artifactId>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.deploy.skip>true</maven.deploy.skip>
+ </properties>
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
-
<artifactId>spark-doris-connector-shims_spark-${spark.major.version}</artifactId>
- <version>${project.version}</version>
+ <artifactId>spark-doris-connector-base</artifactId>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.major.version}</artifactId>
- <version>${spark.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala
similarity index 85%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala
index d9aca8a..4cd5841 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableBase.scala
@@ -19,10 +19,8 @@ package org.apache.doris.spark.catalog
import org.apache.doris.spark.client.DorisFrontendClient
import org.apache.doris.spark.config.{DorisConfig, DorisOptions}
-import org.apache.doris.spark.read.DorisScanBuilder
import org.apache.doris.spark.rest.models.Schema
import org.apache.doris.spark.util.SchemaConvertors
-import org.apache.doris.spark.write.DorisWriteBuilder
import org.apache.spark.sql.connector.catalog.TableCapability._
import org.apache.spark.sql.connector.catalog.{Identifier, SupportsRead,
SupportsWrite, Table, TableCapability}
import org.apache.spark.sql.connector.read.ScanBuilder
@@ -35,7 +33,7 @@ import scala.collection.JavaConverters._
import scala.language.implicitConversions
-class DorisTable(identifier: Identifier, config: DorisConfig, schema:
Option[StructType]) extends Table with SupportsRead with SupportsWrite {
+abstract class DorisTableBase(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]) extends Table with SupportsRead with SupportsWrite {
private lazy val frontend:DorisFrontendClient = new
DorisFrontendClient(config)
@@ -54,12 +52,12 @@ class DorisTable(identifier: Identifier, config:
DorisConfig, schema: Option[Str
override def newScanBuilder(caseInsensitiveStringMap:
CaseInsensitiveStringMap): ScanBuilder = {
config.setProperty(DorisOptions.DORIS_TABLE_IDENTIFIER, name())
- new DorisScanBuilder(config, schema())
+ createScanBuilder(config, schema())
}
override def newWriteBuilder(logicalWriteInfo: LogicalWriteInfo):
WriteBuilder = {
config.setProperty(DorisOptions.DORIS_TABLE_IDENTIFIER, name())
- new DorisWriteBuilder(config, logicalWriteInfo.schema())
+ createWriteBuilder(config, logicalWriteInfo.schema())
}
private implicit def dorisSchemaToStructType(dorisSchema: Schema):
StructType = {
@@ -68,4 +66,8 @@ class DorisTable(identifier: Identifier, config: DorisConfig,
schema: Option[Str
}))
}
+ protected def createScanBuilder(config: DorisConfig, schema: StructType):
ScanBuilder
+
+ protected def createWriteBuilder(config: DorisConfig, schema: StructType):
WriteBuilder
+
}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
similarity index 89%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
index 9ca7f02..2c9ad0f 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalogBase.scala
@@ -20,7 +20,7 @@ package org.apache.doris.spark.catalog
import org.apache.doris.spark.client.DorisFrontendClient
import org.apache.doris.spark.config.{DorisConfig, DorisOptions}
import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException
-import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange,
Table, TableCatalog, TableChange}
+import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange,
SupportsNamespaces, Table, TableCatalog, TableChange}
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -28,13 +28,13 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
import java.util
import scala.collection.JavaConverters._
-class DorisTableCatalog extends DorisTableCatalogBase with TableCatalog {
+trait DorisTableCatalogBase extends TableCatalog with SupportsNamespaces {
- private var catalogName: Option[String] = None
+ protected var catalogName: Option[String] = None
- private var dorisConfig: DorisConfig = _
+ protected var dorisConfig: DorisConfig = _
- private var frontend: DorisFrontendClient = _
+ protected var frontend: DorisFrontendClient = _
override def name(): String = {
require(catalogName.nonEmpty, "The Doris table catalog is not initialed")
@@ -54,7 +54,7 @@ class DorisTableCatalog extends DorisTableCatalogBase with
TableCatalog {
override def loadTable(identifier: Identifier): Table = {
checkIdentifier(identifier)
- new DorisTable(identifier, DorisConfig.fromMap((dorisConfig.toMap.asScala +
+ newTableInstance(identifier,
DorisConfig.fromMap((dorisConfig.toMap.asScala +
(DorisOptions.DORIS_TABLE_IDENTIFIER.getName ->
getFullTableName(identifier))).asJava, false), None)
}
@@ -103,4 +103,6 @@ class DorisTableCatalog extends DorisTableCatalogBase with
TableCatalog {
(identifier.namespace() :+ identifier.name()).map(item =>
s"""`$item`""").mkString(".")
}
+ def newTableInstance(identifier: Identifier, config: DorisConfig, schema:
Option[StructType]): Table
+
}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala
similarity index 80%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala
index 4fad7ed..a03435d 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSourceV2.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/catalog/DorisTableProviderBase.scala
@@ -15,9 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.sql.sources
+package org.apache.doris.spark.catalog
-import org.apache.doris.spark.catalog.DorisTable
import org.apache.doris.spark.config.{DorisConfig, DorisOptions}
import org.apache.spark.sql.connector.catalog.{Identifier, Table,
TableProvider}
import org.apache.spark.sql.connector.expressions.Transform
@@ -26,9 +25,9 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
import java.util
-class DorisDataSourceV2 extends DorisDataSource with TableProvider {
+abstract class DorisTableProviderBase extends TableProvider {
- private var t: Table = _
+ protected var t: Table = _
override def inferSchema(options: CaseInsensitiveStringMap): StructType = {
if (t == null) t = getTable(options)
@@ -41,7 +40,7 @@ class DorisDataSourceV2 extends DorisDataSource with
TableProvider {
val dorisConfig = DorisConfig.fromMap(properties, false)
val tableIdentifier =
dorisConfig.getValue(DorisOptions.DORIS_TABLE_IDENTIFIER)
val tableIdentifierArr = tableIdentifier.split("\\.")
- new DorisTable(Identifier.of(Array[String](tableIdentifierArr(0)),
tableIdentifierArr(1)), dorisConfig, Some(schema))
+ newTableInstance(Identifier.of(Array[String](tableIdentifierArr(0)),
tableIdentifierArr(1)), dorisConfig, Some(schema))
}
}
@@ -51,8 +50,10 @@ class DorisDataSourceV2 extends DorisDataSource with
TableProvider {
val dorisConfig = DorisConfig.fromMap(options, false)
val tableIdentifier =
dorisConfig.getValue(DorisOptions.DORIS_TABLE_IDENTIFIER)
val tableIdentifierArr = tableIdentifier.split("\\.")
- new DorisTable(Identifier.of(Array[String](tableIdentifierArr(0)),
tableIdentifierArr(1)), dorisConfig, None)
+ newTableInstance(Identifier.of(Array[String](tableIdentifierArr(0)),
tableIdentifierArr(1)), dorisConfig, None)
}
}
+ def newTableInstance(identifier: Identifier, config: DorisConfig, schema:
Option[StructType]): Table;
+
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReader.scala
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisPartitionReaderFactory.scala
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScan.scala
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala
similarity index 94%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala
index 2a01a6d..a6a97dc 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderTrait.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/DorisScanBuilderBase.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.connector.read.{Scan,
ScanBuilder, SupportsPushDownF
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
-protected[spark] abstract class DorisScanBuilderTrait(config: DorisConfig,
schema: StructType) extends ScanBuilder
+protected[spark] abstract class DorisScanBuilderBase(config: DorisConfig,
schema: StructType) extends ScanBuilder
with SupportsPushDownFilters
with SupportsPushDownRequiredColumns {
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/read/ScanMode.java
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/sql/sources/DorisSourceProvider.scala
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriter.scala
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisDataWriterFactory.scala
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
similarity index 100%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWrite.scala
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
similarity index 100%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3-base/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml
b/spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml
similarity index 77%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml
index f010540..2dc1f38 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.1/pom.xml
@@ -23,17 +23,18 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
+ <artifactId>spark-doris-connector</artifactId>
<version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
</parent>
- <packaging>jar</packaging>
- <artifactId>spark-doris-connector-shims_spark-3.0</artifactId>
+ <artifactId>spark-doris-connector-spark-3.1</artifactId>
<properties>
- <spark.version>3.0.3</spark.version>
- <spark.major.version>3.0</spark.major.version>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <spark.version>3.1.0</spark.version>
+ <spark.major.version>3.1</spark.major.version>
<scala.version>2.12.18</scala.version>
<scala.major.version>2.12</scala.major.version>
</properties>
@@ -41,13 +42,11 @@
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
- <version>${project.version}</version>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.major.version}</artifactId>
- <scope>provided</scope>
</dependency>
</dependencies>
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
similarity index 93%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
copy to
spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
index 4c44d93..986b253 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-org.apache.doris.spark.sql.sources.DorisDataSourceV2
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
similarity index 55%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
index 521d06c..49a346a 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -15,21 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
-
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+class DorisTable(identifier: Identifier, config: DorisConfig, schema:
Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+ override def createScanBuilder(config: DorisConfig, schema: StructType):
ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema:
StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
similarity index 67%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
index 5450bbf..ca5b8c7 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.read
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String]): Boolean = throw new
UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
copy from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac..9e199af 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderBase(config, schema) {}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
similarity index 65%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
index 521d06c..102ac92 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.1/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -15,21 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.sql.sources
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
-
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+class DorisDataSource extends DorisTableProviderBase with
DorisSourceRegisterTrait with DorisSourceProvider with Serializable {
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml
b/spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml
similarity index 72%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml
index 68ae093..a4d8670 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.2/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.2/pom.xml
@@ -23,16 +23,17 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
+ <artifactId>spark-doris-connector</artifactId>
<version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
</parent>
- <packaging>jar</packaging>
- <artifactId>spark-doris-connector-shims_spark-3.2</artifactId>
+ <artifactId>spark-doris-connector-spark-3.2</artifactId>
<properties>
- <spark.version>3.2.4</spark.version>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <spark.version>3.2.0</spark.version>
<spark.major.version>3.2</spark.major.version>
<scala.version>2.12.18</scala.version>
<scala.major.version>2.12</scala.major.version>
@@ -41,7 +42,11 @@
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.major.version}</artifactId>
</dependency>
</dependencies>
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
similarity index 93%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
copy to
spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
index 4c44d93..986b253 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-org.apache.doris.spark.sql.sources.DorisDataSourceV2
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
similarity index 55%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
index 521d06c..49a346a 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -15,21 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
-
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+class DorisTable(identifier: Identifier, config: DorisConfig, schema:
Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
+ override def createScanBuilder(config: DorisConfig, schema: StructType):
ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema:
StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
similarity index 67%
copy from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
index 5450bbf..ca5b8c7 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.read
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String]): Boolean = throw new
UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac..9e199af 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.1/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderBase(config, schema) {}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
similarity index 65%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
index 521d06c..102ac92 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.2/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -15,21 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.sql.sources
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
-
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+class DorisDataSource extends DorisTableProviderBase with
DorisSourceRegisterTrait with DorisSourceProvider with Serializable {
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml
b/spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml
similarity index 72%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml
index a33eb77..ecc71ed 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.3/pom.xml
@@ -23,16 +23,17 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
+ <artifactId>spark-doris-connector</artifactId>
<version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
</parent>
- <packaging>jar</packaging>
- <artifactId>spark-doris-connector-shims_spark-3.3</artifactId>
+ <artifactId>spark-doris-connector-spark-3.3</artifactId>
<properties>
- <spark.version>3.3.4</spark.version>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <spark.version>3.3.0</spark.version>
<spark.major.version>3.3</spark.major.version>
<scala.version>2.12.18</scala.version>
<scala.major.version>2.12</scala.major.version>
@@ -41,7 +42,11 @@
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.major.version}</artifactId>
</dependency>
</dependencies>
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
similarity index 93%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
copy to
spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
index 4c44d93..986b253 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-org.apache.doris.spark.sql.sources.DorisDataSourceV2
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
similarity index 55%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
index 521d06c..be461af 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
+class DorisTable(identifier: Identifier, config: DorisConfig, schema:
Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+ override def createScanBuilder(config: DorisConfig, schema: StructType):
ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema:
StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
similarity index 67%
copy from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
index 5450bbf..e3ffcc9 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.read
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String], b: Boolean): Boolean =
throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac..9e199af 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.0/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderBase(config, schema) {}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
similarity index 66%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
index 521d06c..c00cd24 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.3/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -15,21 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.sql.sources
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
+class DorisDataSource extends DorisTableProviderBase with
DorisSourceRegisterTrait with Serializable {
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml
b/spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml
similarity index 72%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml
index b06e8bd..eeee285 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.4/pom.xml
@@ -23,16 +23,17 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
+ <artifactId>spark-doris-connector</artifactId>
<version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
</parent>
- <packaging>jar</packaging>
- <artifactId>spark-doris-connector-shims_spark-3.4</artifactId>
+ <artifactId>spark-doris-connector-spark-3.4</artifactId>
<properties>
- <spark.version>3.4.3</spark.version>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <spark.version>3.4.0</spark.version>
<spark.major.version>3.4</spark.major.version>
<scala.version>2.12.18</scala.version>
<scala.major.version>2.12</scala.major.version>
@@ -41,7 +42,11 @@
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.major.version}</artifactId>
</dependency>
</dependencies>
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
similarity index 93%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
copy to
spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
index 4c44d93..986b253 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-org.apache.doris.spark.sql.sources.DorisDataSourceV2
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
similarity index 55%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
index 521d06c..be461af 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
+class DorisTable(identifier: Identifier, config: DorisConfig, schema:
Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+ override def createScanBuilder(config: DorisConfig, schema: StructType):
ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema:
StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
similarity index 67%
copy from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
index 5450bbf..e3ffcc9 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.read
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String], b: Boolean): Boolean =
throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 90%
copy from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index 5450bbf..9e199af 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -18,7 +18,6 @@
package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderBase(config, schema) {}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
similarity index 66%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
index 521d06c..c00cd24 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.4/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -15,21 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.sql.sources
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
+class DorisDataSource extends DorisTableProviderBase with
DorisSourceRegisterTrait with Serializable {
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml
b/spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml
similarity index 70%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml
rename to spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml
index 59293ed..2f498b4 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3-base/pom.xml
+++ b/spark-doris-connector/spark-doris-connector-spark-3.5/pom.xml
@@ -23,17 +23,18 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-shims</artifactId>
+ <artifactId>spark-doris-connector</artifactId>
<version>${revision}</version>
- <relativePath>../pom.xml</relativePath>
</parent>
- <packaging>jar</packaging>
- <artifactId>spark-doris-connector-shims_spark-3-base</artifactId>
+ <artifactId>spark-doris-connector-spark-3.5</artifactId>
<properties>
- <spark.version>3.0.3</spark.version>
- <spark.major.version>3.0</spark.major.version>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <spark.version>3.5.0</spark.version>
+ <spark.major.version>3.5</spark.major.version>
<scala.version>2.12.18</scala.version>
<scala.major.version>2.12</scala.major.version>
</properties>
@@ -41,7 +42,11 @@
<dependencies>
<dependency>
<groupId>org.apache.doris</groupId>
- <artifactId>spark-doris-connector-base</artifactId>
+ <artifactId>spark-doris-connector-spark-3-base</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.major.version}</artifactId>
</dependency>
</dependencies>
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
similarity index 93%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
rename to
spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
index 4c44d93..986b253 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
+++
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/resources/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -15,4 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-org.apache.doris.spark.sql.sources.DorisDataSourceV2
+org.apache.doris.spark.sql.sources.DorisDataSource
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
similarity index 55%
copy from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
copy to
spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
index 521d06c..be461af 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTable.scala
@@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.doris.spark.read.DorisScanBuilder
+import org.apache.doris.spark.write.DorisWriteBuilder
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.connector.read.ScanBuilder
+import org.apache.spark.sql.connector.write.WriteBuilder
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
+class DorisTable(identifier: Identifier, config: DorisConfig, schema:
Option[StructType])
+ extends DorisTableBase(identifier, config, schema) {
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+ override def createScanBuilder(config: DorisConfig, schema: StructType):
ScanBuilder = new DorisScanBuilder(config, schema)
+ override protected def createWriteBuilder(config: DorisConfig, schema:
StructType): WriteBuilder = new DorisWriteBuilder(config, schema)
}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
similarity index 67%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
index 5450bbf..e3ffcc9 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.3/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/catalog/DorisTableCatalog.scala
@@ -15,10 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.read
+package org.apache.doris.spark.catalog
import org.apache.doris.spark.config.DorisConfig
-import org.apache.doris.spark.read.DorisScanBuilderTrait
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisTableCatalog extends DorisTableCatalogBase {
+ override def dropNamespace(strings: Array[String], b: Boolean): Boolean =
throw new UnsupportedOperationException()
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
+}
diff --git
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
similarity index 95%
rename from
spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
index c6abaac..9e199af 100644
---
a/spark-doris-connector/spark-doris-connector-shims/spark-doris-connector-shims_spark-3.4/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/read/DorisScanBuilder.scala
@@ -20,4 +20,4 @@ package org.apache.doris.spark.read
import org.apache.doris.spark.config.DorisConfig
import org.apache.spark.sql.types.StructType
-class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderTrait(config, schema) {}
+class DorisScanBuilder(config: DorisConfig, schema: StructType) extends
DorisScanBuilderBase(config, schema) {}
diff --git
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
similarity index 66%
rename from
spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
rename to
spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
index 521d06c..c00cd24 100644
---
a/spark-doris-connector/spark-doris-connector-spark-3/src/main/scala/org/apache/doris/spark/write/DorisWriteBuilder.scala
+++
b/spark-doris-connector/spark-doris-connector-spark-3.5/src/main/scala/org/apache/doris/spark/sql/sources/DorisDataSource.scala
@@ -15,21 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.spark.write
+package org.apache.doris.spark.sql.sources
+import org.apache.doris.spark.catalog.{DorisTable, DorisTableProviderBase}
import org.apache.doris.spark.config.DorisConfig
-import org.apache.spark.sql.connector.write.streaming.StreamingWrite
-import org.apache.spark.sql.connector.write.{BatchWrite, WriteBuilder}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table}
import org.apache.spark.sql.types.StructType
-class DorisWriteBuilder(config: DorisConfig, schema: StructType) extends
WriteBuilder {
+class DorisDataSource extends DorisTableProviderBase with
DorisSourceRegisterTrait with Serializable {
- override def buildForBatch(): BatchWrite = {
- new DorisWrite(config, schema)
- }
-
- override def buildForStreaming(): StreamingWrite = {
- new DorisWrite(config, schema)
- }
+ override def newTableInstance(identifier: Identifier, config: DorisConfig,
schema: Option[StructType]): Table =
+ new DorisTable(identifier, config, schema)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
