This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 94b2b9bb3 [KYUUBI #5113] Remove dummy Spark Kudu connector module
94b2b9bb3 is described below
commit 94b2b9bb336950723df604cc56cf3fbf529e86a9
Author: Cheng Pan <[email protected]>
AuthorDate: Tue Aug 1 17:05:42 2023 +0800
[KYUUBI #5113] Remove dummy Spark Kudu connector module
### _Why are the changes needed?_
It was planned but actually delayed, remove this dummy module to save CI
and avoid confusing users and release managers.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
Closes #5113 from pan3793/remove-kudu.
Closes #5113
ff8fd2e6a [Cheng Pan] Remove Spark Kudu connector
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/style.yml | 2 +-
dev/kyuubi-codecov/pom.xml | 5 -
.../spark/kyuubi-spark-connector-kudu/pom.xml | 225 ---------------------
.../src/test/resources/kudu-compose.yml | 64 ------
.../src/test/resources/log4j2-test.xml | 43 ----
.../spark/connector/kudu/KuduClientSuite.scala | 32 ---
.../kyuubi/spark/connector/kudu/KuduMixin.scala | 41 ----
pom.xml | 8 -
8 files changed, 1 insertion(+), 419 deletions(-)
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index 11a9580c4..6f575302e 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -67,7 +67,7 @@ jobs:
MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn
-Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true
-Dspotless.check.skip"
build/mvn clean install ${MVN_OPT}
-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds
build/mvn clean install ${MVN_OPT} -pl
extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1
- build/mvn clean install ${MVN_OPT} -pl
extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-kudu,extensions/spark/kyuubi-spark-connector-hive
-Pspark-3.3
+ build/mvn clean install ${MVN_OPT} -pl
extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive
-Pspark-3.3
build/mvn clean install ${MVN_OPT} -pl
extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4
- name: Scalastyle with maven
diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml
index 3dfc7b986..09ced96a1 100644
--- a/dev/kyuubi-codecov/pom.xml
+++ b/dev/kyuubi-codecov/pom.xml
@@ -197,11 +197,6 @@
<artifactId>kyuubi-extension-spark-3-3_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.kyuubi</groupId>
-
<artifactId>kyuubi-spark-connector-kudu_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.kyuubi</groupId>
<artifactId>kyuubi-spark-connector-hive_${scala.binary.version}</artifactId>
diff --git a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml
b/extensions/spark/kyuubi-spark-connector-kudu/pom.xml
deleted file mode 100644
index cb1cec449..000000000
--- a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml
+++ /dev/null
@@ -1,225 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-parent</artifactId>
- <version>1.8.0-SNAPSHOT</version>
- <relativePath>../../../pom.xml</relativePath>
- </parent>
-
- <artifactId>kyuubi-spark-connector-kudu_2.12</artifactId>
- <packaging>jar</packaging>
- <name>Kyuubi Spark Kudu Connector</name>
- <url>https://kyuubi.apache.org/</url>
-
- <dependencies>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-api</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kudu</groupId>
- <artifactId>kudu-client</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scalatestplus</groupId>
- <artifactId>scalacheck-1-17_${scala.binary.version}</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.dimafeng</groupId>
-
<artifactId>testcontainers-scala-scalatest_${scala.binary.version}</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kyuubi</groupId>
- <artifactId>kyuubi-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-runtime</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!--
- Spark requires `commons-collections` and `commons-io` but got them
from transitive
- dependencies of `hadoop-client`. As we are using Hadoop Shaded
Client, we need add
- them explicitly. See more details at SPARK-33212.
- -->
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>jakarta.xml.bind</groupId>
- <artifactId>jakarta.xml.bind-api</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
-
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <configuration>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <artifactSet>
- <includes>
- <include>org.apache.kudu:kudu-client</include>
- <include>org.apache.kyuubi:*</include>
- <include>com.stumbleupon:async</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>org.apache.kudu:kudu-client</artifact>
- <excludes>
- <exclude>META-INF/maven/**</exclude>
- <exclude>META-INF/native/**</exclude>
- <exclude>META-INF/native-image/**</exclude>
- <exclude>MANIFEST.MF</exclude>
- <exclude>LICENSE</exclude>
- <exclude>LICENSE.txt</exclude>
- <exclude>NOTICE</exclude>
- <exclude>NOTICE.txt</exclude>
- <exclude>*.properties</exclude>
- <exclude>**/*.proto</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>org.apache.kudu</pattern>
-
<shadedPattern>${kyuubi.shade.packageName}.org.apache.kudu</shadedPattern>
- <includes>
- <include>org.apache.kudu.**</include>
- </includes>
- </relocation>
- <relocation>
- <pattern>com.stumbleupon:async</pattern>
-
<shadedPattern>${kyuubi.shade.packageName}.com.stumbleupon.async</shadedPattern>
- <includes>
- <include>com.stumbleupon.async.**</include>
- </includes>
- </relocation>
- </relocations>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>shade</goal>
- </goals>
- <phase>package</phase>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <id>prepare-test-jar</id>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <phase>test-compile</phase>
- </execution>
- </executions>
- </plugin>
- </plugins>
-
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
-
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
- </build>
-</project>
diff --git
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml
b/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml
deleted file mode 100644
index 149cd5d47..000000000
---
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/kudu-compose.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-version: "3"
-services:
- kudu-master:
- image: apache/kudu:1.15.0
- hostname: kudu-master
- ports:
- - "7051"
- - "8051"
- command: ["master"]
- environment:
- - KUDU_MASTERS=kudu-master
-
- kudu-tserver-1:
- image: apache/kudu:1.15.0
- depends_on:
- - kudu-master
- hostname: kudu-tserver-1
- ports:
- - "7050"
- - "8050"
- command: ["tserver"]
- environment:
- - KUDU_MASTERS=kudu-master
-
- kudu-tserver-2:
- image: apache/kudu:1.15.0
- depends_on:
- - kudu-master
- hostname: kudu-tserver-2
- ports:
- - "7050"
- - "8050"
- command: [ "tserver" ]
- environment:
- - KUDU_MASTERS=kudu-master
-
- kudu-tserver-3:
- image: apache/kudu:1.15.0
- depends_on:
- - kudu-master
- hostname: kudu-tserver-3
- ports:
- - "7050"
- - "8050"
- command: [ "tserver" ]
- environment:
- - KUDU_MASTERS=kudu-master
diff --git
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml
b/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml
deleted file mode 100644
index bfc40dd6d..000000000
---
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!-- Extra logging related to initialization of Log4j.
- Set to debug or trace if log4j initialization is failing. -->
-<Configuration status="WARN">
- <Appenders>
- <Console name="stdout" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{HH:mm:ss.SSS} %p %c: %m%n"/>
- <Filters>
- <ThresholdFilter level="FATAL"/>
- <RegexFilter regex=".*Thrift error occurred during processing
of message.*" onMatch="DENY" onMismatch="NEUTRAL"/>
- </Filters>
- </Console>
- <File name="file" fileName="target/unit-tests.log">
- <PatternLayout pattern="%d{HH:mm:ss.SSS} %t %p %c{1}: %m%n"/>
- <Filters>
- <RegexFilter regex=".*Thrift error occurred during processing
of message.*" onMatch="DENY" onMismatch="NEUTRAL"/>
- </Filters>
- </File>
- </Appenders>
- <Loggers>
- <Root level="INFO">
- <AppenderRef ref="stdout"/>
- <AppenderRef ref="file"/>
- </Root>
- </Loggers>
-</Configuration>
diff --git
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala
b/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala
deleted file mode 100644
index eebb4719c..000000000
---
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.spark.connector.kudu
-
-import org.apache.kudu.client.KuduClient
-
-import org.apache.kyuubi.KyuubiFunSuite
-
-class KuduClientSuite extends KyuubiFunSuite with KuduMixin {
-
- test("kudu client") {
- val builder = new KuduClient.KuduClientBuilder(kuduMasterUrl)
- val kuduClient = builder.build()
-
- assert(kuduClient.findLeaderMasterServer().getPort === kuduMasterPort)
- }
-}
diff --git
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
b/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
deleted file mode 100644
index dee09db38..000000000
---
a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.spark.connector.kudu
-
-import java.io.File
-
-import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService,
ForAllTestContainer}
-
-import org.apache.kyuubi.{KyuubiFunSuite, Utils}
-
-trait KuduMixin extends KyuubiFunSuite with ForAllTestContainer {
-
- private val KUDU_MASTER_PORT = 7051
-
- override val container: DockerComposeContainer =
- DockerComposeContainer
- .Def(
- composeFiles =
- new
File(Utils.getContextOrKyuubiClassLoader.getResource("kudu-compose.yml").toURI),
- exposedServices = ExposedService("kudu-master", KUDU_MASTER_PORT) ::
Nil)
- .createContainer()
-
- def kuduMasterHost: String = container.getServiceHost("kudu-master",
KUDU_MASTER_PORT)
- def kuduMasterPort: Int = container.getServicePort("kudu-master",
KUDU_MASTER_PORT)
- def kuduMasterUrl: String = s"$kuduMasterHost:$kuduMasterPort"
-}
diff --git a/pom.xml b/pom.xml
index e3da6a6a8..9d30d0ec4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -169,7 +169,6 @@
<junit.version>4.13.2</junit.version>
<kafka.version>3.4.0</kafka.version>
<kubernetes-client.version>6.7.2</kubernetes-client.version>
- <kudu.version>1.15.0</kudu.version>
<kyuubi-shaded-zookeeper.artifacts>kyuubi-shaded-zookeeper-34</kyuubi-shaded-zookeeper.artifacts>
<kyuubi-shaded-zookeeper.version>0.1.0</kyuubi-shaded-zookeeper.version>
<ldapsdk.version>6.0.5</ldapsdk.version>
@@ -1461,11 +1460,6 @@
<version>${sqlite.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.kudu</groupId>
- <artifactId>kudu-client</artifactId>
- <version>${kudu.version}</version>
- </dependency>
<dependency>
<groupId>net.sf.py4j</groupId>
<artifactId>py4j</artifactId>
@@ -2166,7 +2160,6 @@
<module>extensions/spark/kyuubi-extension-spark-common</module>
<module>extensions/spark/kyuubi-extension-spark-3-3</module>
<module>extensions/spark/kyuubi-spark-connector-hive</module>
- <module>extensions/spark/kyuubi-spark-connector-kudu</module>
</modules>
<properties>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
@@ -2178,7 +2171,6 @@
<modules>
<module>extensions/spark/kyuubi-extension-spark-3-4</module>
<module>extensions/spark/kyuubi-spark-connector-hive</module>
- <module>extensions/spark/kyuubi-spark-connector-kudu</module>
</modules>
<properties>
<delta.version>2.4.0</delta.version>
