This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 789dd586cc [GLUTEN-7913][CORE] Flip dependency direction for
gluten-uniffle (#8242)
789dd586cc is described below
commit 789dd586cc6e22caf528982687df0f1cd120d1fd
Author: Hongze Zhang <[email protected]>
AuthorDate: Tue Dec 17 11:12:14 2024 +0800
[GLUTEN-7913][CORE] Flip dependency direction for gluten-uniffle (#8242)
Closes #7913
---
backends-velox/pom.xml | 19 +++++++
.../gluten/uniffle/UniffleShuffleManager.java | 0
.../writer/VeloxUniffleColumnarShuffleWriter.java | 0
.../spark/shuffle/writer/PartitionPusher.scala | 0
gluten-uniffle/.gitkeep | 1 +
gluten-uniffle/package/pom.xml | 29 ----------
gluten-uniffle/pom.xml | 13 +----
gluten-uniffle/velox/pom.xml | 62 ---------------------
package/pom.xml | 2 +-
pom.xml | 64 ++++++++++++++++++++++
10 files changed, 86 insertions(+), 104 deletions(-)
diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index 9349c3c092..240b221864 100755
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -57,6 +57,25 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>uniffle</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>gluten-uniffle</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+
<artifactId>rss-client-spark${spark.major.version}-shaded</artifactId>
+ <version>${uniffle.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
<profile>
<id>iceberg</id>
<activation>
diff --git
a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
b/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
similarity index 100%
rename from
gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
rename to
backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
diff --git
a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
b/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
similarity index 100%
rename from
gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
rename to
backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
diff --git
a/gluten-uniffle/velox/src/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
b/backends-velox/src-uniffle/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
similarity index 100%
rename from
gluten-uniffle/velox/src/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
rename to
backends-velox/src-uniffle/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
diff --git a/gluten-uniffle/.gitkeep b/gluten-uniffle/.gitkeep
new file mode 100644
index 0000000000..f2d1254d27
--- /dev/null
+++ b/gluten-uniffle/.gitkeep
@@ -0,0 +1 @@
+The module is kept for adding common code shared by backends for Uniffle
support in Gluten.
diff --git a/gluten-uniffle/package/pom.xml b/gluten-uniffle/package/pom.xml
deleted file mode 100644
index e49748e7c8..0000000000
--- a/gluten-uniffle/package/pom.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <parent>
- <artifactId>gluten-uniffle</artifactId>
- <groupId>org.apache.gluten</groupId>
- <version>1.3.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>gluten-uniffle-package</artifactId>
- <packaging>jar</packaging>
- <name>Gluten Uniffle Package</name>
-
- <profiles>
- <profile>
- <id>backends-velox</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.gluten</groupId>
- <artifactId>gluten-uniffle-velox</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
- </profiles>
-</project>
diff --git a/gluten-uniffle/pom.xml b/gluten-uniffle/pom.xml
index b7fe4c2e42..efc8ce6555 100644
--- a/gluten-uniffle/pom.xml
+++ b/gluten-uniffle/pom.xml
@@ -11,7 +11,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>gluten-uniffle</artifactId>
- <packaging>pom</packaging>
+ <packaging>jar</packaging>
<name>Gluten Uniffle</name>
<dependencies>
@@ -75,15 +75,4 @@
</plugin>
</plugins>
</build>
- <profiles>
- <profile>
- <id>backends-velox</id>
- <properties>
- </properties>
- <modules>
- <module>velox</module>
- <module>package</module>
- </modules>
- </profile>
- </profiles>
</project>
diff --git a/gluten-uniffle/velox/pom.xml b/gluten-uniffle/velox/pom.xml
deleted file mode 100755
index ab730674fb..0000000000
--- a/gluten-uniffle/velox/pom.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xmlns="http://maven.apache.org/POM/4.0.0";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <parent>
- <artifactId>gluten-uniffle</artifactId>
- <groupId>org.apache.gluten</groupId>
- <version>1.3.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>gluten-uniffle-velox</artifactId>
- <packaging>jar</packaging>
- <name>Gluten Uniffle Velox</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.gluten</groupId>
- <artifactId>backends-velox</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.gluten</groupId>
- <artifactId>gluten-arrow</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
-
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
-
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>com.diffplug.spotless</groupId>
- <artifactId>spotless-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/package/pom.xml b/package/pom.xml
index 230f79d094..b9c114181b 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -78,7 +78,7 @@
<dependencies>
<dependency>
<groupId>org.apache.gluten</groupId>
- <artifactId>gluten-uniffle-package</artifactId>
+ <artifactId>gluten-uniffle</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
diff --git a/pom.xml b/pom.xml
index e6f3709c4c..4d704dc9b4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -495,6 +495,70 @@
<modules>
<module>gluten-uniffle</module>
</modules>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-uniffle-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.basedir}/src-uniffle/main/scala</source>
+ <source>${project.basedir}/src-uniffle/main/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-uniffle-resources</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>add-resource</goal>
+ </goals>
+ <configuration>
+ <resources>
+ <resource>
+
<directory>${project.basedir}/src-uniffle/main/resources</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-uniffle-test-sources</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.basedir}/src-uniffle/test/scala</source>
+ <source>${project.basedir}/src-uniffle/test/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ <execution>
+ <id>add-uniffle-test-resources</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>add-test-resource</goal>
+ </goals>
+ <configuration>
+ <resources>
+ <resource>
+
<directory>${project.basedir}/src-uniffle/test/resources</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</profile>
<profile>
<id>delta</id>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
