This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 5fd68764 [#747] feat(tez): Add Tez Framework (#748)
5fd68764 is described below
commit 5fd68764d6fc9e085c89f9ddabd62d99eea6163e
Author: roryqi <[email protected]>
AuthorDate: Tue Mar 21 19:33:43 2023 +0800
[#747] feat(tez): Add Tez Framework (#748)
### What changes were proposed in this pull request?
Add directories for Apache Tez framework
### Why are the changes needed?
Fix: #747
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
mvn package -Ptez -DskipTests
---
.asf.yaml | 1 +
client-tez/pom.xml | 179 +++++++++++++++++++++
.../spark-2.4.6_dynamic_allocation_support.patch | 0
.../spark-3.1.2_dynamic_allocation_support.patch | 0
.../spark-3.2.1_dynamic_allocation_support.patch | 0
patch/tez/.gitkeep | 16 ++
pom.xml | 22 ++-
7 files changed, 213 insertions(+), 5 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
index 8994f9f8..50e15f79 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -24,6 +24,7 @@ github:
- shuffle
- remote-shuffle-service
- rss
+ - tez
features:
# Enable wiki for documentation
wiki: true
diff --git a/client-tez/pom.xml b/client-tez/pom.xml
new file mode 100644
index 00000000..fcc8931b
--- /dev/null
+++ b/client-tez/pom.xml
@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>uniffle-parent</artifactId>
+ <groupId>org.apache.uniffle</groupId>
+ <version>0.8.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-tez</artifactId>
+ <version>0.8.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <name>Apache Uniffle Client (Tez)</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty-shaded</artifactId>
+ <version>${grpc.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.github.luben</groupId>
+ <artifactId>zstd-jni</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>rss-client-mr-jar</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+
<outputDirectory>${project.build.directory}/shaded</outputDirectory>
+
<shadedArtifactAttached>true</shadedArtifactAttached>
+ <artifactSet>
+ <includes>
+ <include>org.apache.uniffle:*</include>
+ <include>com.google.protobuf:*</include>
+
<include>com.google.code.gson:gson</include>
+ <include>io.grpc:*</include>
+
<include>com.google.android:annotations</include>
+ <include>io.perfmark:perfmark-api</include>
+ <include>io.netty:netty-all</include>
+
<include>com.google.api.grpc:proto-google-common-protos</include>
+
<include>org.codehaus.mojo:animal-sniffer-annotations</include>
+ <include>com.google.guava:*</include>
+
<include>com.fasterxml.jackson.core:jackson-databind</include>
+
<include>com.fasterxml.jackson.core:jackson-core</include>
+
<include>com.fasterxml.jackson.core:jackson-annotations</include>
+
<include>org.roaringbitmap:RoaringBitmap</include>
+ <include>org.roaringbitmap:shims</include>
+ <include>net.jpountz.lz4:lz4</include>
+ </includes>
+ </artifactSet>
+
<finalName>${project.artifactId}-${project.version}</finalName>
+ <transformers>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>LICENSE</exclude>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+
<shadedPattern>${rss.shade.packageName}.com.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.common</pattern>
+
<shadedPattern>${rss.shade.packageName}.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>google.protobuf</pattern>
+
<shadedPattern>${rss.shade.packageName}.google.protobuf</shadedPattern>
+ </relocation>
+ <relocation>
+
<pattern>com.fasterxml.jackson.core</pattern>
+
<shadedPattern>${rss.shade.packageName}.jackson.core</shadedPattern>
+ </relocation>
+ <relocation>
+
<pattern>com.fasterxml.jackson.databind</pattern>
+
<shadedPattern>${rss.shade.packageName}.jackson.databind</shadedPattern>
+ </relocation>
+ <relocation>
+
<pattern>com.fasterxml.jackson.annotation</pattern>
+
<shadedPattern>${rss.shade.packageName}.jackson.annotation</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+
<shadedPattern>${rss.shade.packageName}.io.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.roaringbitmap</pattern>
+
<shadedPattern>${rss.shade.packageName}.org.roaringbitmap</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/spark-patches/spark-2.4.6_dynamic_allocation_support.patch
b/patch/spark/spark-2.4.6_dynamic_allocation_support.patch
similarity index 100%
rename from spark-patches/spark-2.4.6_dynamic_allocation_support.patch
rename to patch/spark/spark-2.4.6_dynamic_allocation_support.patch
diff --git a/spark-patches/spark-3.1.2_dynamic_allocation_support.patch
b/patch/spark/spark-3.1.2_dynamic_allocation_support.patch
similarity index 100%
rename from spark-patches/spark-3.1.2_dynamic_allocation_support.patch
rename to patch/spark/spark-3.1.2_dynamic_allocation_support.patch
diff --git a/spark-patches/spark-3.2.1_dynamic_allocation_support.patch
b/patch/spark/spark-3.2.1_dynamic_allocation_support.patch
similarity index 100%
rename from spark-patches/spark-3.2.1_dynamic_allocation_support.patch
rename to patch/spark/spark-3.2.1_dynamic_allocation_support.patch
diff --git a/patch/tez/.gitkeep b/patch/tez/.gitkeep
new file mode 100644
index 00000000..ecb1860d
--- /dev/null
+++ b/patch/tez/.gitkeep
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index affa0ef2..64f3a76f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1600,11 +1600,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.openjdk.jol</groupId>
- <artifactId>jol-core</artifactId>
- <version>0.16</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
@@ -1614,6 +1609,23 @@
</dependencies>
</dependencyManagement>
</profile>
+ <profile>
+ <id>tez</id>
+ <modules>
+ <module>client-tez</module>
+ </modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
<profile>
<id>kubernetes</id>
<modules>
