TEZ-3390. Package Shuffle Handler as a shaded uber-jar (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/397d6af3 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/397d6af3 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/397d6af3 Branch: refs/heads/master Commit: 397d6af393a29ababaf2b249b320a7f95549e8c8 Parents: bae21b4 Author: Jonathan Eagles <[email protected]> Authored: Tue Aug 9 14:48:58 2016 -0500 Committer: Jonathan Eagles <[email protected]> Committed: Tue Aug 9 14:48:58 2016 -0500 ---------------------------------------------------------------------- TEZ-3334-CHANGES.txt | 1 + pom.xml | 18 +++ tez-plugins/tez-aux-services/pom.xml | 133 ++++++++++++++++--- .../tez/auxservices/FadvisedChunkedFile.java | 7 +- .../tez/auxservices/FadvisedFileRegion.java | 7 +- .../apache/tez/auxservices/ShuffleHandler.java | 11 +- .../tez/auxservices/TestShuffleHandler.java | 6 +- 7 files changed, 150 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/TEZ-3334-CHANGES.txt ---------------------------------------------------------------------- diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt index 6e51be2..ef0a1cb 100644 --- a/TEZ-3334-CHANGES.txt +++ b/TEZ-3334-CHANGES.txt @@ -4,6 +4,7 @@ Apache Tez Change Log INCOMPATIBLE CHANGES: ALL CHANGES: + TEZ-3390. Package Shuffle Handler as a shaded uber-jar TEZ-3378. Move Shuffle Handler configuration into the Tez namespace TEZ-3377. Remove ShuffleHandler dependency on mapred.FadvisedChunkedFile and mapred.FadvisedFileRegion TEZ-3380. Shuffle Handler: Replace primitive wrapper's valueOf method with parse* method to avoid unnecessary boxing/unboxing http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6e4fe40..eb2dc7d 100644 --- a/pom.xml +++ b/pom.xml @@ -39,6 +39,7 @@ <clover.license>${user.home}/clover.license</clover.license> <hadoop.version>2.6.0</hadoop.version> <jetty.version>6.1.26</jetty.version> + <netty.version>3.6.2.Final</netty.version> <pig.version>0.13.0</pig.version> <javac.version>1.7</javac.version> <slf4j.version>1.7.10</slf4j.version> @@ -58,6 +59,7 @@ <frontend-maven-plugin.version>0.0.23</frontend-maven-plugin.version> <findbugs-maven-plugin.version>3.0.1</findbugs-maven-plugin.version> <javadoc-maven-plugin.version>2.9.1</javadoc-maven-plugin.version> + <shade-maven-plugin.version>2.4.3</shade-maven-plugin.version> </properties> <scm> <connection>${scm.url}</connection> @@ -243,6 +245,12 @@ <version>${jetty.version}</version> </dependency> <dependency> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> + <scope>compile</scope> + <version>${netty.version}</version> + </dependency> + <dependency> <groupId>org.mortbay.jetty</groupId> <artifactId>jetty-util</artifactId> <scope>compile</scope> @@ -717,6 +725,11 @@ <version>1.46</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.fusesource.leveldbjni</groupId> + <artifactId>leveldbjni-all</artifactId> + <version>1.8</version> + </dependency> </dependencies> </dependencyManagement> @@ -823,6 +836,11 @@ <version>2.5</version> </plugin> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>${shade-maven-plugin.version}</version> + </plugin> + <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <version>0.10</version> http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/tez-plugins/tez-aux-services/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/pom.xml b/tez-plugins/tez-aux-services/pom.xml index a8876a7..93fef71 100644 --- a/tez-plugins/tez-aux-services/pom.xml +++ b/tez-plugins/tez-aux-services/pom.xml @@ -27,47 +27,82 @@ <dependencies> <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-annotations</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <scope>provided</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - <scope>test</scope> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + <scope>provided</scope> </dependency> <dependency> - <!-- Required for the ShuffleHandler --> - <groupId>org.apache.tez</groupId> - <artifactId>tez-runtime-library</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-common</artifactId> + <artifactId>hadoop-mapreduce-client-common</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-shuffle</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tez</groupId> + <artifactId>tez-api</artifactId> </dependency> <dependency> <groupId>org.apache.tez</groupId> <artifactId>tez-mapreduce</artifactId> </dependency> + <dependency> + <!-- Required for the ShuffleHandler --> + <groupId>org.apache.tez</groupId> + <artifactId>tez-runtime-library</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>org.fusesource.leveldbjni</groupId> + <artifactId>leveldbjni-all</artifactId> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> @@ -98,7 +133,69 @@ <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>false</createDependencyReducedPom> + <minimizeJar>true</minimizeJar> + <artifactSet> + </artifactSet> + <relocations> + <relocation> + <pattern>com.google.common</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>com.google.protobuf</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.commons</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.apache.log4j</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.fusesource.hawtjni</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.fusesource.leveldbjni</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.iq80.leveldb</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.jboss.netty</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>org.slf4j</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + </relocation> + <relocation> + <pattern>javax</pattern> + <shadedPattern>org.apache.tez.shaded.$0</shadedPattern> + <excludes> + <exclude>javax.crypto.*</exclude> + </excludes> + </relocation> + </relocations> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> - </project> http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java index e14b79d..cc3f762 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java @@ -22,19 +22,20 @@ import java.io.FileDescriptor; import java.io.IOException; import java.io.RandomAccessFile; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.io.nativeio.NativeIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; import org.jboss.netty.handler.stream.ChunkedFile; public class FadvisedChunkedFile extends ChunkedFile { - private static final Log LOG = LogFactory.getLog(FadvisedChunkedFile.class); + private static final Logger LOG = LoggerFactory.getLogger(FadvisedChunkedFile.class); private final boolean manageOsCache; private final int readaheadLength; http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java index 0e62345..40789d8 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java @@ -25,12 +25,13 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.io.nativeio.NativeIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; import org.jboss.netty.channel.DefaultFileRegion; @@ -39,7 +40,7 @@ import com.google.common.annotations.VisibleForTesting; public class FadvisedFileRegion extends DefaultFileRegion { - private static final Log LOG = LogFactory.getLog(FadvisedFileRegion.class); + private static final Logger LOG = LoggerFactory.getLogger(FadvisedFileRegion.class); private final boolean manageOsCache; private final int readaheadLength; http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java index 4cc6c4e..af50cbf 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java @@ -55,8 +55,6 @@ import java.util.regex.Pattern; import javax.crypto.SecretKey; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; @@ -129,6 +127,7 @@ import org.jboss.netty.handler.codec.http.QueryStringDecoder; import org.jboss.netty.handler.ssl.SslHandler; import org.jboss.netty.handler.stream.ChunkedWriteHandler; import org.jboss.netty.util.CharsetUtil; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; @@ -143,9 +142,9 @@ import com.google.protobuf.ByteString; public class ShuffleHandler extends AuxiliaryService { - private static final Log LOG = LogFactory.getLog(ShuffleHandler.class); - private static final Log AUDITLOG = - LogFactory.getLog(ShuffleHandler.class.getName()+".audit"); + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(ShuffleHandler.class); + private static final org.slf4j.Logger AUDITLOG = + LoggerFactory.getLogger(ShuffleHandler.class.getName()+".audit"); public static final String SHUFFLE_MANAGE_OS_CACHE = "tez.shuffle.manage.os.cache"; public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true; @@ -738,7 +737,7 @@ public class ShuffleHandler extends AuxiliaryService { } private static class LevelDBLogger implements Logger { - private static final Log LOG = LogFactory.getLog(LevelDBLogger.class); + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(LevelDBLogger.class); @Override public void log(String message) { http://git-wip-us.apache.org/repos/asf/tez/blob/397d6af3/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java index 2ce306d..c2bf361 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java @@ -46,8 +46,6 @@ import java.util.Map; import java.util.zip.CheckedOutputStream; import java.util.zip.Checksum; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataOutputStream; @@ -95,10 +93,12 @@ import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TestShuffleHandler { static final long MiB = 1024 * 1024; - private static final Log LOG = LogFactory.getLog(TestShuffleHandler.class); + private static final Logger LOG = LoggerFactory.getLogger(TestShuffleHandler.class); class MockShuffleHandler extends org.apache.tez.auxservices.ShuffleHandler { @Override
