TEZ-3702. Tez shuffle jar includes service loader entry for ClientProtocolProvider but not the corresponding class (jlowe)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/23b43386 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/23b43386 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/23b43386 Branch: refs/heads/master Commit: 23b43386b8793df5c3b266d5969a2c64b9e3fc79 Parents: d283063 Author: Jason Lowe <[email protected]> Authored: Thu Apr 27 13:49:16 2017 -0500 Committer: Jason Lowe <[email protected]> Committed: Thu Apr 27 13:49:16 2017 -0500 ---------------------------------------------------------------------- tez-plugins/tez-aux-services/pom.xml | 4 ---- .../main/java/org/apache/tez/auxservices/ShuffleHandler.java | 8 +++++--- .../org/apache/tez/auxservices/TestShuffleHandlerJobs.java | 2 -- 3 files changed, 5 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/23b43386/tez-plugins/tez-aux-services/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/pom.xml b/tez-plugins/tez-aux-services/pom.xml index 6a5be45..636c2cb 100644 --- a/tez-plugins/tez-aux-services/pom.xml +++ b/tez-plugins/tez-aux-services/pom.xml @@ -66,10 +66,6 @@ <artifactId>tez-api</artifactId> </dependency> <dependency> - <groupId>org.apache.tez</groupId> - <artifactId>tez-mapreduce</artifactId> - </dependency> - <dependency> <!-- Required for the ShuffleHandler --> <groupId>org.apache.tez</groupId> <artifactId>tez-runtime-library</artifactId> http://git-wip-us.apache.org/repos/asf/tez/blob/23b43386/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java index 5da144f..cb68aa4 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java @@ -67,7 +67,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapred.proto.ShuffleHandlerRecoveryProtos.JobShuffleInfoProto; import org.apache.hadoop.mapreduce.JobID; -import org.apache.tez.mapreduce.hadoop.MRConfig; import org.apache.tez.common.security.JobTokenIdentifier; import org.apache.tez.common.security.JobTokenSecretManager; import org.apache.tez.runtime.library.common.Constants; @@ -217,6 +216,9 @@ public class ShuffleHandler extends AuxiliaryService { public static final String CONNECTION_CLOSE = "close"; + public static final String SHUFFLE_SSL_ENABLED_KEY = "tez.shuffle.ssl.enabled"; + public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false; + public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY = "tez.shuffle.ssl.file.buffer.size"; @@ -770,8 +772,8 @@ public class ShuffleHandler extends AuxiliaryService { public HttpPipelineFactory(Configuration conf) throws Exception { SHUFFLE = getShuffle(conf); - if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, - MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) { + if (conf.getBoolean(SHUFFLE_SSL_ENABLED_KEY, + SHUFFLE_SSL_ENABLED_DEFAULT)) { LOG.info("Encrypted shuffle is enabled."); sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); sslFactory.init(); http://git-wip-us.apache.org/repos/asf/tez/blob/23b43386/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java index f719c13..73c4c13 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java @@ -39,7 +39,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.tez.mapreduce.hadoop.MRJobConfig; import org.apache.tez.test.MiniTezCluster; import org.junit.AfterClass; import org.junit.Assert; @@ -86,7 +85,6 @@ public class TestShuffleHandlerJobs { conf.set(serviceStr, ShuffleHandler.class.getName()); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS - conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir"); conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l); tezCluster.init(conf); tezCluster.start();
