http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java ---------------------------------------------------------------------- diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java index b205a42..fcb42b3 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java @@ -26,8 +26,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; @@ -86,7 +86,7 @@ import com.google.common.collect.HashMultimap; @SuppressWarnings("deprecation") public class TestReduceProcessor { - private static final Log LOG = LogFactory.getLog(TestReduceProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(TestReduceProcessor.class); private static JobConf defaultConf = new JobConf(); private static FileSystem localFs = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml index 90c425e..6ca51b4 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml @@ -116,8 +116,8 @@ <artifactId>guava</artifactId> </dependency> <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> </dependency> <dependency> <groupId>org.mockito</groupId> http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java index d1138b0..4b4487b 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java @@ -24,8 +24,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -45,7 +45,7 @@ import com.google.common.annotations.VisibleForTesting; public class ATSHistoryACLPolicyManager implements HistoryACLPolicyManager { - private final static Log LOG = LogFactory.getLog(ATSHistoryACLPolicyManager.class); + private final static Logger LOG = LoggerFactory.getLogger(ATSHistoryACLPolicyManager.class); TimelineClient timelineClient; Configuration conf; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java index 1e86513..30c6dda 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java @@ -30,8 +30,8 @@ import java.util.Random; import javax.ws.rs.core.MediaType; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -66,7 +66,7 @@ import com.sun.jersey.api.client.WebResource; public class TestATSHistoryWithACLs { - private static final Log LOG = LogFactory.getLog(TestATSHistoryWithACLs.class); + private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryWithACLs.class); protected static MiniTezClusterWithTimeline mrrTezCluster = null; protected static MiniDFSCluster dfsCluster = null; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history/pom.xml ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml index 3e41f1d..2978e88 100644 --- a/tez-plugins/tez-yarn-timeline-history/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history/pom.xml @@ -106,8 +106,8 @@ <artifactId>guava</artifactId> </dependency> <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> </dependency> <dependency> <groupId>org.mockito</groupId> http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java index c68d395..11f38b9 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java @@ -27,8 +27,8 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; @@ -49,7 +49,7 @@ import com.google.common.annotations.VisibleForTesting; public class ATSHistoryLoggingService extends HistoryLoggingService { - private static final Log LOG = LogFactory.getLog(ATSHistoryLoggingService.class); + private static final Logger LOG = LoggerFactory.getLogger(ATSHistoryLoggingService.class); private LinkedBlockingQueue<DAGHistoryEvent> eventQueue = new LinkedBlockingQueue<DAGHistoryEvent>(); http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java index 18ec43e..7d1abbd 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java @@ -18,8 +18,8 @@ package org.apache.tez.dag.history.logging.ats; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; @@ -43,7 +43,7 @@ import static org.mockito.Mockito.when; public class TestATSHistoryLoggingService { - private static final Log LOG = LogFactory.getLog(TestATSHistoryLoggingService.class); + private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryLoggingService.class); private ATSHistoryLoggingService atsHistoryLoggingService; private AppContext appContext; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java index 45bbcc7..448de71 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java @@ -23,8 +23,8 @@ import java.util.Random; import javax.ws.rs.core.MediaType; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -55,7 +55,7 @@ import com.sun.jersey.api.client.WebResource; public class TestATSHistoryWithMiniCluster { - private static final Log LOG = LogFactory.getLog(TestATSHistoryWithMiniCluster.class); + private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryWithMiniCluster.class); protected static MiniTezClusterWithTimeline mrrTezCluster = null; protected static MiniDFSCluster dfsCluster = null; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java ---------------------------------------------------------------------- diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java index b6e39fa..d13ebdb 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java @@ -28,8 +28,8 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileContext; @@ -66,7 +66,7 @@ public class MiniTezClusterWithTimeline extends MiniYARNCluster { public static final String APPJAR = JarFinder.getJar(DAGAppMaster.class); - private static final Log LOG = LogFactory.getLog(MiniTezClusterWithTimeline.class); + private static final Logger LOG = LoggerFactory.getLogger(MiniTezClusterWithTimeline.class); private static final String YARN_CLUSTER_CONFIG = "yarn-site.xml"; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/pom.xml ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml index 2508b98..ad34c2d 100644 --- a/tez-runtime-internals/pom.xml +++ b/tez-runtime-internals/pom.xml @@ -71,8 +71,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java index a0b48f0..2d27c8c 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java @@ -38,8 +38,8 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.CallableWithNdc; @@ -92,8 +92,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; @Private public class LogicalIOProcessorRuntimeTask extends RuntimeTask { - private static final Log LOG = LogFactory - .getLog(LogicalIOProcessorRuntimeTask.class); + private static final Logger LOG = LoggerFactory + .getLogger(LogicalIOProcessorRuntimeTask.class); private final String[] localDirs; /** Responsible for maintaining order of Inputs */ http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java index 0e3f67d..bb6184e 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java @@ -26,8 +26,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.ReflectionUtils; @@ -49,7 +49,7 @@ import com.google.common.collect.Iterables; @Private public class MemoryDistributor { - private static final Log LOG = LogFactory.getLog(MemoryDistributor.class); + private static final Logger LOG = LoggerFactory.getLogger(MemoryDistributor.class); private final int numTotalInputs; private final int numTotalOutputs; @@ -206,7 +206,7 @@ public class MemoryDistributor { private static class RequestorInfo { - private static final Log LOG = LogFactory.getLog(RequestorInfo.class); + private static final Logger LOG = LoggerFactory.getLogger(RequestorInfo.class); private final MemoryUpdateCallback callback; private final InitialMemoryRequestContext requestContext; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java index bd10e07..aebb19b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java @@ -21,8 +21,8 @@ package org.apache.tez.runtime.common.resources; import java.text.DecimalFormat; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -37,7 +37,7 @@ import com.google.common.collect.Lists; @Unstable public class ScalingAllocator implements InitialMemoryAllocator { - private static final Log LOG = LogFactory.getLog(ScalingAllocator.class); + private static final Logger LOG = LoggerFactory.getLogger(ScalingAllocator.class); private Configuration conf; @@ -104,4 +104,4 @@ public class ScalingAllocator implements InitialMemoryAllocator { public Configuration getConf() { return this.conf; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java index d0ad309..6399ded 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java @@ -23,8 +23,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem.Statistics; @@ -42,7 +42,7 @@ import org.apache.tez.dag.api.TezConfiguration; */ public class TaskCounterUpdater { - private static final Log LOG = LogFactory.getLog(TaskCounterUpdater.class); + private static final Logger LOG = LoggerFactory.getLogger(TaskCounterUpdater.class); private final TezCounters tezCounters; private final Configuration conf; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java index c08d95e..06b90d8 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java @@ -21,11 +21,12 @@ package org.apache.tez.runtime.task; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; -import org.apache.log4j.Logger; import org.apache.tez.common.CallableWithNdc; import org.apache.tez.common.ContainerContext; import org.apache.tez.common.ContainerTask; import org.apache.tez.common.TezTaskUmbilicalProtocol; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Responsible for communication between a running Container and the ApplicationMaster. The main @@ -34,7 +35,7 @@ import org.apache.tez.common.TezTaskUmbilicalProtocol; */ public class ContainerReporter extends CallableWithNdc<ContainerTask> { - private static final Logger LOG = Logger.getLogger(ContainerReporter.class); + private static final Logger LOG = LoggerFactory.getLogger(ContainerReporter.class); private final TezTaskUmbilicalProtocol umbilical; private final ContainerContext containerContext; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java index 0841984..235958f 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java @@ -33,7 +33,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.log4j.Logger; import org.apache.tez.common.TezTaskUmbilicalProtocol; import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.TezException; @@ -47,6 +46,8 @@ import org.apache.tez.runtime.api.impl.TezEvent; import org.apache.tez.runtime.api.impl.TezHeartbeatRequest; import org.apache.tez.runtime.api.impl.TezHeartbeatResponse; import org.apache.tez.runtime.api.impl.EventMetaData.EventProducerConsumerType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -65,7 +66,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; */ public class TaskReporter { - private static final Logger LOG = Logger.getLogger(TaskReporter.class); + private static final Logger LOG = LoggerFactory.getLogger(TaskReporter.class); private final TezTaskUmbilicalProtocol umbilical; private final long pollInterval; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java index c031ecb..bd2d025 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java @@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.apache.tez.common.ContainerContext; import org.apache.tez.common.ContainerTask; import org.apache.tez.common.TezCommonUtils; @@ -66,6 +65,8 @@ import org.apache.tez.runtime.api.ExecutionContext; import org.apache.tez.runtime.api.impl.ExecutionContextImpl; import org.apache.tez.runtime.api.impl.TaskSpec; import org.apache.tez.runtime.common.objectregistry.ObjectRegistryImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -79,7 +80,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; public class TezChild { - private static final Logger LOG = Logger.getLogger(TezChild.class); + private static final Logger LOG = LoggerFactory.getLogger(TezChild.class); private final Configuration defaultConf; private final String containerIdString; @@ -156,7 +157,7 @@ public class TezChild { if (LOG.isDebugEnabled()) { LOG.debug("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { - LOG.debug(token); + LOG.debug("",token); } } http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java index e39be37..fd920e4 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner.java @@ -32,7 +32,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSError; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.log4j.Logger; import org.apache.tez.common.CallableWithNdc; import org.apache.tez.common.TezTaskUmbilicalProtocol; import org.apache.tez.dag.api.TezException; @@ -44,6 +43,8 @@ import org.apache.tez.runtime.api.impl.EventMetaData; import org.apache.tez.runtime.api.impl.TaskSpec; import org.apache.tez.runtime.api.impl.TezEvent; import org.apache.tez.runtime.api.impl.TezUmbilical; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.ListenableFuture; @@ -51,7 +52,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; public class TezTaskRunner implements TezUmbilical, ErrorReporter { - private static final Logger LOG = Logger.getLogger(TezTaskRunner.class); + private static final Logger LOG = LoggerFactory.getLogger(TezTaskRunner.class); private final Configuration tezConf; private final LogicalIOProcessorRuntimeTask task; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java ---------------------------------------------------------------------- diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java index 7890f89..91b311f 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java @@ -49,7 +49,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.log4j.Logger; import org.apache.tez.common.ContainerContext; import org.apache.tez.common.ContainerTask; import org.apache.tez.common.TezTaskUmbilicalProtocol; @@ -79,6 +78,8 @@ import org.apache.tez.runtime.common.resources.ScalingAllocator; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.HashMultimap; import com.google.common.util.concurrent.ListenableFuture; @@ -88,7 +89,7 @@ import com.google.common.util.concurrent.MoreExecutors; // Tests in this class cannot be run in parallel. public class TestTaskExecution { - private static final Logger LOG = Logger.getLogger(TestTaskExecution.class); + private static final Logger LOG = LoggerFactory.getLogger(TestTaskExecution.class); private static final String HEARTBEAT_EXCEPTION_STRING = "HeartbeatException"; @@ -379,7 +380,7 @@ public class TestTaskExecution { public static final byte[] CONF_SIGNAL_FATAL_AND_LOOP = new byte[] { 8 }; public static final byte[] CONF_SIGNAL_FATAL_AND_COMPLETE = new byte[] { 16 }; - private static final Logger LOG = Logger.getLogger(TestProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(TestProcessor.class); private static final ReentrantLock processorLock = new ReentrantLock(); private static final Condition processorCondition = processorLock.newCondition(); @@ -528,7 +529,7 @@ public class TestTaskExecution { private static class TezTaskUmbilicalForTest implements TezTaskUmbilicalProtocol { - private static final Logger LOG = Logger.getLogger(TezTaskUmbilicalForTest.class); + private static final Logger LOG = LoggerFactory.getLogger(TezTaskUmbilicalForTest.class); private final List<TezEvent> requestEvents = new LinkedList<TezEvent>(); http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/pom.xml ---------------------------------------------------------------------- diff --git a/tez-runtime-library/pom.xml b/tez-runtime-library/pom.xml index 539ca10..0edf169 100644 --- a/tez-runtime-library/pom.xml +++ b/tez-runtime-library/pom.xml @@ -66,8 +66,8 @@ <artifactId>commons-lang</artifactId> </dependency> <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> </dependency> <dependency> <groupId>commons-codec</groupId> http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java index bbe9dcf..98a5873 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java @@ -24,8 +24,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.tez.dag.api.EdgeProperty; import org.apache.tez.dag.api.EdgeProperty.DataMovementType; @@ -48,8 +48,8 @@ import com.google.common.collect.Multimap; @Private public class InputReadyVertexManager extends VertexManagerPlugin { - private static final Log LOG = - LogFactory.getLog(InputReadyVertexManager.class); + private static final Logger LOG = + LoggerFactory.getLogger(InputReadyVertexManager.class); Map<String, SourceVertexInfo> srcVertexInfo = Maps.newHashMap(); boolean taskIsStarted[]; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java index a993449..f923319 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java @@ -27,8 +27,8 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; @@ -118,8 +118,8 @@ public class ShuffleVertexManager extends VertexManagerPlugin { public static final int TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT = 1; - private static final Log LOG = - LogFactory.getLog(ShuffleVertexManager.class); + private static final Logger LOG = + LoggerFactory.getLogger(ShuffleVertexManager.class); float slowStartMinSrcCompletionFraction; float slowStartMaxSrcCompletionFraction; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/broadcast/output/FileBasedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/broadcast/output/FileBasedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/broadcast/output/FileBasedKVWriter.java index 02922e5..95cea82 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/broadcast/output/FileBasedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/broadcast/output/FileBasedKVWriter.java @@ -20,8 +20,8 @@ package org.apache.tez.runtime.library.broadcast.output; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -48,7 +48,7 @@ import com.google.common.base.Preconditions; @Private public class FileBasedKVWriter extends KeyValuesWriter { - private static final Log LOG = LogFactory.getLog(FileBasedKVWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(FileBasedKVWriter.class); public static final int INDEX_RECORD_LENGTH = 24; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java index cad03ce..ad0eaf9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.tez.dag.api.TezUncheckedException; @@ -38,8 +38,8 @@ import org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFile @Private public class TezRuntimeUtils { - private static final Log LOG = LogFactory - .getLog(TezRuntimeUtils.class); + private static final Logger LOG = LoggerFactory + .getLogger(TezRuntimeUtils.class); public static String getTaskIdentifier(String vertexName, int taskIndex) { return String.format("%s_%06d", vertexName, taskIndex); http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java index c4c38fa..0f1bc5b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataInputBuffer; @@ -45,7 +45,7 @@ import com.google.common.base.Preconditions; @Private public class ValuesIterator<KEY,VALUE> { - private static final Log LOG = LogFactory.getLog(ValuesIterator.class.getName()); + private static final Logger LOG = LoggerFactory.getLogger(ValuesIterator.class.getName()); protected TezRawKeyValueIterator in; //input iterator private KEY key; // current key private KEY nextKey; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java index c2947c2..454d0ed 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java @@ -20,8 +20,8 @@ package org.apache.tez.runtime.library.common.readers; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -43,7 +43,7 @@ import org.apache.tez.runtime.library.common.shuffle.MemoryFetchedInput; @Private public class UnorderedKVReader<K, V> extends KeyValueReader { - private static final Log LOG = LogFactory.getLog(UnorderedKVReader.class); + private static final Logger LOG = LoggerFactory.getLogger(UnorderedKVReader.class); private final ShuffleManager shuffleManager; private final CompressionCodec codec; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java index aabab76..e6da0a4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java @@ -18,8 +18,8 @@ package org.apache.tez.runtime.library.common.serializer; * limitations under the License. */ -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -56,7 +56,7 @@ import java.io.OutputStream; @Unstable public class TezBytesWritableSerialization extends Configured implements Serialization<Writable> { - private static final Log LOG = LogFactory.getLog(TezBytesWritableSerialization.class.getName()); + private static final Logger LOG = LoggerFactory.getLogger(TezBytesWritableSerialization.class.getName()); @Override public boolean accept(Class<?> c) { http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java index f800766..e25301e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalDirAllocator; @@ -35,7 +35,7 @@ import com.google.common.base.Preconditions; public class DiskFetchedInput extends FetchedInput { - private static final Log LOG = LogFactory.getLog(DiskFetchedInput.class); + private static final Logger LOG = LoggerFactory.getLogger(DiskFetchedInput.class); private final FileSystem localFS; private final Path tmpOutputPath; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java index 3661361..3154943 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java @@ -39,8 +39,9 @@ import java.util.concurrent.atomic.AtomicInteger; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; @@ -70,7 +71,7 @@ import com.google.common.base.Preconditions; */ public class Fetcher extends CallableWithNdc<FetchResult> { - private static final Log LOG = LogFactory.getLog(Fetcher.class); + private static final Logger LOG = LoggerFactory.getLogger(Fetcher.class); private static final AtomicInteger fetcherIdGen = new AtomicInteger(0); private final Configuration conf; @@ -646,7 +647,7 @@ public class Fetcher extends CallableWithNdc<FetchResult> { LOG.info("Exception while shutting down fetcher on " + logIdentifier + " : " + e.getMessage()); if (LOG.isDebugEnabled()) { - LOG.debug(e); + LOG.debug(StringUtils.EMPTY, e); } } } http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java index 422cb56..ad6ed19 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HttpConnection.java @@ -30,8 +30,8 @@ import java.util.concurrent.TimeUnit; import javax.net.ssl.HttpsURLConnection; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IOUtils; @@ -47,7 +47,7 @@ import com.google.common.base.Stopwatch; */ public class HttpConnection { - private static final Log LOG = LogFactory.getLog(HttpConnection.class); + private static final Logger LOG = LoggerFactory.getLogger(HttpConnection.class); /** Basic/unit connection timeout (in milliseconds) */ private final static int UNIT_CONNECT_TIMEOUT = 60 * 1000; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java index 0f28136..5c63961 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java @@ -25,8 +25,8 @@ import java.io.OutputStream; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.io.input.BoundedInputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.tez.runtime.library.common.InputAttemptIdentifier; public class LocalDiskFetchedInput extends FetchedInput { - private static final Log LOG = LogFactory.getLog(LocalDiskFetchedInput.class); + private static final Logger LOG = LoggerFactory.getLogger(LocalDiskFetchedInput.class); private final Path inputFile; private final FileSystem localFS; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java index 49e53b0..a9e34f1 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java @@ -18,6 +18,7 @@ package org.apache.tez.runtime.library.common.shuffle; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -32,8 +33,8 @@ import javax.crypto.SecretKey; import com.google.common.base.Preconditions; import com.google.protobuf.ByteString; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputByteBuffer; @@ -61,7 +62,7 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovem public class ShuffleUtils { - private static final Log LOG = LogFactory.getLog(ShuffleUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(ShuffleUtils.class); public static final String SHUFFLE_HANDLER_SERVICE_ID = "mapreduce_shuffle"; static final ThreadLocal<DecimalFormat> MBPS_FORMAT = @@ -102,7 +103,7 @@ public class ShuffleUtils { public static void shuffleToMemory(byte[] shuffleData, InputStream input, int decompressedLength, int compressedLength, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, - Log LOG, String identifier) throws IOException { + Logger LOG, String identifier) throws IOException { try { IFile.Reader.readToMemory(shuffleData, input, compressedLength, codec, ifileReadAhead, ifileReadAheadLength); @@ -111,14 +112,14 @@ public class ShuffleUtils { + identifier); } catch (IOException ioe) { // Close the streams - IOUtils.cleanup(LOG, input); + ioCleanup(input); // Re-throw throw ioe; } } public static void shuffleToDisk(OutputStream output, String hostIdentifier, - InputStream input, long compressedLength, Log LOG, String identifier) + InputStream input, long compressedLength, Logger LOG, String identifier) throws IOException { // Copy data to local-disk long bytesLeft = compressedLength; @@ -142,7 +143,7 @@ public class ShuffleUtils { output.close(); } catch (IOException ioe) { // Close the streams - IOUtils.cleanup(LOG, input, output); + ioCleanup(input, output); // Re-throw throw ioe; } @@ -157,6 +158,19 @@ public class ShuffleUtils { } } + public static void ioCleanup(Closeable... closeables) { + for (Closeable c : closeables) { + if (c == null) + continue; + try { + c.close(); + } catch (IOException e) { + if (LOG.isDebugEnabled()) + LOG.debug("Exception in closing " + c, e); + } + } + } + // TODO NEWTEZ handle ssl shuffle public static StringBuilder constructBaseURIForShuffleHandler(String host, int port, int partition, String appId, boolean sslShuffle) { @@ -392,7 +406,7 @@ public class ShuffleUtils { * @param outputType * @param srcAttemptIdentifier */ - public static void logIndividualFetchComplete(Log log, long millis, long + public static void logIndividualFetchComplete(Logger log, long millis, long bytesCompressed, long bytesDecompressed, String outputType, InputAttemptIdentifier srcAttemptIdentifier) { double rate = 0; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index c4d6ce3..1486848 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -25,8 +25,8 @@ import java.util.List; import com.google.protobuf.ByteString; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; @@ -54,7 +54,7 @@ import com.google.protobuf.InvalidProtocolBufferException; */ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { - private static final Log LOG = LogFactory.getLog(ShuffleInputEventHandlerImpl.class); + private static final Logger LOG = LoggerFactory.getLogger(ShuffleInputEventHandlerImpl.class); private final ShuffleManager shuffleManager; private final FetchedInputAllocator inputAllocator; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java index 3995995..9c53414 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java @@ -46,8 +46,8 @@ import javax.crypto.SecretKey; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalDirAllocator; @@ -97,8 +97,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; // (multiple src-indices), modifications will be required. public class ShuffleManager implements FetcherCallback { - private static final Log LOG = LogFactory.getLog(ShuffleManager.class); - + private static final Logger LOG = LoggerFactory.getLogger(ShuffleManager.class); + private final InputContext inputContext; private final int numInputs; @@ -758,7 +758,7 @@ public class ShuffleManager implements FetcherCallback { } private void reportFatalError(Throwable exception, String message) { - LOG.fatal(message); + LOG.error(message); inputContext.fatalError(exception, message); } http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java index 7a8a95a..31a8651 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java @@ -20,8 +20,8 @@ package org.apache.tez.runtime.library.common.shuffle.impl; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.LocalDirAllocator; @@ -47,7 +47,7 @@ import org.apache.tez.runtime.library.common.shuffle.MemoryFetchedInput; public class SimpleFetchedInputAllocator implements FetchedInputAllocator, FetchedInputCallback { - private static final Log LOG = LogFactory.getLog(SimpleFetchedInputAllocator.class); + private static final Logger LOG = LoggerFactory.getLogger(SimpleFetchedInputAllocator.class); private final Configuration conf; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java index f4fa596..8d20aa7 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java @@ -27,8 +27,8 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; @@ -52,7 +52,7 @@ import com.google.common.annotations.VisibleForTesting; class FetcherOrderedGrouped extends Thread { - private static final Log LOG = LogFactory.getLog(FetcherOrderedGrouped.class); + private static final Logger LOG = LoggerFactory.getLogger(FetcherOrderedGrouped.class); private final Configuration conf; private final boolean localDiskFetchEnabled; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java index 0fd1166..17d57a6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java @@ -20,8 +20,8 @@ package org.apache.tez.runtime.library.common.shuffle.orderedgrouped; import java.io.DataOutputStream; import java.io.IOException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.BoundedByteArrayOutputStream; @@ -33,7 +33,7 @@ import org.apache.tez.runtime.library.common.sort.impl.IFile.Writer; @InterfaceAudience.Private @InterfaceStability.Unstable public class InMemoryWriter extends Writer { - private static final Log LOG = LogFactory.getLog(InMemoryWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(InMemoryWriter.class); // TODO Verify and fix counters if required. http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java index 49baf70..e999af6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java @@ -22,8 +22,8 @@ import java.io.OutputStream; import java.util.Comparator; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -34,7 +34,7 @@ import org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFile class MapOutput { - private static final Log LOG = LogFactory.getLog(MapOutput.class); + private static final Logger LOG = LoggerFactory.getLogger(MapOutput.class); private static AtomicInteger ID = new AtomicInteger(0); public static enum Type { http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java index 54266b4..045b91d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java @@ -28,8 +28,8 @@ import java.util.TreeSet; import com.google.common.annotations.VisibleForTesting; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -73,7 +73,7 @@ import org.apache.tez.runtime.library.hadoop.compat.NullProgressable; @SuppressWarnings(value={"rawtypes"}) public class MergeManager { - private static final Log LOG = LogFactory.getLog(MergeManager.class); + private static final Logger LOG = LoggerFactory.getLogger(MergeManager.class); private final Configuration conf; private final FileSystem localFS; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java index 7e720b4..d4faf51 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java @@ -23,12 +23,12 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; abstract class MergeThread<T> extends Thread { - private static final Log LOG = LogFactory.getLog(MergeThread.class); + private static final Logger LOG = LoggerFactory.getLogger(MergeThread.class); private volatile boolean inProgress = false; private final List<T> inputs = new ArrayList<T>(); http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java index f9f4ec2..f98aa3a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java @@ -28,8 +28,8 @@ import java.util.concurrent.atomic.AtomicReference; import javax.crypto.SecretKey; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -76,7 +76,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; @InterfaceStability.Unstable public class Shuffle implements ExceptionReporter { - private static final Log LOG = LogFactory.getLog(Shuffle.class); + private static final Logger LOG = LoggerFactory.getLogger(Shuffle.class); private static final int PROGRESS_FREQUENCY = 2000; private final Configuration conf; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java index d00b429..32ac766 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java @@ -25,8 +25,8 @@ import java.util.List; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.tez.common.TezCommonUtils; import org.apache.tez.common.TezUtilsInternal; import org.apache.tez.dag.api.TezUncheckedException; @@ -43,7 +43,7 @@ import com.google.protobuf.InvalidProtocolBufferException; public class ShuffleInputEventHandlerOrderedGrouped { - private static final Log LOG = LogFactory.getLog(ShuffleInputEventHandlerOrderedGrouped.class); + private static final Logger LOG = LoggerFactory.getLogger(ShuffleInputEventHandlerOrderedGrouped.class); private final ShuffleScheduler scheduler; private final InputContext inputContext; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java index 5ae10ab..d0b6346 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java @@ -39,8 +39,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IntWritable; import org.apache.tez.common.TezUtilsInternal; @@ -65,7 +65,7 @@ class ShuffleScheduler { } }; - private static final Log LOG = LogFactory.getLog(ShuffleScheduler.class); + private static final Logger LOG = LoggerFactory.getLogger(ShuffleScheduler.class); private static final long INITIAL_PENALTY = 2000l; // 2 seconds private static final float PENALTY_GROWTH_RATE = 1.3f; @@ -355,7 +355,7 @@ class ShuffleScheduler { @VisibleForTesting void reportExceptionForInput(Exception exception) { - LOG.fatal(exception); + LOG.error("Reporting exception for input", exception); shuffle.reportException(exception); } @@ -492,7 +492,7 @@ class ShuffleScheduler { failureCounts.size() == (numInputs - doneMaps)) && !reducerHealthy && (!reducerProgressedEnough || reducerStalled)) { - LOG.fatal("Shuffle failed with too many fetch failures " + "and insufficient progress!" + LOG.error("Shuffle failed with too many fetch failures " + "and insufficient progress!" + "failureCounts=" + failureCounts.size() + ", pendingInputs=" + (numInputs - doneMaps) + ", reducerHealthy=" + reducerHealthy + ", reducerProgressedEnough=" + reducerProgressedEnough + ", reducerStalled=" + reducerStalled); http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java index 4d33a43..c0445c9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java @@ -25,8 +25,8 @@ import java.util.Iterator; import java.util.Map; import com.google.common.collect.Maps; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -63,7 +63,7 @@ import com.google.common.base.Preconditions; @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class ExternalSorter { - private static final Log LOG = LogFactory.getLog(ExternalSorter.class); + private static final Logger LOG = LoggerFactory.getLogger(ExternalSorter.class); public void close() throws IOException { spillFileIndexPaths.clear(); http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java index 049bcef..8dcbf6d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java @@ -27,8 +27,8 @@ import java.util.Iterator; import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -59,7 +59,7 @@ import org.apache.tez.common.counters.TezCounter; @InterfaceAudience.Private @InterfaceStability.Unstable public class IFile { - private static final Log LOG = LogFactory.getLog(IFile.class); + private static final Logger LOG = LoggerFactory.getLogger(IFile.class); public static final int EOF_MARKER = -1; // End of File Marker public static final int RLE_MARKER = -2; // Repeat same key marker public static final int V_END_MARKER = -3; // End of values marker @@ -618,7 +618,15 @@ public class IFile { try { IOUtils.readFully(in, buffer, 0, buffer.length - IFile.HEADER.length); } catch (IOException ioe) { - IOUtils.cleanup(LOG, in); + if(in != null) { + try { + in.close(); + } catch(IOException e) { + if(LOG.isDebugEnabled()) { + LOG.debug("Exception in closing " + in, e); + } + } + } throw ioe; } finally { if (decompressor != null) { http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java index e346793..64c1b51 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java @@ -25,8 +25,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ChecksumException; @@ -60,7 +60,7 @@ public class IFileInputStream extends InputStream { private final boolean readahead; private final int readaheadLength; - public static final Log LOG = LogFactory.getLog(IFileInputStream.class); + public static final Logger LOG = LoggerFactory.getLogger(IFileInputStream.class); private boolean disableChecksumValidation = false; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java index 34a7e3b..0b959cd 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java @@ -38,8 +38,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; @@ -64,7 +64,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; @SuppressWarnings({"unchecked", "rawtypes"}) public class PipelinedSorter extends ExternalSorter { - private static final Log LOG = LogFactory.getLog(PipelinedSorter.class); + private static final Logger LOG = LoggerFactory.getLogger(PipelinedSorter.class); /** * The size of each record in the index file for the map-outputs. http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java index 5dd538a..758e9c7 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java @@ -23,8 +23,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -57,7 +57,7 @@ import org.apache.tez.runtime.library.utils.BufferUtils; @InterfaceStability.Unstable @SuppressWarnings({"unchecked", "rawtypes"}) public class TezMerger { - private static final Log LOG = LogFactory.getLog(TezMerger.class); + private static final Logger LOG = LoggerFactory.getLogger(TezMerger.class); // Local directories http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java index a56249d..a6db0c6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java @@ -31,8 +31,8 @@ import java.util.concurrent.locks.ReentrantLock; import com.google.common.collect.Lists; import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; @@ -60,7 +60,7 @@ import com.google.common.base.Preconditions; @SuppressWarnings({"unchecked", "rawtypes"}) public class DefaultSorter extends ExternalSorter implements IndexedSortable { - private static final Log LOG = LogFactory.getLog(DefaultSorter.class); + private static final Logger LOG = LoggerFactory.getLogger(DefaultSorter.class); // TODO NEWTEZ Progress reporting to Tez framework. (making progress vs %complete) http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java index aafe1db..6382e3a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java @@ -21,8 +21,8 @@ package org.apache.tez.runtime.library.common.task.local.output; import java.io.IOException; import com.google.common.base.Preconditions; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -45,7 +45,7 @@ public class TezTaskOutputFiles extends TezTaskOutput { super(conf, uniqueId); } - private static final Log LOG = LogFactory.getLog(TezTaskOutputFiles.class); + private static final Logger LOG = LoggerFactory.getLogger(TezTaskOutputFiles.class); private static final String SPILL_FILE_DIR_PATTERN = "%s_%d"; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java index d1d575d..30d1adb 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.DefaultCodec; @@ -44,7 +44,7 @@ import org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput; @SuppressWarnings("rawtypes") public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { - private static final Log LOG = LogFactory.getLog(BaseUnorderedPartitionedKVWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(BaseUnorderedPartitionedKVWriter.class); protected final OutputContext outputContext; protected final Configuration conf; http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index 8b5f196..19f6130 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -37,8 +37,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -64,6 +62,8 @@ import org.apache.tez.runtime.library.common.sort.impl.IFile.Writer; import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord; import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -77,7 +77,7 @@ import com.google.protobuf.ByteString; public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWriter { - private static final Log LOG = LogFactory.getLog(UnorderedPartitionedKVWriter.class); + private static final Logger LOG = LoggerFactory.getLogger(UnorderedPartitionedKVWriter.class); private static final int INT_SIZE = 4; private static final int NUM_META = 3; // Number of meta fields. @@ -434,7 +434,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit spillLock.unlock(); } if (spillException != null) { - LOG.fatal("Error during spill, throwing"); + LOG.error("Error during spill, throwing"); // Assuming close will be called on the same thread as the write cleanup(); currentBuffer.cleanup(); @@ -849,7 +849,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit LOG.info("Adding spill event for spill (final update=" + isFinalUpdate + "), spillId=" + spillNumber); outputContext.sendEvents(Collections.singletonList(compEvent)); } catch (IOException e) { - LOG.fatal("Error in sending pipelined events", e); + LOG.error("Error in sending pipelined events", e); outputContext.fatalError(e, "Error in sending pipelined events"); } } @@ -879,7 +879,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit availableBuffers.add(result.wrappedBuffer); } catch (Throwable e) { - LOG.fatal("Failure while attempting to reset buffer after spill", e); + LOG.error("Failure while attempting to reset buffer after spill", e); outputContext.fatalError(e, "Failure while attempting to reset buffer after spill"); } @@ -904,7 +904,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit public void onFailure(Throwable t) { // spillException setup to throw an exception back to the user. Requires synchronization. // Consider removing it in favor of having Tez kill the task - LOG.fatal("Failure while spilling to disk", t); + LOG.error("Failure while spilling to disk", t); spillException = t; outputContext.fatalError(t, "Failure while spilling to disk"); spillLock.lock(); @@ -961,4 +961,4 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit this.spillIndex = spillIndex; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/tez/blob/6d0b10a8/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java ---------------------------------------------------------------------- diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java index 9031eb0..a807bad 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java @@ -27,8 +27,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.conf.Configuration; @@ -70,7 +70,7 @@ import com.google.common.base.Preconditions; @Public public class OrderedGroupedKVInput extends AbstractLogicalInput { - static final Log LOG = LogFactory.getLog(OrderedGroupedKVInput.class); + static final Logger LOG = LoggerFactory.getLogger(OrderedGroupedKVInput.class); protected TezRawKeyValueIterator rawIter = null; protected Configuration conf;
