Updated Branches: refs/heads/sqoop2 f73c0282a -> 1c87cb762
SQOOP-762 Improve logging in mapreduce execution engine (Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/1c87cb76 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/1c87cb76 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/1c87cb76 Branch: refs/heads/sqoop2 Commit: 1c87cb7629934ca3861b804c91cb1e23292f9d8e Parents: f73c028 Author: Bilung Lee <[email protected]> Authored: Thu Dec 13 17:33:29 2012 -0800 Committer: Bilung Lee <[email protected]> Committed: Thu Dec 13 17:33:29 2012 -0800 ---------------------------------------------------------------------- .../apache/sqoop/job/etl/HdfsExportPartition.java | 16 ++++++++-- .../sqoop/job/etl/HdfsSequenceExportExtractor.java | 14 +++++++- .../sqoop/job/etl/HdfsTextExportExtractor.java | 14 +++++++- .../sqoop/job/mr/SqoopDestroyerExecutor.java | 1 + .../java/org/apache/sqoop/job/mr/SqoopMapper.java | 2 + .../job/mr/SqoopOutputFormatLoadExecutor.java | 4 ++ .../mapreduce/src/test/resources/log4j.properties | 24 +++++++++++++++ 7 files changed, 68 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportPartition.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportPartition.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportPartition.java index 0e0e53f..cdbdaa8 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportPartition.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsExportPartition.java @@ -139,9 +139,19 @@ public class HdfsExportPartition extends Partition { public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); - sb.append(files[0]); - for(int i = 1; i < files.length; i++) { - sb.append(", " + files[i]); + boolean first = true; + for(int i = 0; i < files.length; i++) { + if(first) { + first = false; + } else { + sb.append(", "); + } + + sb.append(files[i]); + sb.append(" (offset=").append(offsets[i]); + sb.append(", end=").append(offsets[i] + lengths[i]); + sb.append(", length=").append(lengths[i]); + sb.append(")"); } sb.append("}"); return sb.toString(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java index 1f6714d..0693a09 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java @@ -19,6 +19,8 @@ package org.apache.sqoop.job.etl; import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; @@ -32,6 +34,9 @@ import org.apache.sqoop.job.io.DataWriter; public class HdfsSequenceExportExtractor extends Extractor { + public static final Log LOG = + LogFactory.getLog(HdfsSequenceExportExtractor.class.getName()); + private Configuration conf; private DataWriter datawriter; @@ -51,6 +56,7 @@ public class HdfsSequenceExportExtractor extends Extractor { try { HdfsExportPartition p = (HdfsExportPartition)partition; + LOG.info("Working on partition: " + p); int numFiles = p.getNumberOfFiles(); for (int i=0; i<numFiles; i++) { extractFile(p.getFile(i), p.getOffset(i), p.getLength(i)); @@ -60,9 +66,13 @@ public class HdfsSequenceExportExtractor extends Extractor { } } - private void extractFile(Path file, long offset, long length) + private void extractFile(Path file, long start, long length) throws IOException { - long start = offset; + long end = start + length; + LOG.info("Extracting file " + file); + LOG.info("\t from offset " + start); + LOG.info("\t to offset " + end); + LOG.info("\t of length " + length); SequenceFile.Reader filereader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file)); http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java index 7f1b144..c412c81 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java @@ -20,6 +20,8 @@ package org.apache.sqoop.job.etl; import java.io.IOException; import java.nio.charset.Charset; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -40,6 +42,9 @@ import org.apache.sqoop.job.io.DataWriter; public class HdfsTextExportExtractor extends Extractor { + public static final Log LOG = + LogFactory.getLog(HdfsTextExportExtractor.class.getName()); + private Configuration conf; private DataWriter datawriter; @@ -59,6 +64,7 @@ public class HdfsTextExportExtractor extends Extractor { try { HdfsExportPartition p = (HdfsExportPartition)partition; + LOG.info("Working on partition: " + p); int numFiles = p.getNumberOfFiles(); for (int i=0; i<numFiles; i++) { extractFile(p.getFile(i), p.getOffset(i), p.getLength(i)); @@ -68,10 +74,13 @@ public class HdfsTextExportExtractor extends Extractor { } } - private void extractFile(Path file, long offset, long length) + private void extractFile(Path file, long start, long length) throws IOException { - long start = offset; long end = start + length; + LOG.info("Extracting file " + file); + LOG.info("\t from offset " + start); + LOG.info("\t to offset " + end); + LOG.info("\t of length " + length); FileSystem fs = file.getFileSystem(conf); FSDataInputStream filestream = fs.open(file); @@ -129,6 +138,7 @@ public class HdfsTextExportExtractor extends Extractor { datawriter.writeCsvRecord(line.toString()); } + LOG.info("Extracting ended on position: " + fileseeker.getPos()); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java index 36eb65d..f4ef95a 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopDestroyerExecutor.java @@ -54,6 +54,7 @@ public class SqoopDestroyerExecutor { Object configConnection = ConfigurationUtils.getConnectorConnection(configuration); Object configJob = ConfigurationUtils.getConnectorJob(configuration); + LOG.info("Executing destroyer class " + destroyer.getClass()); destroyer.destroy(success, subContext, configConnection, configJob); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java index fcedf52..ce847f4 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java @@ -74,8 +74,10 @@ public class SqoopMapper SqoopSplit split = context.getCurrentKey(); try { + LOG.info("Running extractor class " + extractorName); extractor.run(subContext, configConnection, configJob, split.getPartition(), new MapDataWriter(context)); + LOG.info("Extractor has finished"); context.getCounter(SqoopCounters.ROWS_READ) .increment(extractor.getRowsRead()); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java index 976d80b..38e2292 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopOutputFormatLoadExecutor.java @@ -85,10 +85,12 @@ public class SqoopOutputFormatLoadExecutor { @Override public void close(TaskAttemptContext context) throws InterruptedException, IOException { + LOG.info("SqoopOutputFormatLoadExecutor::SqoopRecordWriter is about to be closed"); free.acquire(); writerFinished = true; filled.release(); waitForConsumer(); + LOG.info("SqoopOutputFormatLoadExecutor::SqoopRecordWriter is closed"); } } @@ -194,7 +196,9 @@ public class SqoopOutputFormatLoadExecutor { } try { + LOG.info("Running loader class " + loaderName); loader.load(subContext, configConnection, configJob, reader); + LOG.info("Loader has finished"); } catch (Throwable t) { readerFinished = true; LOG.error("Error while loading data out of MR job.", t); http://git-wip-us.apache.org/repos/asf/sqoop/blob/1c87cb76/execution/mapreduce/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/test/resources/log4j.properties b/execution/mapreduce/src/test/resources/log4j.properties new file mode 100644 index 0000000..44ffced --- /dev/null +++ b/execution/mapreduce/src/test/resources/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=DEBUG, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
