Author: blee
Date: Mon Jun 18 21:49:42 2012
New Revision: 1351503
URL: http://svn.apache.org/viewvc?rev=1351503&view=rev
Log:
SQOOP-436 Enable verbose logging for MapReduce jobs
Added:
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java (with
props)
Modified:
sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java
sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java
Modified: sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java Mon Jun 18 21:49:42
2012
@@ -38,6 +38,7 @@ import com.cloudera.sqoop.lib.LargeObjec
import com.cloudera.sqoop.tool.SqoopTool;
import com.cloudera.sqoop.util.RandomHash;
import com.cloudera.sqoop.util.StoredAsProperty;
+import org.apache.sqoop.util.LoggingUtils;
/**
* Configurable state used by Sqoop tools.
@@ -85,6 +86,7 @@ public class SqoopOptions implements Clo
// arguments in the appropriate tools. The names of all command-line args
// are stored as constants in BaseSqoopTool.
+ @StoredAsProperty("verbose") private boolean verbose;
@StoredAsProperty("db.connect.string") private String connectString;
@StoredAsProperty("db.table") private String tableName;
private String [] columns; // Array stored as db.column.list.
@@ -560,6 +562,11 @@ public class SqoopOptions implements Clo
// Delimiters were previously memoized; don't let the tool override
// them with defaults.
this.areDelimsManuallySet = true;
+
+ // If we loaded true verbose flag, we need to apply it
+ if (this.verbose) {
+ LoggingUtils.setDebugLevel();
+ }
}
/**
@@ -806,6 +813,9 @@ public class SqoopOptions implements Clo
// Creating instances for user specific mapping
this.mapColumnHive = new Properties();
this.mapColumnJava = new Properties();
+
+ // We do not want to be verbose too much if not explicitly needed
+ this.verbose = false;
}
/**
@@ -893,6 +903,14 @@ public class SqoopOptions implements Clo
}
}
+ public boolean getVerbose() {
+ return verbose;
+ }
+
+ public void setVerbose(boolean beVerbose) {
+ this.verbose = beVerbose;
+ }
+
/**
* Get the temporary directory; guaranteed to end in File.separator
* (e.g., '/').
Modified:
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java Mon
Jun 18 21:49:42 2012
@@ -22,13 +22,12 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Mapper;
/**
* Identity mapper that continuously reports progress via a background thread.
*/
public class AutoProgressMapper<KEYIN, VALIN, KEYOUT, VALOUT>
- extends Mapper<KEYIN, VALIN, KEYOUT, VALOUT> {
+ extends SqoopMapper<KEYIN, VALIN, KEYOUT, VALOUT> {
public static final Log LOG = LogFactory.getLog(
AutoProgressMapper.class.getName());
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java Mon Jun
18 21:49:42 2012
@@ -351,6 +351,7 @@ public class ExportJobBase extends JobBa
// Set the external jar to use for the job.
job.getConfiguration().set("mapred.jar", ormJarFile);
+ propagateOptionsToJob(job);
configureInputFormat(job, tableName, tableClassName, null);
configureOutputFormat(job, tableName, tableClassName);
configureMapper(job, tableName, tableClassName);
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java Mon Jun
18 21:49:42 2012
@@ -190,6 +190,7 @@ public class ImportJobBase extends JobBa
// Set the external jar to use for the job.
job.getConfiguration().set("mapred.jar", ormJarFile);
+ propagateOptionsToJob(job);
configureInputFormat(job, tableName, tableClassName, splitByCol);
configureOutputFormat(job, tableName, tableClassName);
configureMapper(job, tableName, tableClassName);
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java Mon Jun 18
21:49:42 2012
@@ -57,6 +57,8 @@ public class JobBase {
private ClassLoader prevClassLoader = null;
+ public static final String PROPERTY_VERBOSE = "sqoop.verbose";
+
public JobBase() {
this(null);
}
@@ -322,4 +324,18 @@ public class JobBase {
log.info("A jobtracker restart is required for these settings");
log.info("to take effect.");
}
+
+ /**
+ * Save interesting options to constructed job. Goal here is to propagate
some
+ * of them to the job itself, so that they can be easily accessed. We're
+ * propagating only interesting global options (like verbose flag).
+ *
+ * @param job Destination job to save options
+ */
+ protected void propagateOptionsToJob(Job job) {
+ Configuration configuration = job.getConfiguration();
+
+ // So far, propagate only verbose flag
+ configuration.setBoolean(PROPERTY_VERBOSE, options.getVerbose());
+ }
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java Mon Jun 18
21:49:42 2012
@@ -102,6 +102,8 @@ public class MergeJob extends JobBase {
oldPath = oldPath.makeQualified(fs);
newPath = newPath.makeQualified(fs);
+ propagateOptionsToJob(job);
+
FileInputFormat.addInputPath(job, oldPath);
FileInputFormat.addInputPath(job, newPath);
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java Mon
Jun 18 21:49:42 2012
@@ -30,7 +30,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Mapper;
import org.apache.sqoop.util.AsyncSink;
import org.apache.sqoop.util.JdbcUrl;
import org.apache.sqoop.util.PerfCounters;
@@ -46,7 +45,7 @@ import com.cloudera.sqoop.util.LoggingAs
* Mapper that opens up a pipe to mysqldump and pulls data directly.
*/
public class MySQLDumpMapper
- extends Mapper<String, NullWritable, String, NullWritable> {
+ extends SqoopMapper<String, NullWritable, String, NullWritable> {
public static final Log LOG = LogFactory.getLog(
MySQLDumpMapper.class.getName());
@@ -496,7 +495,9 @@ public class MySQLDumpMapper
// CHECKSTYLE:ON
@Override
- protected void setup(Context context) {
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ super.setup(context);
this.conf = context.getConfiguration();
}
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java Mon
Jun 18 21:49:42 2012
@@ -28,7 +28,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Mapper;
import org.apache.sqoop.util.AsyncSink;
import org.apache.sqoop.util.JdbcUrl;
import org.apache.sqoop.util.LoggingAsyncSink;
@@ -48,7 +47,7 @@ import com.cloudera.sqoop.manager.MySQLU
* used to interface with mysqlimport.
*/
public class MySQLExportMapper<KEYIN, VALIN>
- extends Mapper<KEYIN, VALIN, NullWritable, NullWritable> {
+ extends SqoopMapper<KEYIN, VALIN, NullWritable, NullWritable> {
public static final Log LOG = LogFactory.getLog(
MySQLExportMapper.class.getName());
Modified:
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java
Mon Jun 18 21:49:42 2012
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import com.cloudera.sqoop.manager.MySQLUtils;
-import com.cloudera.sqoop.mapreduce.MySQLExportMapper;;
+import com.cloudera.sqoop.mapreduce.MySQLExportMapper;
/**
* mysqlimport-based exporter which accepts lines of text from files
Added: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java?rev=1351503&view=auto
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java (added)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java Mon Jun 18
21:49:42 2012
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.sqoop.util.LoggingUtils;
+
+import java.io.IOException;
+
+/**
+ * Base sqoop mapper class that is convenient place for common functionality.
+ * Other specific mappers are highly encouraged to inherit from this class.
+ */
+public abstract class SqoopMapper<KI, VI, KO, VO>
+ extends Mapper<KI, VI, KO, VO> {
+
+ @Override
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ super.setup(context);
+
+ Configuration configuration = context.getConfiguration();
+
+ // Propagate verbose flag if needed
+ if (configuration.getBoolean(JobBase.PROPERTY_VERBOSE, false)) {
+ LoggingUtils.setDebugLevel();
+ }
+ }
+}
Propchange: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java Mon Jun 18
21:49:42 2012
@@ -33,8 +33,6 @@ import org.apache.commons.cli.OptionGrou
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
import com.cloudera.sqoop.ConnFactory;
import com.cloudera.sqoop.Sqoop;
@@ -45,6 +43,7 @@ import com.cloudera.sqoop.cli.ToolOption
import com.cloudera.sqoop.lib.DelimiterSet;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.metastore.JobData;
+import org.apache.sqoop.util.LoggingUtils;
/**
* Layer on top of SqoopTool that provides some basic common code
@@ -630,8 +629,8 @@ public abstract class BaseSqoopTool exte
// common options.
if (in.hasOption(VERBOSE_ARG)) {
// Immediately switch into DEBUG logging.
- Logger.getLogger("org.apache.sqoop").setLevel(Level.DEBUG);
- Logger.getLogger("com.cloudera.apache").setLevel(Level.DEBUG);
+ out.setVerbose(true);
+ LoggingUtils.setDebugLevel();
LOG.debug("Enabled debug logging.");
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java Mon Jun 18
21:49:42 2012
@@ -21,6 +21,8 @@ package org.apache.sqoop.util;
import java.sql.SQLException;
import org.apache.commons.logging.Log;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
/**
* A helper class for logging.
@@ -44,5 +46,9 @@ public final class LoggingUtils {
}
}
+ public static void setDebugLevel() {
+ Logger.getLogger("org.apache.sqoop").setLevel(Level.DEBUG);
+ Logger.getLogger("com.cloudera.apache").setLevel(Level.DEBUG);
+ }
}
