Updated Branches: refs/heads/trunk c499f4909 -> ebeb93351
SQOOP-650 Add ability to name mapreduce job generated by Sqoop (Jarek Jarcec Cecho via Cheolsoo Park) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ebeb9335 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ebeb9335 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ebeb9335 Branch: refs/heads/trunk Commit: ebeb93351335cc73d6d77ef1fd0d7316655defa3 Parents: c499f49 Author: Cheolsoo Park <[email protected]> Authored: Tue Nov 6 16:28:57 2012 -0800 Committer: Cheolsoo Park <[email protected]> Committed: Tue Nov 6 16:28:57 2012 -0800 ---------------------------------------------------------------------- src/java/org/apache/sqoop/SqoopOptions.java | 13 ++++++++++++- .../org/apache/sqoop/mapreduce/ExportJobBase.java | 3 +++ .../org/apache/sqoop/mapreduce/ImportJobBase.java | 3 +++ src/java/org/apache/sqoop/tool/BaseSqoopTool.java | 1 + src/java/org/apache/sqoop/tool/ExportTool.java | 8 ++++++++ src/java/org/apache/sqoop/tool/ImportTool.java | 8 ++++++++ 6 files changed, 35 insertions(+), 1 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/SqoopOptions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java index 92c9996..2b5db29 100644 --- a/src/java/org/apache/sqoop/SqoopOptions.java +++ b/src/java/org/apache/sqoop/SqoopOptions.java @@ -87,6 +87,9 @@ public class SqoopOptions implements Cloneable { // are stored as constants in BaseSqoopTool. @StoredAsProperty("verbose") private boolean verbose; + + @StoredAsProperty("mapreduce.job.name") private String mapreduceJobName; + @StoredAsProperty("db.connect.string") private String connectString; @StoredAsProperty("db.table") private String tableName; private String [] columns; // Array stored as db.column.list. @@ -1056,7 +1059,7 @@ public class SqoopOptions implements Cloneable { LOG.error("prompt the user to enter the password while being executed"); LOG.error("as Oozie tasks. Please enable sqoop.metastore.client.record"); LOG.error(".password in sqoop-site.xml, or provide the password"); - LOG.error("explictly using --password in the command tag of the Oozie"); + LOG.error("explicitly using --password in the command tag of the Oozie"); LOG.error("workflow file."); return null; } @@ -1070,6 +1073,14 @@ public class SqoopOptions implements Cloneable { this.password = securePasswordEntry(); } + public String getMapreduceJobName() { + return mapreduceJobName; + } + + public void setMapreduceJobName(String mapredJobName) { + this.mapreduceJobName = mapredJobName; + } + public void setPassword(String pass) { this.password = pass; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java index b9eaa83..d311ec1 100644 --- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java @@ -357,6 +357,9 @@ public class ExportJobBase extends JobBase { try { // Set the external jar to use for the job. job.getConfiguration().set("mapred.jar", ormJarFile); + if (options.getMapreduceJobName() != null) { + job.setJobName(options.getMapreduceJobName()); + } propagateOptionsToJob(job); configureInputFormat(job, tableName, tableClassName, null); http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java index efc2c37..057de84 100644 --- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java @@ -195,6 +195,9 @@ public class ImportJobBase extends JobBase { try { // Set the external jar to use for the job. job.getConfiguration().set("mapred.jar", ormJarFile); + if (options.getMapreduceJobName() != null) { + job.setJobName(options.getMapreduceJobName()); + } propagateOptionsToJob(job); configureInputFormat(job, tableName, tableClassName, splitByCol); http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/tool/BaseSqoopTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java index 0c9cda6..1ab3e5f 100644 --- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java +++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java @@ -104,6 +104,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool { public static final String HIVE_PARTITION_VALUE_ARG = "hive-partition-value"; public static final String CREATE_HIVE_TABLE_ARG = "create-hive-table"; + public static final String MAPREDUCE_JOB_NAME = "mapreduce-job-name"; public static final String NUM_MAPPERS_ARG = "num-mappers"; public static final String NUM_MAPPERS_SHORT_ARG = "m"; public static final String COMPRESS_ARG = "compress"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/tool/ExportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ExportTool.java b/src/java/org/apache/sqoop/tool/ExportTool.java index 494887d..6ac3a89 100644 --- a/src/java/org/apache/sqoop/tool/ExportTool.java +++ b/src/java/org/apache/sqoop/tool/ExportTool.java @@ -137,6 +137,10 @@ public class ExportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { .hasArg().withDescription("Columns to export to table") .withLongOpt(COLUMNS_ARG) .create()); + exportOpts.addOption(OptionBuilder.withArgName("name") + .hasArg().withDescription("Set name for generated mapreduce job") + .withLongOpt(MAPREDUCE_JOB_NAME) + .create()); exportOpts.addOption(OptionBuilder.withArgName("n") .hasArg().withDescription("Use 'n' map tasks to export in parallel") .withLongOpt(NUM_MAPPERS_ARG) @@ -243,6 +247,10 @@ public class ExportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { out.setNumMappers(Integer.parseInt(in.getOptionValue(NUM_MAPPERS_ARG))); } + if (in.hasOption(MAPREDUCE_JOB_NAME)) { + out.setMapreduceJobName(in.getOptionValue(MAPREDUCE_JOB_NAME)); + } + if (in.hasOption(EXPORT_PATH_ARG)) { out.setExportDir(in.getOptionValue(EXPORT_PATH_ARG)); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/tool/ImportTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java index 415b315..932e39b 100644 --- a/src/java/org/apache/sqoop/tool/ImportTool.java +++ b/src/java/org/apache/sqoop/tool/ImportTool.java @@ -576,6 +576,10 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { .hasArg().withDescription("Use 'n' map tasks to import in parallel") .withLongOpt(NUM_MAPPERS_ARG) .create(NUM_MAPPERS_SHORT_ARG)); + importOpts.addOption(OptionBuilder.withArgName("name") + .hasArg().withDescription("Set name for generated mapreduce job") + .withLongOpt(MAPREDUCE_JOB_NAME) + .create()); importOpts.addOption(OptionBuilder .withDescription("Enable compression") .withLongOpt(COMPRESS_ARG) @@ -774,6 +778,10 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool { out.setNumMappers(Integer.parseInt(in.getOptionValue(NUM_MAPPERS_ARG))); } + if (in.hasOption(MAPREDUCE_JOB_NAME)) { + out.setMapreduceJobName(in.getOptionValue(MAPREDUCE_JOB_NAME)); + } + if (in.hasOption(COMPRESS_ARG)) { out.setUseCompression(true); }
