Updated Branches: refs/heads/trunk 86812b853 -> f2ba7eaae
SQOOP-937: Dont generate ORM files for Direct mode connectors (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f2ba7eaa Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f2ba7eaa Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f2ba7eaa Branch: refs/heads/trunk Commit: f2ba7eaae25ac23c88651e1d14b9624aa5ded6ca Parents: 86812b8 Author: Jarek Jarcec Cecho <[email protected]> Authored: Wed Mar 20 14:30:04 2013 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Wed Mar 20 14:30:04 2013 -0700 ---------------------------------------------------------------------- src/java/org/apache/sqoop/manager/ConnManager.java | 15 +++++ .../apache/sqoop/manager/DirectNetezzaManager.java | 5 ++ .../org/apache/sqoop/manager/ExportJobContext.java | 7 ++ .../org/apache/sqoop/manager/ImportJobContext.java | 7 ++ .../org/apache/sqoop/mapreduce/ExportJobBase.java | 10 +++- .../org/apache/sqoop/mapreduce/ImportJobBase.java | 9 ++- src/java/org/apache/sqoop/mapreduce/JobBase.java | 1 + src/java/org/apache/sqoop/orm/ClassWriter.java | 15 ++++- src/java/org/apache/sqoop/tool/CodeGenTool.java | 16 +++++- .../com/cloudera/sqoop/orm/TestClassWriter.java | 48 +++++++++++++++ 10 files changed, 125 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ConnManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/ConnManager.java b/src/java/org/apache/sqoop/manager/ConnManager.java index 1b32dc9..a1ac38e 100644 --- a/src/java/org/apache/sqoop/manager/ConnManager.java +++ b/src/java/org/apache/sqoop/manager/ConnManager.java @@ -639,5 +639,20 @@ public abstract class ConnManager { public String getInputBoundsQuery(String splitByCol, String sanitizedQuery) { return null; } + + /** + * This method allows the ConnManager to override the generation of ORM + * classes if the SQOOP generated classes are not used by it. + * A return value of false from this method means that the SQOOP ORM + * classes are needed to use with the connector. + * A return value of true indicates that the connection manager does not + * use the SQOOP ORM classes. For example, in the Direct mode of some of + * the connectors, the text files are directly processed by DB specific + * facilities without even being passed through the SQOOP process and + * in those circumstances, it makes sense to disable the ORM generation. + */ + public boolean isORMFacilitySelfManaged() { + return false; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java index 0a1e605..ef98936 100644 --- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java +++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java @@ -246,4 +246,9 @@ public class DirectNetezzaManager extends NetezzaManager { public boolean supportsStagingForExport() { return false; } + + @Override + public boolean isORMFacilitySelfManaged() { + return true; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ExportJobContext.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/ExportJobContext.java b/src/java/org/apache/sqoop/manager/ExportJobContext.java index 5699e2f..2a6f2b5 100644 --- a/src/java/org/apache/sqoop/manager/ExportJobContext.java +++ b/src/java/org/apache/sqoop/manager/ExportJobContext.java @@ -18,6 +18,9 @@ package org.apache.sqoop.manager; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.util.Jars; + import com.cloudera.sqoop.SqoopOptions; /** @@ -35,6 +38,10 @@ public class ExportJobContext { final SqoopOptions opts) { this.tableName = table; this.jarFile = jar; + if (this.jarFile == null) { + // Set the jarFile to the hadoop core jar file. + this.jarFile = Jars.getJarPathForClass(Configuration.class); + } this.options = opts; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ImportJobContext.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/ImportJobContext.java b/src/java/org/apache/sqoop/manager/ImportJobContext.java index 09a7abe..354cd15 100644 --- a/src/java/org/apache/sqoop/manager/ImportJobContext.java +++ b/src/java/org/apache/sqoop/manager/ImportJobContext.java @@ -21,7 +21,10 @@ package org.apache.sqoop.manager; import org.apache.hadoop.mapreduce.InputFormat; import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat; import com.cloudera.sqoop.SqoopOptions; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.sqoop.util.Jars; /** * A set of parameters describing an import operation; this is passed to @@ -40,6 +43,10 @@ public class ImportJobContext { final SqoopOptions opts, final Path destination) { this.tableName = table; this.jarFile = jar; + if (this.jarFile == null) { + // Set the jarFile to the hadoop core jar file. + this.jarFile = Jars.getJarPathForClass(Configuration.class); + } this.options = opts; this.inputFormatClass = DataDrivenDBInputFormat.class; this.destination = destination; http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java index ff84974..1065d0b 100644 --- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java @@ -321,8 +321,14 @@ public class ExportJobBase extends JobBase { } } - String tableClassName = - new TableClassName(options).getClassForTable(outputTableName); + + String tableClassName = null; + if (!cmgr.isORMFacilitySelfManaged()) { + tableClassName = + new TableClassName(options).getClassForTable(outputTableName); + } + // For ORM self managed, we leave the tableClassName to null so that + // we don't check for non-existing classes. String ormJarFile = context.getJarFile(); LOG.info("Beginning export of " + outputTableName); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java index f766532..2465f3f 100644 --- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java @@ -196,9 +196,14 @@ public class ImportJobBase extends JobBase { } else { LOG.info("Beginning query import."); } + String tableClassName = null; + if (!getContext().getConnManager().isORMFacilitySelfManaged()) { + tableClassName = + new TableClassName(options).getClassForTable(tableName); + } + // For ORM self managed, we leave the tableClassName to null so that + // we don't check for non-existing classes. - String tableClassName = - new TableClassName(options).getClassForTable(tableName); loadJars(conf, ormJarFile, tableClassName); Job job = new Job(conf); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/JobBase.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java index 4e7723f..0df1156 100644 --- a/src/java/org/apache/sqoop/mapreduce/JobBase.java +++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java @@ -220,6 +220,7 @@ public class JobBase { */ protected void loadJars(Configuration conf, String ormJarFile, String tableClassName) throws IOException { + boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address")) || "local".equals(conf.get("mapred.job.tracker")); if (isLocal) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/orm/ClassWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java index 982e444..0202f7f 100644 --- a/src/java/org/apache/sqoop/orm/ClassWriter.java +++ b/src/java/org/apache/sqoop/orm/ClassWriter.java @@ -1064,13 +1064,24 @@ public class ClassWriter { return cleanedColNames; } + /** + * Made this a separate method to overcome the 150 line limit of checkstyle. + */ + private void logORMSelfGenerationMessage() { + LOG.info("The connection manager declares that it self manages mapping" + + " between records & fields and rows & columns. No class will" + + " will be generated."); + } /** * Generate the ORM code for the class. */ public void generate() throws IOException { Map<String, Integer> columnTypes = getColumnTypes(); - + if (connManager.isORMFacilitySelfManaged()) { + logORMSelfGenerationMessage(); + return; + } if (columnTypes == null) { throw new IOException("No columns to generate for ClassWriter"); } @@ -1110,7 +1121,6 @@ public class ClassWriter { } columnTypes.put(identifier, type); } - // Check that all explicitly mapped columns are present in result set Properties mapping = options.getMapColumnJava(); if (mapping != null && !mapping.isEmpty()) { @@ -1207,7 +1217,6 @@ public class ClassWriter { // ignored because we're closing. } } - if (null != ostream) { try { ostream.close(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/tool/CodeGenTool.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java index 8a4aa42..dd34a97 100644 --- a/src/java/org/apache/sqoop/tool/CodeGenTool.java +++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java @@ -71,10 +71,24 @@ public class CodeGenTool extends com.cloudera.sqoop.tool.BaseSqoopTool { // This code generator is being invoked as part of an import or export // process, and the user has pre-specified a jar and class to use. // Don't generate. + if (manager.isORMFacilitySelfManaged()) { + // No need to generated any ORM. Ignore any jar file given on + // command line also. + LOG.info("The connection manager declares that it self manages mapping" + + " between records & fields and rows & columns. The jar file " + + " provided will have no effect"); + } LOG.info("Using existing jar: " + existingJar); return existingJar; } - + if (manager.isORMFacilitySelfManaged()) { + // No need to generated any ORM. Ignore any jar file given on + // command line also. + LOG.info("The connection manager declares that it self manages mapping" + + " between records & fields and rows & columns. No class will" + + " will be generated."); + return null; + } LOG.info("Beginning code generation"); CompilationManager compileMgr = new CompilationManager(options); ClassWriter classWriter = new ClassWriter(options, manager, tableName, http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/test/com/cloudera/sqoop/orm/TestClassWriter.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java index 3b77571..bfb25b0 100644 --- a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java +++ b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java @@ -38,6 +38,7 @@ import org.junit.Before; import org.junit.Test; import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.TestConnFactory.DummyManager; import com.cloudera.sqoop.manager.ConnManager; import com.cloudera.sqoop.testutil.DirUtil; import com.cloudera.sqoop.testutil.HsqldbTestServer; @@ -485,4 +486,51 @@ public class TestClassWriter extends TestCase { } fail("we shouldn't successfully generate code"); } + + private void runFailedGenerationTest(String [] argv, + String classNameToCheck) { + File codeGenDirFile = new File(CODE_GEN_DIR); + File classGenDirFile = new File(JAR_GEN_DIR); + + try { + options = new ImportTool().parseArguments(argv, + null, options, true); + } catch (Exception e) { + LOG.error("Could not parse options: " + e.toString()); + } + + CompilationManager compileMgr = new CompilationManager(options); + ClassWriter writer = new ClassWriter(options, manager, + HsqldbTestServer.getTableName(), compileMgr); + + try { + writer.generate(); + compileMgr.compile(); + fail("ORM class file generation succeeded when it was expected to fail"); + } catch (IOException ioe) { + LOG.error("Got IOException from ORM generation as expected : " + + ioe.toString()); + } + } + /** + * A dummy manager that declares that it ORM is self managed. + */ + public static class DummyDirectManager extends DummyManager { + @Override + public boolean isORMFacilitySelfManaged() { + return true; + } + } + + @Test + public void testNoClassGeneration() throws Exception { + manager = new DummyDirectManager(); + String [] argv = { + "--bindir", + JAR_GEN_DIR, + "--outdir", + CODE_GEN_DIR, + }; + runFailedGenerationTest(argv, HsqldbTestServer.getTableName()); + } }
