Repository: sqoop Updated Branches: refs/heads/trunk 690541533 -> afd6a8610
SQOOP-3159: Sqoop (export + --table) with Oracle table_name having '$' fails with error (ORA-00942 or java.lang.NoClassDefFoundError) (Szabolcs Vasas via Anna Szonyi) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/afd6a861 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/afd6a861 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/afd6a861 Branch: refs/heads/trunk Commit: afd6a8610e8ac4291e52c2c167fe7b8d6f706be8 Parents: 6905415 Author: Anna Szonyi <[email protected]> Authored: Mon Mar 27 16:15:02 2017 +0200 Committer: Anna Szonyi <[email protected]> Committed: Mon Mar 27 16:15:02 2017 +0200 ---------------------------------------------------------------------- .../apache/sqoop/orm/CompilationManager.java | 54 +++++-- src/test/com/cloudera/sqoop/TestExport.java | 10 ++ .../sqoop/manager/OracleExportTest.java | 16 +++ .../OracleSpecialCharacterTableImportTest.java | 130 +++++++++++++++++ .../sqoop/orm/TestCompilationManager.java | 139 +++++++++++++++++++ 5 files changed, 334 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/java/org/apache/sqoop/orm/CompilationManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java index c1a656b..3322c8b 100644 --- a/src/java/org/apache/sqoop/orm/CompilationManager.java +++ b/src/java/org/apache/sqoop/orm/CompilationManager.java @@ -45,6 +45,8 @@ import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.util.FileListing; import com.cloudera.sqoop.util.Jars; +import static org.apache.commons.lang3.StringUtils.substringBeforeLast; + /** * Manages the compilation of a bunch of .java files into .class files * and eventually a jar. @@ -60,6 +62,9 @@ public class CompilationManager { public static final Log LOG = LogFactory.getLog( CompilationManager.class.getName()); + private static final String INNER_CLASS_SEPARATOR = "$"; + private static final String CLASS_EXTENSION = ".class"; + private static final String JAVA_EXTENSION = ".java"; private SqoopOptions options; private List<String> sources; @@ -299,26 +304,17 @@ public class CompilationManager { // with the base directory where class files were put; // we only record the subdir parts in the zip entry. String fullPath = entry.getAbsolutePath(); - String chompedPath = fullPath.substring(baseDirName.length()); - int indexOfDollarSign = chompedPath.indexOf("$"); - String innerTypesChompedPath = chompedPath - .substring(0, indexOfDollarSign == -1 ? chompedPath.length() : indexOfDollarSign); - - boolean include = chompedPath.endsWith(".class") - && (sources.contains( - chompedPath.substring(0, chompedPath.length() - ".class".length()) - + ".java") - || sources.contains(innerTypesChompedPath + ".java")); - - if (include) { + String classFileName = fullPath.substring(baseDirName.length()); + + if (includeFileInJar(classFileName)) { // include this file. if (Shell.WINDOWS) { // In Windows OS, elements in jar files still need to have a path // separator of '/' rather than the default File.separator which is '\' - chompedPath = chompedPath.replace(File.separator, "/"); + classFileName = classFileName.replace(File.separator, "/"); } - LOG.debug("Got classfile: " + entry.getPath() + " -> " + chompedPath); - ZipEntry ze = new ZipEntry(chompedPath); + LOG.debug("Got classfile: " + entry.getPath() + " -> " + classFileName); + ZipEntry ze = new ZipEntry(classFileName); jstream.putNextEntry(ze); copyFileToStream(entry, jstream); jstream.closeEntry(); @@ -327,6 +323,34 @@ public class CompilationManager { } } + boolean includeFileInJar(String classFileName) { + if (!classFileName.endsWith(CLASS_EXTENSION)) { + return false; + } + + String className = substringBeforeLast(classFileName, CLASS_EXTENSION); + String sourceFileOfClass = className + JAVA_EXTENSION; + + return isInnerClass(sourceFileOfClass) || isOuterClass(sourceFileOfClass); + } + + boolean isInnerClass(String sourceFileName) { + String potentialOuterClassName = potentialOuterClassNameOf(sourceFileName); + + String sourceFileOfPotentialOuterClass = potentialOuterClassName + JAVA_EXTENSION; + + return isOuterClass(sourceFileOfPotentialOuterClass); + } + + boolean isOuterClass(String sourceFileOfClass) { + return sources.contains(sourceFileOfClass); + } + + + String potentialOuterClassNameOf(String className) { + return substringBeforeLast(className, INNER_CLASS_SEPARATOR); + } + /** * Create an output jar file to use when executing MapReduce jobs. */ http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/com/cloudera/sqoop/TestExport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestExport.java b/src/test/com/cloudera/sqoop/TestExport.java index df5a663..b2edc53 100644 --- a/src/test/com/cloudera/sqoop/TestExport.java +++ b/src/test/com/cloudera/sqoop/TestExport.java @@ -885,4 +885,14 @@ public class TestExport extends ExportJobTestCase { assertColMinAndMax(forIdx(2), gen2); assertColMinAndMax(forIdx(1), genNull); } + + protected void testExportToTableWithName(String tableName) throws IOException, SQLException { + final int TOTAL_RECORDS = 10; + setCurTableName(tableName); + createTextFile(0, TOTAL_RECORDS, false); + createTable(); + runExport(getArgv(true, 10, 10)); + verifyExport(TOTAL_RECORDS); + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/com/cloudera/sqoop/manager/OracleExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/OracleExportTest.java b/src/test/com/cloudera/sqoop/manager/OracleExportTest.java index ec56cbe..fe2e265 100644 --- a/src/test/com/cloudera/sqoop/manager/OracleExportTest.java +++ b/src/test/com/cloudera/sqoop/manager/OracleExportTest.java @@ -294,4 +294,20 @@ public class OracleExportTest extends TestExport { "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled", "true"))); verifyExport(TOTAL_RECORDS); } + + @Test + public void testExportToTableWithNameEndingWithDollarSign() throws IOException, SQLException { + testExportToTableWithName("DOLLAR$"); + } + + @Test + public void testExportToTableWithNameContainingDollarSign() throws IOException, SQLException { + testExportToTableWithName("FOO$BAR"); + } + + @Test + public void testExportToTableWithNameContainingHashtag() throws IOException, SQLException { + testExportToTableWithName("FOO#BAR"); + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java b/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java new file mode 100644 index 0000000..907be49 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle; + +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.manager.OracleUtils; +import com.cloudera.sqoop.testutil.CommonArgs; +import com.cloudera.sqoop.testutil.ImportJobTestCase; +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class OracleSpecialCharacterTableImportTest extends ImportJobTestCase { + + @Parameterized.Parameters(name = "tableName = {0}") + public static Iterable<? extends Object> tableNameParameters() { + return Arrays.asList("DOLLAR$","FOO$BAR", "T#1"); + } + + public static final Log LOG = LogFactory.getLog( + OracleSpecialCharacterTableImportTest.class.getName()); + + private final String tableName; + + @Override + protected boolean useHsqldbTestServer() { + return false; + } + + @Override + protected String getConnectString() { + return OracleUtils.CONNECT_STRING; + } + + @Override + protected SqoopOptions getSqoopOptions(Configuration conf) { + SqoopOptions opts = new SqoopOptions(conf); + OracleUtils.setOracleAuth(opts); + return opts; + } + + @Override + protected void dropTableIfExists(String table) throws SQLException { + OracleUtils.dropTable(table, getManager()); + } + + @After + public void tearDown() { + try { + OracleUtils.dropTable(getTableName(), getManager()); + } catch (SQLException e) { + LOG.error("Test table could not be dropped", e); + } + super.tearDown(); + } + + protected String [] getArgv() { + ArrayList<String> args = new ArrayList<String>(); + + CommonArgs.addHadoopFlags(args); + + args.add("--connect"); + args.add(getConnectString()); + args.add("--username"); + args.add(OracleUtils.ORACLE_USER_NAME); + args.add("--password"); + args.add(OracleUtils.ORACLE_USER_PASS); + args.add("--target-dir"); + args.add(getWarehouseDir()); + args.add("--num-mappers"); + args.add("1"); + args.add("--table"); + args.add(getTableName()); + + return args.toArray(new String[0]); + } + + public OracleSpecialCharacterTableImportTest(String tableName) { + this.tableName = tableName; + } + + @Test + public void testImportWithTableNameContainingSpecialCharacters() throws IOException { + String [] types = { "VARCHAR(50)"}; + String [] vals = { "'hello, world!'"}; + setCurTableName(tableName); + createTableWithColTypes(types, vals); + String[] args = getArgv(); + runImport(args); + + Path warehousePath = new Path(this.getWarehouseDir()); + Path filePath = new Path(warehousePath, "part-m-00000"); + String output = Files.toString(new File(filePath.toString()), Charsets.UTF_8); + + assertEquals("hello, world!", output.trim()); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/org/apache/sqoop/orm/TestCompilationManager.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/orm/TestCompilationManager.java b/src/test/org/apache/sqoop/orm/TestCompilationManager.java new file mode 100644 index 0000000..6e91d31 --- /dev/null +++ b/src/test/org/apache/sqoop/orm/TestCompilationManager.java @@ -0,0 +1,139 @@ +package org.apache.sqoop.orm; + +import com.cloudera.sqoop.SqoopOptions; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class TestCompilationManager { + + private CompilationManager compilationManager; + + private SqoopOptions sqoopOptions; + + @Before + public void before() { + sqoopOptions = mock(SqoopOptions.class); + compilationManager = new CompilationManager(sqoopOptions); + } + + @Test + public void testPotentialOuterClassNameOfWithPlainClassName() { + String inputClassName = "FooBar"; + String expectedOutput = inputClassName; + + assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName)); + } + + @Test + public void testPotentialOuterClassNameOfWithClassNameEndingWithDollar() { + String inputClassName = "FooBar$"; + String expectedOutput = "FooBar"; + + assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName)); + } + + @Test + public void testPotentialOuterClassNameOfWithInnerClassName() { + String inputClassName = "Foo$Bar"; + String expectedOutput = "Foo"; + + assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName)); + } + + @Test + public void testPotentialOuterClassNameOfWithOuterClassNameContainingDollarSign() { + String inputClassName = "Foo$Bar$1"; + String expectedOutput = "Foo$Bar"; + + assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName)); + } + + @Test + public void testPotentialOuterClassNameOfWithOuterClassNameEndingWithDollarSign() { + String inputClassName = "FooBar$$1"; + String expectedOutput = "FooBar$"; + + assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName)); + } + + @Test + public void testIsInnerClassWithOuterClass() { + String inputSourceFileName = "Foo$Bar.java"; + compilationManager.addSourceFile(inputSourceFileName); + boolean expectedOutput = false; + + assertEquals(expectedOutput, compilationManager.isInnerClass(inputSourceFileName)); + } + + @Test + public void testIsInnerClassWithNonExistingClass() { + String inputSourceFileName = "ThisDoesNotExist.java"; + boolean expectedOutput = false; + + assertEquals(expectedOutput, compilationManager.isInnerClass(inputSourceFileName)); + } + + @Test + public void testIsInnerClassWithInnerClass() { + String inputSourceFileName = "Foo$Bar$1.java"; + compilationManager.addSourceFile("Foo$Bar.java"); + boolean expectedOutput = true; + + assertEquals(expectedOutput, compilationManager.isInnerClass(inputSourceFileName)); + } + + @Test + public void testIsOuterClassWithExistingSourceFile() { + String inputSourceFileName = "OuterClass.java"; + compilationManager.addSourceFile(inputSourceFileName); + boolean expectedOutput = true; + + assertEquals(expectedOutput, compilationManager.isOuterClass(inputSourceFileName)); + } + + @Test + public void testIsOuterClassWithNonExistingSourceFile() { + String inputSourceFileName = "OuterClass.java"; + boolean expectedOutput = false; + + assertEquals(expectedOutput, compilationManager.isOuterClass(inputSourceFileName)); + } + + @Test + public void testIncludeFileInJarWithNonClassFile() { + String inputClassFileName = "FooBar.txt"; + boolean expectedOutput = false; + + assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName)); + } + + @Test + public void testIncludeFileInJarWithSourceFilePresent() { + String inputClassFileName = "FooBar.class"; + boolean expectedOutput = true; + compilationManager.addSourceFile("FooBar.java"); + + assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName)); + } + + @Test + public void testIncludeFileInJarWithSourceFileNotPresent() { + String inputClassFileName = "FooBar.class"; + boolean expectedOutput = false; + + assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName)); + } + + @Test + public void testIncludeFileInJarWithOuterClassSourceFilePresent() { + String inputClassFileName = "FooBar$1.class"; + boolean expectedOutput = true; + compilationManager.addSourceFile("FooBar.java"); + + assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName)); + } + +} \ No newline at end of file
