Updated Branches: refs/heads/trunk 658c15848 -> 5d767e796
SQOOP-1071: Avro import is failing when columns are starting with underscore (Jarek Jarcec Cecho via Kate Ting) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5d767e79 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5d767e79 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5d767e79 Branch: refs/heads/trunk Commit: 5d767e796d70830657496558514c349074fbb3eb Parents: 658c158 Author: Kate Ting <[email protected]> Authored: Mon Jun 10 17:17:28 2013 -0400 Committer: Kate Ting <[email protected]> Committed: Mon Jun 10 17:17:28 2013 -0400 ---------------------------------------------------------------------- .../apache/sqoop/orm/AvroSchemaGenerator.java | 2 +- src/test/com/cloudera/sqoop/TestAvroImport.java | 21 +++++++++++++ .../sqoop/testutil/BaseSqoopTestCase.java | 33 +++++++++++++++----- 3 files changed, 47 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java index 864fa0d..806bace 100644 --- a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java +++ b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java @@ -60,7 +60,7 @@ public class AvroSchemaGenerator { List<Field> fields = new ArrayList<Field>(); for (String columnName : columnNames) { - String cleanedCol = ClassWriter.toIdentifier(columnName); + String cleanedCol = ClassWriter.toJavaIdentifier(columnName); int sqlType = columnTypes.get(columnName); Schema avroSchema = toAvroSchema(sqlType, columnName); Field field = new Field(cleanedCol, avroSchema, null, null); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/test/com/cloudera/sqoop/TestAvroImport.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/TestAvroImport.java b/src/test/com/cloudera/sqoop/TestAvroImport.java index 32d1bbe..34a7d41 100644 --- a/src/test/com/cloudera/sqoop/TestAvroImport.java +++ b/src/test/com/cloudera/sqoop/TestAvroImport.java @@ -181,6 +181,27 @@ public class TestAvroImport extends ImportJobTestCase { assertEquals("DATA_COL0", new Utf8("10"), record1.get("DATA_COL0")); } + public void testFirstUnderscoreInColumnName() throws IOException { + String [] names = { "_NAME" }; + String [] types = { "INT" }; + String [] vals = { "1987" }; + createTableWithColTypesAndNames(names, types, vals); + + runImport(getOutputArgv(true, null)); + + Path outputFile = new Path(getTablePath(), "part-m-00000.avro"); + DataFileReader<GenericRecord> reader = read(outputFile); + Schema schema = reader.getSchema(); + assertEquals(Schema.Type.RECORD, schema.getType()); + List<Field> fields = schema.getFields(); + assertEquals(types.length, fields.size()); + + checkField(fields.get(0), "__NAME", Type.INT); + + GenericRecord record1 = reader.next(); + assertEquals("__NAME", 1987, record1.get("__NAME")); + } + private void checkField(Field field, String name, Type type) { assertEquals(name, field.name()); assertEquals(Schema.Type.UNION, field.schema().getType()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java index d6afbc8..877d7f8 100644 --- a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java +++ b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java @@ -290,18 +290,24 @@ public abstract class BaseSqoopTestCase extends TestCase { } /** - * Create a table with a set of columns and add a row of values. + * Create a table with a set of columns with their names and add a row of values. + * @param colNames Column names * @param colTypes the types of the columns to make * @param vals the SQL text for each value to insert */ - protected void createTableWithColTypes(String [] colTypes, String [] vals) { + protected void createTableWithColTypesAndNames(String[] colNames, + String[] colTypes, + String[] vals) { + assert colNames != null; + assert colTypes != null; + assert colNames.length == colTypes.length; + Connection conn = null; PreparedStatement statement = null; String createTableStr = null; String columnDefStr = ""; String columnListStr = ""; String valueListStr = ""; - String [] myColNames = new String[colTypes.length]; try { try { @@ -310,9 +316,7 @@ public abstract class BaseSqoopTestCase extends TestCase { conn = getManager().getConnection(); for (int i = 0; i < colTypes.length; i++) { - String colName = BASE_COL_NAME + Integer.toString(i); - columnDefStr += colName + " " + colTypes[i]; - myColNames[i] = colName; + columnDefStr += colNames[i] + " " + colTypes[i]; if (i < colTypes.length - 1) { columnDefStr += ", "; } @@ -342,7 +346,7 @@ public abstract class BaseSqoopTestCase extends TestCase { if (vals!=null) { for (int i = 0; i < colTypes.length; i++) { - columnListStr += myColNames[i]; + columnListStr += colNames[i]; valueListStr += vals[i]; if (i < colTypes.length - 1) { columnListStr += ", "; @@ -375,7 +379,7 @@ public abstract class BaseSqoopTestCase extends TestCase { } conn.commit(); - this.colNames = myColNames; + this.colNames = colNames; } catch (SQLException se) { if (null != conn) { try { @@ -389,6 +393,19 @@ public abstract class BaseSqoopTestCase extends TestCase { } /** + * Create a table with a set of columns and add a row of values. + * @param colTypes the types of the columns to make + * @param vals the SQL text for each value to insert + */ + protected void createTableWithColTypes(String [] colTypes, String [] vals) { + String[] colNames = new String[colTypes.length]; + for( int i = 0; i < colTypes.length; i++) { + colNames[i] = BASE_COL_NAME + Integer.toString(i); + } + createTableWithColTypesAndNames(colNames, colTypes, vals); + } + + /** * Create a table with a single column and put a data element in it. * @param colType the type of the column to create * @param val the value to insert (reformatted as a string)
