Repository: sqoop Updated Branches: refs/heads/sqoop2 d1c47b22d -> 5b5042c1f
SQOOP-2508: Sqoop2: Findbugs: Fix warnings in connector-sdk module (Colin Ma via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5b5042c1 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5b5042c1 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5b5042c1 Branch: refs/heads/sqoop2 Commit: 5b5042c1f23e39c14882e868eb42ca6e1780a9d0 Parents: d1c47b2 Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Aug 20 09:16:28 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Aug 20 09:16:28 2015 -0700 ---------------------------------------------------------------------- .../sqoop/connector/common/SqoopAvroUtils.java | 7 ++- .../sqoop/connector/common/SqoopIDFUtils.java | 66 ++++++++++++-------- .../idf/AVROIntermediateDataFormat.java | 20 +++--- .../idf/JSONIntermediateDataFormat.java | 18 +++--- .../connector/common/TestSqoopIDFUtils.java | 8 +-- 5 files changed, 65 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b5042c1/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java index 857271a..985149c 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java @@ -74,6 +74,7 @@ public class SqoopAvroUtils { switch (column.getType()) { case ARRAY: case SET: + assert column instanceof AbstractComplexListType; AbstractComplexListType listColumn = (AbstractComplexListType) column; return Schema.createArray(toAvroFieldType(listColumn.getListType())); case UNKNOWN: @@ -93,20 +94,21 @@ public class SqoopAvroUtils { case ENUM: return createEnumSchema(column); case FIXED_POINT: - Long byteSize = ((FixedPoint) column).getByteSize(); if (SqoopIDFUtils.isInteger(column)) { return Schema.create(Schema.Type.INT); } else { return Schema.create(Schema.Type.LONG); } case FLOATING_POINT: - byteSize = ((FloatingPoint) column).getByteSize(); + assert column instanceof FloatingPoint; + Long byteSize = ((FloatingPoint) column).getByteSize(); if (byteSize != null && byteSize <= (Float.SIZE/Byte.SIZE)) { return Schema.create(Schema.Type.FLOAT); } else { return Schema.create(Schema.Type.DOUBLE); } case MAP: + assert column instanceof org.apache.sqoop.schema.type.Map; org.apache.sqoop.schema.type.Map mapColumn = (org.apache.sqoop.schema.type.Map) column; return Schema.createArray(toAvroFieldType(mapColumn.getValue())); case TEXT: @@ -117,6 +119,7 @@ public class SqoopAvroUtils { } public static Schema createEnumSchema(Column column) { + assert column instanceof org.apache.sqoop.schema.type.Enum; Set<String> options = ((org.apache.sqoop.schema.type.Enum) column).getOptions(); List<String> listOptions = new ArrayList<String>(options); return Schema.createEnum(column.getName(), null, SQOOP_SCHEMA_NAMESPACE, listOptions); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b5042c1/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java index 2a7aa1b..52ff2b8 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java @@ -49,7 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.regex.Matcher; +import java.util.Collections; /** * Utility methods for connectors to encode data into the sqoop expected formats @@ -68,7 +68,7 @@ public class SqoopIDFUtils { // implementation. public static final String BYTE_FIELD_CHARSET = "ISO-8859-1"; - public static final Map<Character, String> ORIGINALS = new TreeMap<Character, String>(); + private static final Map<Character, String> ORIGINALS = new TreeMap<Character, String>(); public static final char CSV_SEPARATOR_CHARACTER = ','; public static final char ESCAPE_CHARACTER = '\\'; @@ -77,19 +77,19 @@ public class SqoopIDFUtils { private static final Map<Character, Character> REPLACEMENTS = new TreeMap<Character, Character>(); static { - ORIGINALS.put(new Character((char)0x00), new String(new char[] { ESCAPE_CHARACTER, '0' })); - ORIGINALS.put(new Character((char)0x0A), new String(new char[] { ESCAPE_CHARACTER, 'n' })); - ORIGINALS.put(new Character((char)0x0D), new String(new char[] { ESCAPE_CHARACTER, 'r' })); - ORIGINALS.put(new Character((char)0x1A), new String(new char[] { ESCAPE_CHARACTER, 'Z' })); - ORIGINALS.put(new Character((char)0x22), new String(new char[] { ESCAPE_CHARACTER, '"' })); - ORIGINALS.put(new Character((char)0x27), new String(new char[] { ESCAPE_CHARACTER, '\'' })); + ORIGINALS.put(Character.valueOf((char)0x00), new String(new char[] { ESCAPE_CHARACTER, '0' })); + ORIGINALS.put(Character.valueOf((char)0x0A), new String(new char[] { ESCAPE_CHARACTER, 'n' })); + ORIGINALS.put(Character.valueOf((char)0x0D), new String(new char[] { ESCAPE_CHARACTER, 'r' })); + ORIGINALS.put(Character.valueOf((char)0x1A), new String(new char[] { ESCAPE_CHARACTER, 'Z' })); + ORIGINALS.put(Character.valueOf((char)0x22), new String(new char[] { ESCAPE_CHARACTER, '"' })); + ORIGINALS.put(Character.valueOf((char)0x27), new String(new char[] { ESCAPE_CHARACTER, '\'' })); - REPLACEMENTS.put('0', new Character((char)0x00)); - REPLACEMENTS.put('n', new Character((char)0x0A)); - REPLACEMENTS.put('r', new Character((char)0x0D)); - REPLACEMENTS.put('Z', new Character((char)0x1A)); - REPLACEMENTS.put('"', new Character((char)0x22)); - REPLACEMENTS.put('\'', new Character((char)0x27)); + REPLACEMENTS.put('0', Character.valueOf((char)0x00)); + REPLACEMENTS.put('n', Character.valueOf((char)0x0A)); + REPLACEMENTS.put('r', Character.valueOf((char)0x0D)); + REPLACEMENTS.put('Z', Character.valueOf((char)0x1A)); + REPLACEMENTS.put('"', Character.valueOf((char)0x22)); + REPLACEMENTS.put('\'', Character.valueOf((char)0x27)); } // http://www.joda.org/joda-time/key_format.html provides details on the @@ -106,14 +106,15 @@ public class SqoopIDFUtils { public static final DateTimeFormatter tfWithFraction = DateTimeFormat.forPattern("HH:mm:ss.SSS"); public static final DateTimeFormatter tfWithNoFraction = DateTimeFormat.forPattern("HH:mm:ss"); - public static final String[] TRUE_BIT_VALUES = new String[] { "1", "true", "TRUE" }; - public static final Set<String> TRUE_BIT_SET = new HashSet<String>(Arrays.asList(TRUE_BIT_VALUES)); - public static final String[] FALSE_BIT_VALUES = new String[] { "0", "false", "FALSE" }; - public static final Set<String> FALSE_BIT_SET = new HashSet<String>(Arrays.asList(FALSE_BIT_VALUES)); + private static final String[] TRUE_BIT_VALUES = new String[] { "1", "true", "TRUE" }; + public static final Set<String> TRUE_BIT_SET = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(TRUE_BIT_VALUES))); + private static final String[] FALSE_BIT_VALUES = new String[] { "0", "false", "FALSE" }; + public static final Set<String> FALSE_BIT_SET = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(FALSE_BIT_VALUES))); // ******** Number Column Type utils*********** public static boolean isInteger(Column column) { + assert column instanceof FixedPoint; Long byteSize = ((FixedPoint) column).getByteSize(); Boolean signed = ((FixedPoint) column).isSigned(); @@ -129,15 +130,15 @@ public class SqoopIDFUtils { public static String toCSVFixedPoint(Object obj, Column column) { if (isInteger(column)) { if (obj instanceof Number) { - return new Integer(((Number)obj).intValue()).toString(); + return Integer.toString(((Number) obj).intValue()); } else { - return new Integer(obj.toString()).toString(); + return Integer.valueOf(obj.toString()).toString(); } } else { if (obj instanceof Number) { - return new Long(((Number)obj).longValue()).toString(); + return Long.toString(((Number) obj).longValue()); } else { - return new Long(obj.toString()).toString(); + return Long.valueOf(obj.toString()).toString(); } } } @@ -153,6 +154,7 @@ public class SqoopIDFUtils { } public static String toCSVFloatingPoint(Object obj, Column column) { + assert column instanceof FloatingPoint; Long byteSize = ((FloatingPoint) column).getByteSize(); if (byteSize != null && byteSize <= (Float.SIZE / Byte.SIZE)) { return ((Float) obj).toString(); @@ -163,6 +165,7 @@ public class SqoopIDFUtils { public static Object toFloatingPoint(String csvString, Column column) { Object returnValue; + assert column instanceof FloatingPoint; Long byteSize = ((FloatingPoint) column).getByteSize(); if (byteSize != null && byteSize <= (Float.SIZE / Byte.SIZE)) { returnValue = Float.valueOf(csvString); @@ -177,6 +180,7 @@ public class SqoopIDFUtils { } public static Object toDecimal(String csvString, Column column) { + assert column instanceof org.apache.sqoop.schema.type.Decimal; Integer precision = ((org.apache.sqoop.schema.type.Decimal) column).getPrecision(); Integer scale = ((org.apache.sqoop.schema.type.Decimal) column).getScale(); BigDecimal bd = null; @@ -190,7 +194,7 @@ public class SqoopIDFUtils { // we have decided to use the default MathContext DEFAULT_ROUNDINGMODE // which is RoundingMode.HALF_UP, // we are aware that there may be some loss - bd.setScale(scale, RoundingMode.HALF_UP); + bd = bd.setScale(scale, RoundingMode.HALF_UP); } return bd; } @@ -218,11 +222,14 @@ public class SqoopIDFUtils { // *********** DATE and TIME Column Type utils ********** public static String toCSVDate(Object obj) { + assert obj instanceof org.joda.time.LocalDate; org.joda.time.LocalDate date = (org.joda.time.LocalDate) obj; return encloseWithQuotes(df.print(date)); } public static String toCSVTime(Object obj, Column col) { + assert col instanceof org.apache.sqoop.schema.type.Time; + assert obj instanceof org.joda.time.LocalTime; if (((org.apache.sqoop.schema.type.Time) col).hasFraction()) { return encloseWithQuotes(tfWithFraction.print((org.joda.time.LocalTime) obj)); } else { @@ -241,6 +248,8 @@ public class SqoopIDFUtils { // *********** DATE TIME Column Type utils ********** public static String toCSVLocalDateTime(Object obj, Column col) { + assert obj instanceof org.joda.time.LocalDateTime; + assert col instanceof org.apache.sqoop.schema.type.DateTime; org.joda.time.LocalDateTime localDateTime = (org.joda.time.LocalDateTime) obj; org.apache.sqoop.schema.type.DateTime column = (org.apache.sqoop.schema.type.DateTime) col; if (column.hasFraction()) { @@ -251,6 +260,8 @@ public class SqoopIDFUtils { } public static String toCSVDateTime(Object obj, Column col) { + assert obj instanceof org.joda.time.DateTime; + assert col instanceof org.apache.sqoop.schema.type.DateTime; org.joda.time.DateTime dateTime = (org.joda.time.DateTime) obj; org.apache.sqoop.schema.type.DateTime column = (org.apache.sqoop.schema.type.DateTime) col; if (column.hasFraction() && column.hasTimezone()) { @@ -267,7 +278,8 @@ public class SqoopIDFUtils { public static Object toDateTime(String csvString, Column column) { Object returnValue; String dateTime = removeQuotes(csvString); - org.apache.sqoop.schema.type.DateTime col = ((org.apache.sqoop.schema.type.DateTime) column); + assert column instanceof org.apache.sqoop.schema.type.DateTime; + org.apache.sqoop.schema.type.DateTime col = (org.apache.sqoop.schema.type.DateTime) column; if (col.hasFraction() && col.hasTimezone()) { // After calling withOffsetParsed method, a string // '2004-06-09T10:20:30-08:00' will create a datetime with a zone of @@ -288,7 +300,8 @@ public class SqoopIDFUtils { public static Long toDateTimeInMillis(String csvString, Column column) { long returnValue; String dateTime = removeQuotes(csvString); - org.apache.sqoop.schema.type.DateTime col = ((org.apache.sqoop.schema.type.DateTime) column); + assert column instanceof org.apache.sqoop.schema.type.DateTime; + org.apache.sqoop.schema.type.DateTime col = (org.apache.sqoop.schema.type.DateTime) column; if (col.hasFraction() && col.hasTimezone()) { // After calling withOffsetParsed method, a string // '2004-06-09T10:20:30-08:00' will create a datetime with a zone of @@ -370,6 +383,7 @@ public class SqoopIDFUtils { public static String toCSVList(Object[] list, Column column) { List<Object> elementList = new ArrayList<Object>(); for (int n = 0; n < list.length; n++) { + assert column instanceof AbstractComplexListType; Column listType = ((AbstractComplexListType) column).getListType(); // 2 level nesting supported if (isColumnListType(listType)) { @@ -751,7 +765,7 @@ public class SqoopIDFUtils { if (csvArray.length != columns.length) { throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0001, - "The data " + csvArray + " has the wrong number of fields."); + "The data " + Arrays.toString(csvArray) + " has the wrong number of fields."); } Object[] objectArray = new Object[csvArray.length]; http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b5042c1/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java index fef8202..68eb6eb 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java @@ -120,6 +120,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe public void write(DataOutput out) throws IOException { // do we need to write the schema? DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(avroSchema); + assert out instanceof DataOutputStream; BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder((DataOutputStream) out, null); writer.write(data, encoder); } @@ -130,6 +131,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe @Override public void read(DataInput in) throws IOException { DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(avroSchema); + assert in instanceof InputStream; Decoder decoder = DecoderFactory.get().binaryDecoder((InputStream) in, null); data = reader.read(null, decoder); } @@ -238,7 +240,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe if (objectArray.length != columns.length) { throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0001, - "The data " + objectArray.toString() + " has the wrong number of fields."); + "The data " + Arrays.toString(objectArray) + " has the wrong number of fields."); } // get avro schema from sqoop schema GenericRecord avroObject = new GenericData.Record(avroSchema); @@ -257,9 +259,6 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe case SET: avroObject.put(columns[i].getName(), toList((Object[]) objectArray[i])); break; - case MAP: - avroObject.put(columns[i].getName(), objectArray[i]); - break; case ENUM: GenericData.EnumSymbol enumValue = new GenericData.EnumSymbol(createEnumSchema(columns[i]), (String) objectArray[i]); @@ -272,6 +271,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe case UNKNOWN: avroObject.put(columns[i].getName(), ByteBuffer.wrap((byte[]) objectArray[i])); break; + case MAP: case FIXED_POINT: case FLOATING_POINT: avroObject.put(columns[i].getName(), objectArray[i]); @@ -298,7 +298,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe .getTime()); break; case BIT: - avroObject.put(columns[i].getName(), Boolean.valueOf((Boolean) objectArray[i])); + avroObject.put(columns[i].getName(), Boolean.valueOf(objectArray[i].toString())); break; default: throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0001, @@ -412,13 +412,12 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe case SET: object[nameIndex] = toObjectArray((List<Object>) obj); break; - case MAP: - object[nameIndex] = obj; - break; case ENUM: // stored as enum symbol case TEXT: // stored as UTF8 + case DECIMAL: + // stored as string object[nameIndex] = obj.toString(); break; case BINARY: @@ -426,15 +425,12 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe // stored as byte buffer object[nameIndex] = getBytesFromByteBuffer(obj); break; + case MAP: case FIXED_POINT: case FLOATING_POINT: // stored as java objects in avro as well object[nameIndex] = obj; break; - case DECIMAL: - // stored as string - object[nameIndex] = obj.toString(); - break; case DATE: Long dateInMillis = (Long) obj; object[nameIndex] = new org.joda.time.LocalDate(dateInMillis); http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b5042c1/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java index 0fac41a..9708c05 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java @@ -37,6 +37,7 @@ import org.json.simple.parser.ParseException; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -223,7 +224,7 @@ public class JSONIntermediateDataFormat extends IntermediateDataFormat<JSONObjec Column[] columns = schema.getColumnsArray(); if (objectArray.length != columns.length) { - throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0001, "The data " + objectArray.toString() + throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0001, "The data " + Arrays.toString(objectArray) + " has the wrong number of fields."); } JSONObject json = new JSONObject(); @@ -251,14 +252,12 @@ public class JSONIntermediateDataFormat extends IntermediateDataFormat<JSONObjec jsonObject.putAll(map); json.put(columns[i].getName(), jsonObject); break; - case ENUM: - case TEXT: - json.put(columns[i].getName(), objectArray[i]); - break; case BINARY: case UNKNOWN: json.put(columns[i].getName(), Base64.encodeBase64String((byte[]) objectArray[i])); break; + case ENUM: + case TEXT: case FIXED_POINT: case FLOATING_POINT: case DECIMAL: @@ -363,13 +362,12 @@ public class JSONIntermediateDataFormat extends IntermediateDataFormat<JSONObjec } Column[] columns = schema.getColumnsArray(); Object[] object = new Object[columns.length]; - - Set<String> jsonKeyNames = json.keySet(); - for (String name : jsonKeyNames) { - Integer nameIndex = schema.getColumnNameIndex(name); + Set<Map.Entry<String,Object>> entrySet = json.entrySet(); + for (Map.Entry<String,Object> entry : entrySet) { + Integer nameIndex = schema.getColumnNameIndex(entry.getKey()); Column column = columns[nameIndex]; - Object obj = json.get(name); + Object obj = entry.getValue(); // null is a possible value if (obj == null && !column.isNullable()) { throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0005, http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b5042c1/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java index f9b676b..aead9ef 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java @@ -328,16 +328,16 @@ public class TestSqoopIDFUtils { Decimal col = new Decimal("dd", 4, 2); assertTrue(toDecimal(text, col) instanceof BigDecimal); BigDecimal bd = (BigDecimal) toDecimal(text, col); - assertEquals("23.44", toCSVDecimal(bd)); + assertEquals(toCSVDecimal(bd), "23.44"); } @Test public void testToDecimaPoint2ReturnsDecimal() { - String text = "23.44444444"; + String text = "123456.44444444"; Decimal col = new Decimal("dd", 8, 2); assertTrue(toDecimal(text, col) instanceof BigDecimal); BigDecimal bd = (BigDecimal) toDecimal(text, col); - assertEquals("23.444444", toCSVDecimal(bd)); + assertEquals(toCSVDecimal(bd), "123456.44"); } @Test @@ -346,7 +346,7 @@ public class TestSqoopIDFUtils { Decimal col = new Decimal("dd", null, null); assertTrue(toDecimal(text, col) instanceof BigDecimal); BigDecimal bd = (BigDecimal) toDecimal(text, col); - assertEquals("23.44444444", toCSVDecimal(bd)); + assertEquals(toCSVDecimal(bd), "23.44444444"); } @Test
