Repository: sqoop Updated Branches: refs/heads/sqoop2 9deed1c2c -> 4ac3c3862
SQOOP-2022: Sqoop2: SqoopIDFUtils uses bit size instead of byteSize to check for INT/LONG/FLOAT/Double (Veena Basavaraj via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/4ac3c386 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/4ac3c386 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/4ac3c386 Branch: refs/heads/sqoop2 Commit: 4ac3c3862bb0a7af3d300badbfe51cb43e0d097c Parents: 9deed1c Author: Abraham Elmahrek <[email protected]> Authored: Wed Jan 21 18:30:39 2015 -0800 Committer: Abraham Elmahrek <[email protected]> Committed: Wed Jan 21 18:32:08 2015 -0800 ---------------------------------------------------------------------- .../sqoop/json/util/SchemaSerialization.java | 10 +- .../apache/sqoop/schema/type/FixedPoint.java | 34 ++---- .../apache/sqoop/schema/type/FloatingPoint.java | 10 +- .../json/util/TestSchemaSerialization.java | 4 +- .../connector/jdbc/util/SqlTypesUtils.java | 13 +- .../sqoop/connector/jdbc/TestExtractor.java | 4 +- .../connector/jdbc/TestFromInitializer.java | 4 +- .../connector/common/AvroDataTypeUtil.java | 5 +- .../sqoop/connector/common/SqoopAvroUtils.java | 4 +- .../sqoop/connector/common/SqoopIDFUtils.java | 23 +++- .../connector/common/TestSqoopIDFUtils.java | 118 +++++++++++++++++-- .../idf/TestAVROIntermediateDataFormat.java | 16 +-- .../idf/TestCSVIntermediateDataFormat.java | 90 +++++++------- .../idf/TestJSONIntermediateDataFormat.java | 16 +-- .../java/org/apache/sqoop/job/TestMatching.java | 10 +- .../apache/sqoop/job/util/MRJobTestUtil.java | 2 +- 16 files changed, 232 insertions(+), 131 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java index a6425fe..4ec2f65 100644 --- a/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java +++ b/common/src/main/java/org/apache/sqoop/json/util/SchemaSerialization.java @@ -80,7 +80,7 @@ public class SchemaSerialization { private static final String TIMEZONE = "timezone"; private static final String PRECISION = "precision"; private static final String SCALE = "scale"; - private static final String UNSIGNED = "unsigned"; + private static final String SIGNED = "signed"; private static final String JDBC_TYPE = "jdbc-type"; @SuppressWarnings("unchecked") @@ -166,7 +166,7 @@ public class SchemaSerialization { break; case FIXED_POINT: ret.put(BYTE_SIZE, ((FixedPoint) column).getByteSize()); - ret.put(UNSIGNED, ((FixedPoint) column).getUnsigned()); + ret.put(SIGNED, ((FixedPoint) column).isSigned()); break; case FLOATING_POINT: ret.put(BYTE_SIZE, ((FloatingPoint) column).getByteSize()); @@ -251,13 +251,13 @@ public class SchemaSerialization { output = new Enum(name, options); break; case FIXED_POINT: - Boolean unsigned = (Boolean) obj.get(UNSIGNED); + Boolean signed = (Boolean) obj.get(SIGNED); Long fixedPointByteSize = (Long) obj.get(BYTE_SIZE); - output = new FixedPoint(name).setByteSize(fixedPointByteSize).setUnsigned(unsigned); + output = new FixedPoint(name, fixedPointByteSize, signed); break; case FLOATING_POINT: Long floatingPointByteSize = (Long) obj.get(BYTE_SIZE); - output = new FloatingPoint(name).setByteSize(floatingPointByteSize); + output = new FloatingPoint(name, floatingPointByteSize); break; case MAP: output = new Map(name, key, value); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/common/src/main/java/org/apache/sqoop/schema/type/FixedPoint.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/schema/type/FixedPoint.java b/common/src/main/java/org/apache/sqoop/schema/type/FixedPoint.java index 84a76ae..7488ae9 100644 --- a/common/src/main/java/org/apache/sqoop/schema/type/FixedPoint.java +++ b/common/src/main/java/org/apache/sqoop/schema/type/FixedPoint.java @@ -41,40 +41,26 @@ public class FixedPoint extends AbstractNumber { */ private Long byteSize; - private Boolean unsigned; + private Boolean signed; - public FixedPoint(String name) { - super(name); - } - - public FixedPoint(String name, Long byteSize, Boolean unsigned) { + public FixedPoint(String name, Long byteSize, Boolean signed) { super(name); this.byteSize = byteSize; - this.unsigned = unsigned; + this.signed = signed; } - public FixedPoint(String name, Boolean nullable, Long byteSize, Boolean unsigned) { + public FixedPoint(String name, Boolean nullable, Long byteSize, Boolean signed) { super(name, nullable); this.byteSize = byteSize; - this.unsigned = unsigned; + this.signed = signed; } public Long getByteSize() { return byteSize; } - public FixedPoint setByteSize(Long byteSize) { - this.byteSize = byteSize; - return this; - } - - public Boolean getUnsigned() { - return unsigned; - } - - public FixedPoint setUnsigned(Boolean unsigned) { - this.unsigned = unsigned; - return this; + public Boolean isSigned() { + return signed; } @Override @@ -87,7 +73,7 @@ public class FixedPoint extends AbstractNumber { return new StringBuilder("FixedPoint{") .append(super.toString()) .append(",byteSize=").append(byteSize) - .append(",unsigned=").append(unsigned) + .append(",signed=").append(signed) .append("}") .toString(); } @@ -102,7 +88,7 @@ public class FixedPoint extends AbstractNumber { if (byteSize != null ? !byteSize.equals(that.byteSize) : that.byteSize != null) return false; - if (unsigned != null ? !unsigned.equals(that.unsigned) : that.unsigned != null) + if (signed != null ? !signed.equals(that.signed) : that.signed != null) return false; return true; @@ -112,7 +98,7 @@ public class FixedPoint extends AbstractNumber { public int hashCode() { int result = super.hashCode(); result = 31 * result + (byteSize != null ? byteSize.hashCode() : 0); - result = 31 * result + (unsigned != null ? unsigned.hashCode() : 0); + result = 31 * result + (signed != null ? signed.hashCode() : 0); return result; } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/common/src/main/java/org/apache/sqoop/schema/type/FloatingPoint.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/schema/type/FloatingPoint.java b/common/src/main/java/org/apache/sqoop/schema/type/FloatingPoint.java index 4fa0c30..5f359e1 100644 --- a/common/src/main/java/org/apache/sqoop/schema/type/FloatingPoint.java +++ b/common/src/main/java/org/apache/sqoop/schema/type/FloatingPoint.java @@ -32,13 +32,10 @@ public class FloatingPoint extends AbstractNumber { /** This field will come handy in connector that might require to use the size information on the schema object to do additional type mappings in their source - Read more infomration : https://issues.apache.org/jira/secure/attachment/12589331/Sqoop2Datatypes.pdf + Read more information : https://issues.apache.org/jira/secure/attachment/12589331/Sqoop2Datatypes.pdf */ private Long byteSize; - public FloatingPoint(String name) { - super(name); - } public FloatingPoint(String name, Long byteSize) { super(name); this.byteSize = byteSize; @@ -53,11 +50,6 @@ public class FloatingPoint extends AbstractNumber { return byteSize; } - public FloatingPoint setByteSize(Long byteSize) { - this.byteSize = byteSize; - return this; - } - @Override public ColumnType getType() { return ColumnType.FLOATING_POINT; http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java index 0cb7175..b7741ec 100644 --- a/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java +++ b/common/src/test/java/org/apache/sqoop/json/util/TestSchemaSerialization.java @@ -193,8 +193,8 @@ public class TestSchemaSerialization { .addColumn(new DateTime("e", true, true)) .addColumn(new Decimal("f")) .addColumn(new Enum("g", Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(new String[] { "X", "Y" }))))) - .addColumn(new FixedPoint("h")) - .addColumn(new FloatingPoint("i")) + .addColumn(new FixedPoint("h", 2L, false)) + .addColumn(new FloatingPoint("i", 4L)) .addColumn(new Map("j", new Text("j1"), new Text("j2"))) .addColumn(new Set("k", new Text("k1"))) .addColumn(new Text("l")) http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/util/SqlTypesUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/util/SqlTypesUtils.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/util/SqlTypesUtils.java index 8228397..d84498e 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/util/SqlTypesUtils.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/util/SqlTypesUtils.java @@ -48,9 +48,13 @@ public class SqlTypesUtils { switch (sqlType) { case Types.SMALLINT: case Types.TINYINT: - return new FixedPoint(columnName).setByteSize(2L); + // only supports signed values + return new FixedPoint(columnName, 2L, true); case Types.INTEGER: - return new FixedPoint(columnName).setByteSize(4L); + // only supports signed values + return new FixedPoint(columnName, 4L, true); + case Types.BIGINT: + return new FixedPoint(columnName, 8L, true); case Types.CLOB: case Types.VARCHAR: @@ -72,12 +76,13 @@ public class SqlTypesUtils { case Types.FLOAT: case Types.REAL: + return new FloatingPoint(columnName, 4L); case Types.DOUBLE: - return new FloatingPoint(columnName); + return new FloatingPoint(columnName, 8L); + //TODO:SQOOP-2027 The following mapping needs to be revisited case Types.NUMERIC: case Types.DECIMAL: - case Types.BIGINT: return new Decimal(columnName); case Types.BIT: http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java index 83599d3..f3ec607 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExtractor.java @@ -100,7 +100,7 @@ public class TestExtractor { Schema schema = new Schema("TestExtractor"); // dummy columns added, all we need is the column count to match to the // result set - schema.addColumn(new FixedPoint("c1")).addColumn(new Decimal("c2")).addColumn(new Text("c3")).addColumn(new Date("c4")); + schema.addColumn(new FixedPoint("c1",2L, true)).addColumn(new Decimal("c2")).addColumn(new Text("c3")).addColumn(new Date("c4")); ExtractorContext extractorContext = new ExtractorContext(context, writer, schema); @@ -140,7 +140,7 @@ public class TestExtractor { Schema schema = new Schema("TestExtractor"); // dummy columns added, all we need is the column count to match to the // result set - schema.addColumn(new FixedPoint("c1")).addColumn(new Text("c2")).addColumn(new Date("c3")); + schema.addColumn(new FixedPoint("c1", 2L, true)).addColumn(new Text("c2")).addColumn(new Date("c3")); ExtractorContext extractorContext = new ExtractorContext(context, writer, schema); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java index 249cd34..52003ab 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestFromInitializer.java @@ -102,8 +102,8 @@ public class TestFromInitializer { */ public Schema getSchema(String name) { return new Schema(name) - .addColumn(new FixedPoint("ICOL").setByteSize(4L)) - .addColumn(new FloatingPoint("DCOL")) + .addColumn(new FixedPoint("ICOL", 4L, true)) + .addColumn(new FloatingPoint("DCOL", 8L)) .addColumn(new Text("VCOL")) ; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/AvroDataTypeUtil.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/AvroDataTypeUtil.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/AvroDataTypeUtil.java index 759bd9f..43469a6 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/AvroDataTypeUtil.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/AvroDataTypeUtil.java @@ -76,12 +76,13 @@ public class AvroDataTypeUtil { assert type != Schema.Type.UNION; switch (type) { case INT: + return new FixedPoint(name, 4L, true); case LONG: - return new FixedPoint(name); + return new FixedPoint(name, 8L, true); case STRING: return new Text(name); case DOUBLE: - return new FloatingPoint(name); + return new FloatingPoint(name, 8L); case BOOLEAN: return new Bit(name); case BYTES: http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java index d62a8a4..5e6f238 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopAvroUtils.java @@ -94,14 +94,14 @@ public class SqoopAvroUtils { return createEnumSchema(column); case FIXED_POINT: Long byteSize = ((FixedPoint) column).getByteSize(); - if (byteSize != null && byteSize <= Integer.SIZE) { + if (SqoopIDFUtils.isInteger(column)) { return Schema.create(Schema.Type.INT); } else { return Schema.create(Schema.Type.LONG); } case FLOATING_POINT: byteSize = ((FloatingPoint) column).getByteSize(); - if (byteSize != null && byteSize <= Float.SIZE) { + if (byteSize != null && byteSize <= (Float.SIZE/Byte.SIZE)) { return Schema.create(Schema.Type.FLOAT); } else { return Schema.create(Schema.Type.DOUBLE); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java index 8630369..32f6a86 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/common/SqoopIDFUtils.java @@ -97,9 +97,21 @@ public class SqoopIDFUtils { // ******** Number Column Type utils*********** - public static String toCSVFixedPoint(Object obj, Column column) { + public static boolean isInteger(Column column) { Long byteSize = ((FixedPoint) column).getByteSize(); - if (byteSize != null && byteSize <= Integer.SIZE) { + Boolean signed = ((FixedPoint) column).isSigned(); + + if (byteSize == null) { + return false; + } + if (signed != null && !signed) { + byteSize *= 2; + } + return byteSize <= (Integer.SIZE / Byte.SIZE); + } + + public static String toCSVFixedPoint(Object obj, Column column) { + if (isInteger(column)) { return ((Integer) obj).toString(); } else { return ((Long) obj).toString(); @@ -108,8 +120,7 @@ public class SqoopIDFUtils { public static Object toFixedPoint(String csvString, Column column) { Object returnValue; - Long byteSize = ((FixedPoint) column).getByteSize(); - if (byteSize != null && byteSize <= Integer.SIZE) { + if (isInteger(column)) { returnValue = Integer.valueOf(csvString); } else { returnValue = Long.valueOf(csvString); @@ -119,7 +130,7 @@ public class SqoopIDFUtils { public static String toCSVFloatingPoint(Object obj, Column column) { Long byteSize = ((FloatingPoint) column).getByteSize(); - if (byteSize != null && byteSize <= Float.SIZE) { + if (byteSize != null && byteSize <= (Float.SIZE/Byte.SIZE)) { return ((Float) obj).toString(); } else { return ((Double) obj).toString(); @@ -129,7 +140,7 @@ public class SqoopIDFUtils { public static Object toFloatingPoint(String csvString, Column column) { Object returnValue; Long byteSize = ((FloatingPoint) column).getByteSize(); - if (byteSize != null && byteSize <= Float.SIZE) { + if (byteSize != null && byteSize <= (Float.SIZE/Byte.SIZE)) { returnValue = Float.valueOf(csvString); } else { returnValue = Double.valueOf(csvString); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java index 1cef714..63c9535 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/common/TestSqoopIDFUtils.java @@ -20,8 +20,12 @@ package org.apache.sqoop.connector.common; import static org.testng.Assert.*; import static org.apache.sqoop.connector.common.SqoopIDFUtils.*; +import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.schema.type.AbstractComplexListType; import org.apache.sqoop.schema.type.Array; +import org.apache.sqoop.schema.type.Column; +import org.apache.sqoop.schema.type.FixedPoint; +import org.apache.sqoop.schema.type.FloatingPoint; import org.apache.sqoop.schema.type.Text; import org.testng.annotations.Test; @@ -35,7 +39,8 @@ public class TestSqoopIDFUtils { public static String getByteFieldString(byte[] byteFieldData) { try { - return new StringBuilder("'").append(new String(byteFieldData, BYTE_FIELD_CHARSET)).append("'").toString(); + return new StringBuilder("'").append(new String(byteFieldData, BYTE_FIELD_CHARSET)) + .append("'").toString(); } catch (UnsupportedEncodingException e) { // Should never get to this point because ISO-8859-1 is a standard codec. return null; @@ -115,12 +120,14 @@ public class TestSqoopIDFUtils { assertEquals(toString, expectedString); } + @Test public void testExample5EncodeToCSVString() { String test = new String(new char[] { 0x0A }); String encodedText = toCSVString(test); assertEquals(encodedText, "'\\n'"); } + @Test public void testExample5ToString() { String test = "'\\n'"; String expectedString = new String(new char[] { 0x0A }); @@ -128,6 +135,7 @@ public class TestSqoopIDFUtils { assertEquals(toString, expectedString); } + @Test public void testExample6EncodeToCSVString() { String test = new String(new char[] { 0x0D }); String encodedText = toCSVString(test); @@ -135,6 +143,101 @@ public class TestSqoopIDFUtils { } @Test + public void testToCSVFixedPointWithIntSignedAsInteger() { + Column col = new FixedPoint("ft", 2L, true); + String encodedText = toCSVFixedPoint(1, col); + assertTrue(Integer.valueOf(encodedText) instanceof Integer); + } + + @Test + public void testToCSVFixedPointWithIntSize4SignedAsInteger() { + Column col = new FixedPoint("ft", 4L, true); + String encodedText = toCSVFixedPoint(1, col); + assertTrue(Integer.valueOf(encodedText) instanceof Integer); + } + + @Test(expectedExceptions = Exception.class) + public void testToCSVFixedPointWithLongSignedAsInteger() { + Column col = new FixedPoint("ft", 4L, true); + Long test = 459999999444L; + String encodedText = toCSVFixedPoint(test, col); + // should be a long + assertTrue(Integer.valueOf(encodedText) instanceof Integer); + } + + @Test + public void testToCSVFixedPointWithIntSize2UnSignedAsInteger() { + Column col = new FixedPoint("ft", 2L, false); + Integer test = 45999999; + String encodedText = toCSVFixedPoint(test, col); + assertTrue(Integer.valueOf(encodedText) instanceof Integer); + } + + @Test + public void testToCSVFixedPointWithIntSize16UnSignedAsLong() { + Column col = new FixedPoint("ft", 16L, false); + Long test = 1000000000L; + String encodedText = toCSVFixedPoint(test, col); + assertTrue(Long.valueOf(encodedText) instanceof Long); + } + + @Test + public void testToCSVFixedPointWithIntUnSignedAsLong() { + Column col = new FixedPoint("ft", 4L, false); + // java does not have a concept of unsigned int, so it has to be a long for + // testing + long test = 100000000900000L; + String encodedText = toCSVFixedPoint(test, col); + assertTrue(Long.valueOf(encodedText) instanceof Long); + } + + @Test + public void testToFixedPointReturnsInt() { + Column col = new FixedPoint("fixt", 4L, true); + assertTrue(toFixedPoint("233", col) instanceof Integer); + } + + @Test + public void testToFixedPointReturnsLong() { + Column col = new FixedPoint("fixt", 8L, true); + assertTrue(toFixedPoint("233", col) instanceof Long); + } + + @Test + public void testToFixedPointUnsignedReturnsLong() { + Column col = new FixedPoint("fixt", 4L, false); + assertTrue(toFixedPoint("2333333333333333", col) instanceof Long); + } + + @Test + public void testToCSVFloatingPointAsFloat() { + Column col = new FloatingPoint("ft", 2L); + Float test = 2.3F; + String encodedText = toCSVFloatingPoint(test, col); + assertTrue(Float.valueOf(encodedText) instanceof Float); + } + + @Test + public void testToFloatingPointReturnsFloat() { + Column col = new FloatingPoint("ft", 4L); + assertTrue(toFloatingPoint("2.33", col) instanceof Float); + } + + @Test + public void testToCSVFloatingPointAsDouble() { + Column col = new FloatingPoint("ft", 5L); + Double test = 2.3D; + String encodedText = toCSVFloatingPoint(test, col); + assertTrue(Double.valueOf(encodedText) instanceof Double); + } + + @Test + public void testToFloatingPointReturnsDouble() { + Column col = new FloatingPoint("ft", 8L); + assertTrue(toFloatingPoint("2.33", col) instanceof Double); + } + + @Test public void testEncodeByteToCSVString() { // byte[0] = \r byte[1] = -112, byte[1] = 54 - 2's complements byte[] bytes = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54 }; @@ -170,23 +273,22 @@ public class TestSqoopIDFUtils { list.add("B"); Map<Object, Object> map = new HashMap<Object, Object>(); map.put("A", list); - org.apache.sqoop.schema.type.Map mapCol = new org.apache.sqoop.schema.type.Map("a", new Text("t"), new Array("r", new Text( - "tr"))); + org.apache.sqoop.schema.type.Map mapCol = new org.apache.sqoop.schema.type.Map("a", new Text( + "t"), new Array("r", new Text("tr"))); String encodedText = toCSVMap(map, mapCol); assertEquals(encodedText, "'{\"A\":[\"A\",\"B\"]}'"); } - + @Test public void testParseCSVString() { - String csv= "'hello, world','34',45"; + String csv = "'hello, world','34',45"; String[] arr = parseCSVString(csv); assertEquals(arr.length, 3); assertEquals(arr[0], "'hello, world'"); assertEquals(arr[1], "'34'"); assertEquals(arr[2], "45"); - - } + } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java index dd7a3e1..b9a5ea1 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java @@ -79,22 +79,22 @@ public class TestAVROIntermediateDataFormat { options.add("NUME"); enumCol = new org.apache.sqoop.schema.type.Enum("seven").setOptions(options); sqoopSchema - .addColumn(new FixedPoint("one")) - .addColumn(new FixedPoint("two", 2L, false)) + .addColumn(new FixedPoint("one", 8L, true)) + .addColumn(new FixedPoint("two", 2L, true)) .addColumn(new Text("three")) .addColumn(new Text("four")) .addColumn(new Binary("five")) .addColumn(new Text("six")) .addColumn(enumCol) - .addColumn(new Array("eight", new Array("array", new FixedPoint("ft")))) + .addColumn(new Array("eight", new Array("array", new FixedPoint("ft",2L, false)))) .addColumn(new org.apache.sqoop.schema.type.Map("nine", new Text("t1"), new Text("t2"))) .addColumn(new Bit("ten")) .addColumn(new org.apache.sqoop.schema.type.DateTime("eleven", true, false)) .addColumn(new org.apache.sqoop.schema.type.Time("twelve", false)) .addColumn(new org.apache.sqoop.schema.type.Date("thirteen")) - .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("fourteen")) + .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("fourteen", 4L)) .addColumn( - new org.apache.sqoop.schema.type.Set("fifteen", new Array("set", new FixedPoint("ftw")))); + new org.apache.sqoop.schema.type.Set("fifteen", new Array("set", new FixedPoint("ftw", 2L, false)))); dataFormat = new AVROIntermediateDataFormat(sqoopSchema); avroSchema = SqoopAvroUtils.createAvroSchema(sqoopSchema); } @@ -165,7 +165,7 @@ public class TestAVROIntermediateDataFormat { assertEquals(dateTime, out[10]); assertEquals(time, out[11]); assertEquals(date, out[12]); - assertEquals(13.44, out[13]); + assertEquals(13.44f, out[13]); assertEquals(set.length, 2); assertEquals(Arrays.deepToString(set), Arrays.deepToString((Object[]) out[14])); @@ -215,7 +215,7 @@ public class TestAVROIntermediateDataFormat { avroObject.put("eleven", dateTime.toDate().getTime()); avroObject.put("twelve", time.toDateTimeToday().getMillis()); avroObject.put("thirteen", date.toDate().getTime()); - avroObject.put("fourteen", 13.44); + avroObject.put("fourteen", 13.44f); List<Object> givenSetOne = new ArrayList<Object>(); givenSetOne.add(11); givenSetOne.add(12); @@ -288,7 +288,7 @@ public class TestAVROIntermediateDataFormat { out[11] = time; out[12] = date; - out[13] = 13.44; + out[13] = 13.44f; Object[] set0 = new Object[2]; set0[0] = 11; set0[1] = 12; http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java index 2556711..cbbbd44 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java @@ -43,6 +43,7 @@ import org.apache.sqoop.schema.type.Date; import org.apache.sqoop.schema.type.DateTime; import org.apache.sqoop.schema.type.Decimal; import org.apache.sqoop.schema.type.FixedPoint; +import org.apache.sqoop.schema.type.FloatingPoint; import org.apache.sqoop.schema.type.Text; import org.apache.sqoop.schema.type.Time; import org.joda.time.DateTimeZone; @@ -63,8 +64,8 @@ public class TestCSVIntermediateDataFormat { @Test public void testNullInputAsCSVTextInObjectArrayOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")) - .addColumn(new FixedPoint("2")) + schema.addColumn(new FixedPoint("1", 2L, true)) + .addColumn(new FixedPoint("2", 4L, false)) .addColumn(new Text("3")) .addColumn(new Text("4")) .addColumn(new Binary("5")) @@ -78,7 +79,7 @@ public class TestCSVIntermediateDataFormat { @Test(expectedExceptions = SqoopException.class) public void testEmptyInputAsCSVTextInObjectArrayOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")).addColumn(new FixedPoint("2")).addColumn(new Text("3")).addColumn(new Text("4")) + schema.addColumn(new FixedPoint("1", 2L, false)).addColumn(new FixedPoint("2", 2L, false)).addColumn(new Text("3")).addColumn(new Text("4")) .addColumn(new Binary("5")).addColumn(new Text("6")); dataFormat = new CSVIntermediateDataFormat(schema); dataFormat.setCSVTextData(""); @@ -88,12 +89,12 @@ public class TestCSVIntermediateDataFormat { @Test public void testNullValueAsObjectArrayInAndCSVTextOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")).addColumn(new Decimal("2")).addColumn(new Text("3")) + schema.addColumn(new FixedPoint("1", 2L, false)).addColumn(new Decimal("2")).addColumn(new Text("3")) .addColumn(new Array("4", new Text("t"))).addColumn(new Binary("5")) .addColumn(new org.apache.sqoop.schema.type.Map("6", new Text("t1"), new Text("t2"))).addColumn(new Bit("7")) .addColumn(new org.apache.sqoop.schema.type.DateTime("8", false, false)) .addColumn(new org.apache.sqoop.schema.type.Time("9", false)).addColumn(new org.apache.sqoop.schema.type.Date("10")) - .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11")) + .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11", 4L)) .addColumn(new org.apache.sqoop.schema.type.Set("12", new Text("t4"))) .addColumn(new org.apache.sqoop.schema.type.Enum("13")).addColumn(new org.apache.sqoop.schema.type.Unknown("14")); @@ -112,12 +113,12 @@ public class TestCSVIntermediateDataFormat { @Test public void testNullValueAsObjectArrayInAndObjectArrayOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")).addColumn(new Decimal("2")).addColumn(new Text("3")) + schema.addColumn(new FixedPoint("1", 2L, true)).addColumn(new Decimal("2")).addColumn(new Text("3")) .addColumn(new Array("4", new Text("t"))).addColumn(new Binary("5")) .addColumn(new org.apache.sqoop.schema.type.Map("6", new Text("t1"), new Text("t2"))).addColumn(new Bit("7")) .addColumn(new org.apache.sqoop.schema.type.DateTime("8", false, false)) .addColumn(new org.apache.sqoop.schema.type.Time("9", false)).addColumn(new org.apache.sqoop.schema.type.Date("10")) - .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11")) + .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11", 4L)) .addColumn(new org.apache.sqoop.schema.type.Set("12", new Text("t4"))) .addColumn(new org.apache.sqoop.schema.type.Enum("13")).addColumn(new org.apache.sqoop.schema.type.Unknown("14")); @@ -135,12 +136,12 @@ public class TestCSVIntermediateDataFormat { @Test public void testNullValueAsCSVTextInAndObjectArrayOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")).addColumn(new Decimal("2")).addColumn(new Text("3")) + schema.addColumn(new FixedPoint("1", 2L, true)).addColumn(new Decimal("2")).addColumn(new Text("3")) .addColumn(new Array("4", new Text("t"))).addColumn(new Binary("5")) .addColumn(new org.apache.sqoop.schema.type.Map("6", new Text("t1"), new Text("t2"))).addColumn(new Bit("7")) .addColumn(new org.apache.sqoop.schema.type.DateTime("8", false, false)) .addColumn(new org.apache.sqoop.schema.type.Time("9", false)).addColumn(new org.apache.sqoop.schema.type.Date("10")) - .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11")) + .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11", 4L)) .addColumn(new org.apache.sqoop.schema.type.Set("12", new Text("t4"))) .addColumn(new org.apache.sqoop.schema.type.Enum("13")).addColumn(new org.apache.sqoop.schema.type.Unknown("14")); @@ -158,12 +159,12 @@ public class TestCSVIntermediateDataFormat { @Test public void testNullValueAsCSVTextInAndCSVTextOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")).addColumn(new Decimal("2")).addColumn(new Text("3")) + schema.addColumn(new FixedPoint("1", 2L, true)).addColumn(new Decimal("2")).addColumn(new Text("3")) .addColumn(new Array("4", new Text("t"))).addColumn(new Binary("5")) .addColumn(new org.apache.sqoop.schema.type.Map("6", new Text("t1"), new Text("t2"))).addColumn(new Bit("7")) .addColumn(new org.apache.sqoop.schema.type.DateTime("8", false, false)) .addColumn(new org.apache.sqoop.schema.type.Time("9", false)).addColumn(new org.apache.sqoop.schema.type.Date("10")) - .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11")) + .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("11", 4L)) .addColumn(new org.apache.sqoop.schema.type.Set("12", new Text("t4"))) .addColumn(new org.apache.sqoop.schema.type.Enum("13")).addColumn(new org.apache.sqoop.schema.type.Unknown("14")); @@ -184,13 +185,13 @@ public class TestCSVIntermediateDataFormat { @Test public void testInputAsCSVTextInCSVTextOut() { Schema schema = new Schema("test"); - schema.addColumn(new org.apache.sqoop.schema.type.Enum("1")).addColumn(new FixedPoint("2")) - .addColumn(new FixedPoint("3")).addColumn(new Text("4")).addColumn(new Text("5")) - .addColumn(new Binary("6")).addColumn(new Text("7")); + schema.addColumn(new org.apache.sqoop.schema.type.Enum("1")).addColumn(new FixedPoint("2", 2L, false)) + .addColumn(new FixedPoint("3", 2L, false)).addColumn(new Text("4")).addColumn(new Text("5")) + .addColumn(new Binary("6")).addColumn(new Text("7")).addColumn(new FloatingPoint("ft", 4L)); String testData = "'ENUM',10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) - + "'"; + + "',13.44"; dataFormat = new CSVIntermediateDataFormat(schema); dataFormat.setCSVTextData(testData); assertEquals(testData, dataFormat.getCSVTextData()); @@ -199,13 +200,13 @@ public class TestCSVIntermediateDataFormat { @Test public void testInputAsCSVTextInAndDataOut() { Schema schema = new Schema("test"); - schema.addColumn(new org.apache.sqoop.schema.type.Enum("1")).addColumn(new FixedPoint("2")) - .addColumn(new FixedPoint("3")).addColumn(new Text("4")).addColumn(new Text("5")) - .addColumn(new Binary("6")).addColumn(new Text("7")); + schema.addColumn(new org.apache.sqoop.schema.type.Enum("1")).addColumn(new FixedPoint("2", 2L, false)) + .addColumn(new FixedPoint("3", 2L, false)).addColumn(new Text("4")).addColumn(new Text("5")) + .addColumn(new Binary("6")).addColumn(new Text("7")).addColumn(new FloatingPoint("ft", 4L)); String testData = "'ENUM',10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) - + "'"; + + "',13.44"; dataFormat = new CSVIntermediateDataFormat(schema); dataFormat.setCSVTextData(testData); assertEquals(testData, dataFormat.getData()); @@ -230,36 +231,39 @@ public class TestCSVIntermediateDataFormat { //byte[0] = -112, byte[1] = 54 - 2's complements String testData = "10,34,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54}) - + ",'\\n','TEST_ENUM'"; + + ",'\\n','TEST_ENUM',13.44"; Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")) - .addColumn(new FixedPoint("2")) + schema.addColumn(new FixedPoint("1", 8L, true)) + .addColumn(new FixedPoint("2", 4L, true)) .addColumn(new Text("3")) .addColumn(new Text("4")) .addColumn(new Binary("5")) .addColumn(new Text("6")) - .addColumn(new org.apache.sqoop.schema.type.Enum("7")); - + .addColumn(new org.apache.sqoop.schema.type.Enum("7")) + .addColumn(new FloatingPoint("ft", 2L)); dataFormat = new CSVIntermediateDataFormat(schema); dataFormat.setCSVTextData(testData); Object[] out = dataFormat.getObjectData(); - assertEquals(new Long(10),out[0]); - assertEquals(new Long(34),out[1]); + // long + assertEquals(10L,out[0]); + // integer + assertEquals(34,out[1]); assertEquals("54",out[2]); assertEquals("random data",out[3]); assertEquals(-112, ((byte[]) out[4])[0]); assertEquals(54, ((byte[])out[4])[1]); assertEquals("\n", out[5].toString()); assertEquals("TEST_ENUM", out[6].toString()); + assertEquals(13.44f, out[7]); } @Test public void testInputAsObjectArayInCSVTextOrDataOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")) - .addColumn(new FixedPoint("2")) + schema.addColumn(new FixedPoint("1", 8L, true)) + .addColumn(new FixedPoint("2", 4L, true)) .addColumn(new Text("3")) .addColumn(new Text("4")) .addColumn(new Binary("5")) @@ -270,7 +274,7 @@ public class TestCSVIntermediateDataFormat { byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54}; Object[] in = new Object[7]; in[0] = new Long(10); - in[1] = new Long(34); + in[1] = new Integer(34); in[2] = "54"; in[3] = "random data"; in[4] = byteFieldData; @@ -291,8 +295,8 @@ public class TestCSVIntermediateDataFormat { //Test escapable sequences too. //byte[0] = -112, byte[1] = 54 - 2's complements Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")) - .addColumn(new FixedPoint("2")) + schema.addColumn(new FixedPoint("1", 8L, true)) + .addColumn(new FixedPoint("2", 2L, true)) .addColumn(new Text("3")) .addColumn(new Text("4")) .addColumn(new Binary("5")) @@ -303,7 +307,7 @@ public class TestCSVIntermediateDataFormat { Object[] in = new Object[7]; in[0] = new Long(10); - in[1] = new Long(34); + in[1] = new Integer(34); in[2] = "54"; in[3] = "random data"; in[4] = new byte[] { (byte) -112, (byte) 54}; @@ -321,8 +325,8 @@ public class TestCSVIntermediateDataFormat { @Test public void testObjectArrayWithNullInCSVTextOut() { Schema schema = new Schema("test"); - schema.addColumn(new FixedPoint("1")) - .addColumn(new FixedPoint("2")) + schema.addColumn(new FixedPoint("1", 8L, true)) + .addColumn(new FixedPoint("2", 2L, true)) .addColumn(new Text("3")) .addColumn(new Text("4")) .addColumn(new Binary("5")) @@ -334,7 +338,7 @@ public class TestCSVIntermediateDataFormat { byte[] byteFieldData = new byte[] { (byte) 0x0D, (byte) -112, (byte) 54}; Object[] in = new Object[7]; in[0] = new Long(10); - in[1] = new Long(34); + in[1] = new Integer(34); in[2] = null; in[3] = "random data"; in[4] = byteFieldData; @@ -800,7 +804,7 @@ public class TestCSVIntermediateDataFormat { @Test public void testArrayOfIntegers() { Schema schema = new Schema("test"); - schema.addColumn(new org.apache.sqoop.schema.type.Array("1", new FixedPoint("fn"))); + schema.addColumn(new org.apache.sqoop.schema.type.Array("1", new FixedPoint("fn", 2L, false))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); Object[] givenArray = { 1, 2 }; @@ -817,7 +821,7 @@ public class TestCSVIntermediateDataFormat { @Test public void testListOfIntegers() { Schema schema = new Schema("test"); - schema.addColumn(new org.apache.sqoop.schema.type.Array("1", new FixedPoint("fn"))); + schema.addColumn(new org.apache.sqoop.schema.type.Array("1", new FixedPoint("fn", 2L, false))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); List<Integer> givenList = new ArrayList<Integer>(); @@ -835,7 +839,7 @@ public class TestCSVIntermediateDataFormat { public void testSetOfIntegers() { Schema schema = new Schema("test"); - schema.addColumn(new org.apache.sqoop.schema.type.Set("1", new FixedPoint("fn"))); + schema.addColumn(new org.apache.sqoop.schema.type.Set("1", new FixedPoint("fn", 2L, false))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); Set<Integer> givenSet = new HashSet<Integer>(); @@ -873,7 +877,7 @@ public class TestCSVIntermediateDataFormat { public void testArrayOfObjectsWithObjectArrayInObjectArrayOut() { Schema schema = new Schema("test"); schema.addColumn(new org.apache.sqoop.schema.type.Array("1", - new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft")))); + new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft", 2L, false)))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); Object[] givenArrayOne = { 11, 12 }; @@ -899,7 +903,7 @@ public class TestCSVIntermediateDataFormat { public void testArrayOfObjectsWithCSVTextInObjectArrayOut() { Schema schema = new Schema("test"); schema.addColumn(new org.apache.sqoop.schema.type.Array("1", - new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft")))); + new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft", 2L, false)))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); Object[] givenArrayOne = { 11, 12 }; @@ -924,7 +928,7 @@ public class TestCSVIntermediateDataFormat { public void testArrayOfObjectsWithCSVTextInCSVTextOut() { Schema schema = new Schema("test"); schema.addColumn(new org.apache.sqoop.schema.type.Array("1", - new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft")))); + new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft", 2L, false)))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); String input = "'[[11, 12],[14, 15]]','text'"; @@ -938,7 +942,7 @@ public class TestCSVIntermediateDataFormat { public void testArrayOfObjectsWithObjectArrayInCSVTextOut() { Schema schema = new Schema("test"); schema.addColumn(new org.apache.sqoop.schema.type.Array("1", - new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft")))); + new org.apache.sqoop.schema.type.Array("array", new FixedPoint("ft", 2L, false)))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); Object[] givenArrayOne = { 11, 12 }; @@ -981,7 +985,7 @@ public class TestCSVIntermediateDataFormat { public void testMapWithComplexIntegerListValueWithObjectArrayInObjectArrayOut() { Schema schema = new Schema("test"); schema.addColumn(new org.apache.sqoop.schema.type.Map("1", new Text("key"), new Array("value", - new FixedPoint("number")))); + new FixedPoint("number", 2L, true)))); schema.addColumn(new org.apache.sqoop.schema.type.Text("2")); dataFormat = new CSVIntermediateDataFormat(schema); Map<Object, Object> givenMap = new HashMap<Object, Object>(); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestJSONIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestJSONIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestJSONIntermediateDataFormat.java index 12c66c2..0d9a1cf 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestJSONIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestJSONIntermediateDataFormat.java @@ -60,22 +60,22 @@ public class TestJSONIntermediateDataFormat { private void createJSONIDF() { Schema schema = new Schema("test"); schema - .addColumn(new FixedPoint("1")) - .addColumn(new FixedPoint("2", 2L, false)) + .addColumn(new FixedPoint("1", 8L, true)) + .addColumn(new FixedPoint("2", 4L, true)) .addColumn(new Text("3")) .addColumn(new Text("4")) .addColumn(new Binary("5")) .addColumn(new Text("6")) .addColumn(new org.apache.sqoop.schema.type.Enum("7")) - .addColumn(new Array("8", new Array("array", new FixedPoint("ft")))) + .addColumn(new Array("8", new Array("array", new FixedPoint("ft", 2L, false)))) .addColumn(new org.apache.sqoop.schema.type.Map("9", new Text("t1"), new Text("t2"))) .addColumn(new Bit("10")) .addColumn(new org.apache.sqoop.schema.type.DateTime("11", true, false)) .addColumn(new org.apache.sqoop.schema.type.Time("12", false)) .addColumn(new org.apache.sqoop.schema.type.Date("13")) - .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("14")) + .addColumn(new org.apache.sqoop.schema.type.FloatingPoint("14", 4L)) .addColumn( - new org.apache.sqoop.schema.type.Set("15", new Array("set", new FixedPoint("ftw")))); + new org.apache.sqoop.schema.type.Set("15", new Array("set", new FixedPoint("ftw", 2L, false)))); dataFormat = new JSONIntermediateDataFormat(schema); } @@ -151,7 +151,7 @@ public class TestJSONIntermediateDataFormat { assertEquals(dateTime, out[10]); assertEquals(time, out[11]); assertEquals(date, out[12]); - assertEquals(13.44, out[13]); + assertEquals(13.44f, out[13]); assertEquals(set.length, 2); assertEquals(Arrays.deepToString(set), Arrays.deepToString((Object[]) out[14])); @@ -202,7 +202,7 @@ public class TestJSONIntermediateDataFormat { json.put("11", SqoopIDFUtils.removeQuotes(dateTime)); json.put("12", SqoopIDFUtils.removeQuotes(time)); json.put("13", SqoopIDFUtils.removeQuotes(date)); - json.put("14", 13.44); + json.put("14", 13.44f); JSONArray givenSetOne = new JSONArray(); givenSetOne.add(11); @@ -282,7 +282,7 @@ public class TestJSONIntermediateDataFormat { out[12] = date; - out[13] = 13.44; + out[13] = 13.44f; Object[] set0 = new Object[2]; set0[0] = 11; set0[1] = 12; http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java index 4945584..f658d09 100644 --- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java +++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/TestMatching.java @@ -88,12 +88,12 @@ public class TestMatching { Schema from2 = new Schema("FROM-2"); Schema to2 = new Schema("TO-2"); - from1.addColumn(new FixedPoint("1").setByteSize(2L)).addColumn(new FloatingPoint("2")) + from1.addColumn(new FixedPoint("1", 4L, true)).addColumn(new FloatingPoint("2", 8L)) .addColumn(new org.apache.sqoop.schema.type.Text("3")); - to1.addColumn(new FixedPoint("1").setByteSize(2L)).addColumn(new FloatingPoint("2")) - .addColumn(new org.apache.sqoop.schema.type.Text("3")); - from2.addColumn(new FixedPoint("1").setByteSize(2L)).addColumn(new FloatingPoint("2")); - to2.addColumn(new FixedPoint("1").setByteSize(2L)).addColumn(new FloatingPoint("2")); + to1.addColumn(new FixedPoint("1", 4L, true)).addColumn(new FloatingPoint("2", 8L)) + .addColumn(new org.apache.sqoop.schema.type.Text("3")); + from2.addColumn(new FixedPoint("1", 4L, true)).addColumn(new FloatingPoint("2", 8L)); + to2.addColumn(new FixedPoint("1", 4L, true)).addColumn(new FloatingPoint("2", 8L)); parameters.add(new Object[]{ emptyFrom, http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ac3c386/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java b/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java index ce39a78..92ad1b8 100644 --- a/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java +++ b/execution/mapreduce/src/test/java/org/apache/sqoop/job/util/MRJobTestUtil.java @@ -70,7 +70,7 @@ public class MRJobTestUtil { public static Schema getTestSchema() { Schema schema = new Schema("Test"); - schema.addColumn(new FixedPoint("1").setByteSize(2L)).addColumn(new FloatingPoint("2")) + schema.addColumn(new FixedPoint("1", 4L, true)).addColumn(new FloatingPoint("2", 8L)) .addColumn(new Text("3")); return schema; }
