Repository: sqoop Updated Branches: refs/heads/sqoop2 e64598af4 -> edb42dbdc
SQOOP-2828: Sqoop2: AvroIntermediateDataFormat should read Decimals as BigDecimals instead of Strings (Abraham Fine via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/edb42dbd Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/edb42dbd Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/edb42dbd Branch: refs/heads/sqoop2 Commit: edb42dbdc29f3834a7bb4eea291e15c8fce77053 Parents: e64598a Author: Jarek Jarcec Cecho <[email protected]> Authored: Wed Feb 10 17:03:39 2016 -0800 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Wed Feb 10 17:03:39 2016 -0800 ---------------------------------------------------------------------- .../idf/AVROIntermediateDataFormat.java | 6 ++++-- .../idf/TestAVROIntermediateDataFormat.java | 22 +++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/edb42dbd/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java index e409fc1..b55f7a0 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/AVROIntermediateDataFormat.java @@ -352,7 +352,7 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe break; case DECIMAL: // stored as string - csvString.append(toCSVDecimal(obj)); + csvString.append(toCSVDecimal(new BigDecimal(obj.toString()))); break; case DATE: // stored as long @@ -417,9 +417,11 @@ public class AVROIntermediateDataFormat extends IntermediateDataFormat<GenericRe // stored as enum symbol case TEXT: // stored as UTF8 + object[nameIndex] = obj.toString(); + break; case DECIMAL: // stored as string - object[nameIndex] = obj.toString(); + object[nameIndex] = new BigDecimal(obj.toString()); break; case BINARY: case UNKNOWN: http://git-wip-us.apache.org/repos/asf/sqoop/blob/edb42dbd/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java index 8475720..3c4d7de 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestAVROIntermediateDataFormat.java @@ -35,12 +35,14 @@ import org.apache.sqoop.schema.type.Array; import org.apache.sqoop.schema.type.Binary; import org.apache.sqoop.schema.type.Bit; import org.apache.sqoop.schema.type.Column; +import org.apache.sqoop.schema.type.Decimal; import org.apache.sqoop.schema.type.FixedPoint; import org.apache.sqoop.schema.type.Text; import org.joda.time.LocalDateTime; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -80,7 +82,7 @@ public class TestAVROIntermediateDataFormat { enumCol = new org.apache.sqoop.schema.type.Enum("seven").setOptions(options); sqoopSchema .addColumn(new FixedPoint("one", 8L, true)) - .addColumn(new FixedPoint("two", 2L, true)) + .addColumn(new Decimal("two", 4, 2)) .addColumn(new Text("three")) .addColumn(new Text("four")) .addColumn(new Binary("five")) @@ -105,7 +107,7 @@ public class TestAVROIntermediateDataFormat { @Test public void testInputAsCSVTextInAndDataOut() { - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -116,7 +118,7 @@ public class TestAVROIntermediateDataFormat { @Test public void testInputAsCSVTextInAndObjectArrayOut() { - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -129,7 +131,7 @@ public class TestAVROIntermediateDataFormat { private void assertObjectArray() { Object[] out = dataFormat.getObjectData(); assertEquals(10L, out[0]); - assertEquals(34, out[1]); + assertEquals(new BigDecimal("34.56"), out[1]); assertEquals("54", out[2]); assertEquals("random data", out[3]); assertEquals(-112, ((byte[]) out[4])[0]); @@ -173,7 +175,7 @@ public class TestAVROIntermediateDataFormat { @Test public void testInputAsCSVTextInCSVTextOut() { - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -184,7 +186,7 @@ public class TestAVROIntermediateDataFormat { private GenericRecord createAvroGenericRecord() { GenericRecord avroObject = new GenericData.Record(avroSchema); avroObject.put("one", 10L); - avroObject.put("two", 34); + avroObject.put("two", "34.56"); avroObject.put("three", new Utf8("54")); avroObject.put("four", new Utf8("random data")); // store byte array in byte buffer @@ -235,7 +237,7 @@ public class TestAVROIntermediateDataFormat { @Test public void testInputAsDataInAndCSVOut() { - String csvExpected = "10,34,'54','random data'," + String csvExpected = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -260,7 +262,7 @@ public class TestAVROIntermediateDataFormat { private Object[] createObjectArray() { Object[] out = new Object[15]; out[0] = 10L; - out[1] = 34; + out[1] = new BigDecimal("34.56"); out[2] = "54"; out[3] = "random data"; out[4] = new byte[] { (byte) -112, (byte) 54 }; @@ -322,7 +324,7 @@ public class TestAVROIntermediateDataFormat { public void testInputAsObjectArrayInAndCSVOut() { Object[] out = createObjectArray(); dataFormat.setObjectData(out); - String csvText = "10,34,'54','random data'," + String csvText = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'" + String.valueOf(0x0A) + "','ENUM'," + csvArray + "," + map + ",true," + csvDateTime + "," + csvTime + "," + csvDate + ",13.44," + csvSet; @@ -339,7 +341,7 @@ public class TestAVROIntermediateDataFormat { // **************test cases for empty and null schema******************* @Test(expectedExceptions = SqoopException.class) public void testEmptySchema() { - String testData = "10,34,'54','random data'," + String testData = "10,34.56,'54','random data'," + getByteFieldString(new byte[] { (byte) -112, (byte) 54 }) + ",'\\n'"; // no coumns Schema schema = new Schema("Test");
