Repository: sqoop Updated Branches: refs/heads/sqoop2 947f53281 -> 213caf071
SQOOP-2229: Sqoop2: Kite connector should handle doubles (Abraham Elmahrek via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/213caf07 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/213caf07 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/213caf07 Branch: refs/heads/sqoop2 Commit: 213caf07146e53b4c4a841758800990b6fa535e3 Parents: 947f532 Author: Jarek Jarcec Cecho <[email protected]> Authored: Wed Apr 1 08:56:37 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Wed Apr 1 08:56:37 2015 -0700 ---------------------------------------------------------------------- .../sqoop/connector/kite/util/KiteDataTypeUtil.java | 10 +++++++++- .../connector/hive/FromRDBMSToKiteHiveTest.java | 7 +------ .../connector/kite/FromRDBMSToKiteTest.java | 14 +++++--------- 3 files changed, 15 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/213caf07/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java ---------------------------------------------------------------------- diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java index 1074d37..86aa44f 100644 --- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java +++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/util/KiteDataTypeUtil.java @@ -25,6 +25,7 @@ import org.apache.sqoop.connector.common.FileFormat; import org.apache.sqoop.schema.type.Column; import org.apache.sqoop.schema.type.ColumnType; import org.apache.sqoop.schema.type.FixedPoint; +import org.apache.sqoop.schema.type.FloatingPoint; import org.joda.time.LocalDate; import org.joda.time.LocalDateTime; import org.kitesdk.data.Format; @@ -142,7 +143,14 @@ public class KiteDataTypeUtil { } } case FLOATING_POINT: - return Schema.Type.DOUBLE; + FloatingPoint fp = (FloatingPoint)column; + if(fp.getByteSize() <= 4L) { + return Schema.Type.FLOAT; + } else if(fp.getByteSize() <= 8L) { + return Schema.Type.DOUBLE; + } else { + throw new IllegalArgumentException("Unsupported size of FloatingPoint column " + fp.getByteSize()); + } case MAP: return Schema.Type.MAP; case TEXT: http://git-wip-us.apache.org/repos/asf/sqoop/blob/213caf07/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java index 6e49312..a4a5b04 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java @@ -17,7 +17,6 @@ */ package org.apache.sqoop.integration.connector.hive; -import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.test.asserts.ProviderAsserts; import org.apache.sqoop.common.test.db.TableName; import org.apache.sqoop.connector.common.FileFormat; @@ -26,7 +25,6 @@ import org.apache.sqoop.model.MDriverConfig; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; import org.apache.sqoop.test.testcases.HiveConnectorTestCase; -import org.apache.sqoop.test.utils.HdfsUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -67,10 +65,8 @@ public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase { MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId()); // Set rdbms "FROM" config - MConfigList fromConfig = job.getFromJobConfig(); fillRdbmsFromConfig(job, "id"); - // TODO: Kite have troubles with some data types, so we're limiting the columns to int only - fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id")); + job.getFromJobConfig().getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id")); // Fill the Kite "TO" config MConfigList toConfig = job.getToJobConfig(); @@ -82,7 +78,6 @@ public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase { driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1); saveJob(job); - executeJob(job); // Assert correct output http://git-wip-us.apache.org/repos/asf/sqoop/blob/213caf07/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java index b3ec1a9..5c90501 100644 --- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java +++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java @@ -17,7 +17,6 @@ */ package org.apache.sqoop.integration.connector.kite; -import org.apache.sqoop.common.Direction; import org.apache.sqoop.connector.common.FileFormat; import org.apache.sqoop.model.MConfigList; import org.apache.sqoop.model.MDriverConfig; @@ -70,9 +69,7 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase { // Set rdbms "FROM" config fillRdbmsFromConfig(job, "id"); - // TODO: Kite have troubles with some data types, so we're limiting the columns to int only - MConfigList fromConfig = job.getFromJobConfig(); - fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id")); + job.getFromJobConfig().getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id")); // Fill the Kite "TO" config MConfigList toConfig = job.getToJobConfig(); @@ -84,15 +81,14 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase { driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1); saveJob(job); - executeJob(job); // Assert correct output assertTo( - "\"1\"", - "\"2\"", - "\"3\"", - "\"4\"" + "\"1\"", + "\"2\"", + "\"3\"", + "\"4\"" ); } }
