This is an automated email from the ASF dual-hosted git repository.
mgaido pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-livy.git
The following commit(s) were added to refs/heads/master by this push:
new 7847c3f [LIVY-699][THRIFT] Fix resultSet.getBigDecimal throw
java.sql.SQLException: Illegal conversion
7847c3f is described below
commit 7847c3fe8309dfd2d2089dac68044603745a3499
Author: runzhiwang <runzhiw...@tencent.com>
AuthorDate: Tue Nov 12 11:26:48 2019 +0100
[LIVY-699][THRIFT] Fix resultSet.getBigDecimal throw java.sql.SQLException:
Illegal conversion
## What changes were proposed in this pull request?
[LIVY-699][THRIFT] Fix resultSet.getBigDecimal throw java.sql.SQLException:
Illegal conversion.
Follows are steps to reproduce the problem:
1. `create table test(id decimal)`.
2. Then `resultSet.getBigDecimal(1)` will throw:` java.sql.SQLException:
Illegal conversion`. The reason is
`getSchema().getColumnDescriptorAt(columnIndex - 1).getType();` at
https://github.com/apache/hive/blob/master/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java#L415
return string, so cannot pass the check `val instanceof BigDecimal `at
https://github.com/apache/hive/blob/master/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java#L133,
so throw `java.sql.SQLExceptio [...]
3. So the root cause is the error return of `getType()`, which should
return decimal other than string.
4. Regarding to date and timestamp, though the return type is string,
hive-jdbc has done the transformation from string to date and timestamp in the
following links. But I think it is necessary to return the right type.
https://github.com/apache/hive/blob/master/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java#L255
https://github.com/apache/hive/blob/master/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java#L571
Additionally, SparkThrift return decimal type instead of string in the same
case, if user use `getBigDecimal` and migrate from SparkThrift to Livy, it will
throw exception. So it is necessary to return decimal instead of string in livy.
## How was this patch tested?
Add `SchemaIT.scala` to test the return of column type.
Author: runzhiwang <runzhiw...@tencent.com>
Closes #247 from runzhiwang/support-type.
---
.../apache/livy/thriftserver/types/Schema.scala | 6 +
.../livy/thriftserver/ThriftServerSuites.scala | 134 ++++++++++++++++++---
.../apache/livy/thriftserver/session/DataType.java | 3 +
3 files changed, 126 insertions(+), 17 deletions(-)
diff --git
a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
index 6e06474..0f4c642 100644
---
a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
+++
b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
@@ -41,6 +41,9 @@ case class BasicDataType(name: String) extends FieldType {
case "float" => DataType.FLOAT
case "double" => DataType.DOUBLE
case "binary" => DataType.BINARY
+ case _ if name.contains("decimal") => DataType.DECIMAL
+ case "timestamp" => DataType.TIMESTAMP
+ case "date" => DataType.DATE
case _ => DataType.STRING
}
}
@@ -100,6 +103,9 @@ object Schema {
case DataType.FLOAT => TTypeId.FLOAT_TYPE
case DataType.DOUBLE => TTypeId.DOUBLE_TYPE
case DataType.BINARY => TTypeId.BINARY_TYPE
+ case DataType.DECIMAL => TTypeId.DECIMAL_TYPE
+ case DataType.TIMESTAMP => TTypeId.TIMESTAMP_TYPE
+ case DataType.DATE => TTypeId.DATE_TYPE
case _ => TTypeId.STRING_TYPE
}
val primitiveEntry = new TPrimitiveTypeEntry(typeId)
diff --git
a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
index 1939a56..19abb0d 100644
---
a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
+++
b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
@@ -17,7 +17,7 @@
package org.apache.livy.thriftserver
-import java.sql.{Connection, Date, SQLException, Statement, Types}
+import java.sql.{Connection, Date, SQLException, Statement, Timestamp, Types}
import scala.collection.mutable.ArrayBuffer
import scala.io.Source
@@ -34,33 +34,133 @@ trait CommonThriftTests {
def dataTypesTest(statement: Statement, mapSupported: Boolean): Unit = {
val resultSet = statement.executeQuery(
- "select 1, 'a', cast(null as int), 1.2345, CAST('2018-08-06' as date), "
+
- "CAST('123' as BINARY)")
+ "select cast(1 as tinyint)," +
+ "cast(2 as smallint)," +
+ "cast(3 as int)," +
+ "cast(4 as bigint)," +
+ "cast(5.5 as float)," +
+ "cast(6.6 as double)," +
+ "cast(7.7 as decimal(10, 1))," +
+ "cast(true as boolean)," +
+ "cast('123' as binary)," +
+ "cast('string_val' as string)," +
+ "cast('varchar_val' as varchar(20))," +
+ "cast('char_val' as char(20))," +
+ "cast('2018-08-06 09:11:15' as timestamp)," +
+ "cast('2018-08-06' as date)")
+
+ val rsMetaData = resultSet.getMetaData()
+
resultSet.next()
- assert(resultSet.getInt(1) == 1)
- assert(resultSet.getString(2) == "a")
- assert(resultSet.getInt(3) == 0)
- assert(resultSet.wasNull())
- assert(resultSet.getDouble(4) == 1.2345)
- assert(resultSet.getDate(5) == Date.valueOf("2018-08-06"))
- val resultBytes = Source.fromInputStream(resultSet.getBinaryStream(6))
+
+ assert(resultSet.getByte(1) == 1)
+ assert(rsMetaData.getColumnTypeName(1) == "tinyint")
+
+ assert(resultSet.getShort(2) == 2)
+ assert(rsMetaData.getColumnTypeName(2) == "smallint")
+
+ assert(resultSet.getInt(3) == 3)
+ assert(rsMetaData.getColumnTypeName(3) == "int")
+
+ assert(resultSet.getLong(4) == 4)
+ assert(rsMetaData.getColumnTypeName(4) == "bigint")
+
+ assert(resultSet.getFloat(5) == 5.5)
+ assert(rsMetaData.getColumnTypeName(5) == "float")
+
+ assert(resultSet.getDouble(6) == 6.6)
+ assert(rsMetaData.getColumnTypeName(6) == "double")
+
+ assert(resultSet.getBigDecimal(7).doubleValue() == 7.7)
+ assert(rsMetaData.getColumnTypeName(7) == "decimal")
+
+ assert(resultSet.getBoolean(8) == true)
+ assert(rsMetaData.getColumnTypeName(8) == "boolean")
+
+ val resultBytes = Source.fromInputStream(resultSet.getBinaryStream(9))
.map(_.toByte).toArray
assert("123".getBytes.sameElements(resultBytes))
+ assert(rsMetaData.getColumnTypeName(9) == "binary")
+
+ assert(resultSet.getString(10) == "string_val")
+ assert(rsMetaData.getColumnTypeName(10) == "string")
+
+ assert(resultSet.getString(11) == "varchar_val")
+ assert(rsMetaData.getColumnTypeName(11) == "string")
+
+ assert(resultSet.getString(12) == "char_val")
+ assert(rsMetaData.getColumnTypeName(12) == "string")
+
+ assert(resultSet.getTimestamp(13).
+ compareTo(Timestamp.valueOf("2018-08-06 09:11:15")) == 0)
+ assert(rsMetaData.getColumnTypeName(13) == "timestamp")
+
+ assert(resultSet.getDate(14).
+ compareTo(Date.valueOf("2018-08-06")) == 0)
+ assert(rsMetaData.getColumnTypeName(14) == "date")
+
assert(!resultSet.next())
- val resultSetWithNulls = statement.executeQuery("select cast(null as
string), " +
- "cast(null as decimal), cast(null as double), cast(null as date), null")
+ val resultSetWithNulls = statement.executeQuery(
+ "select cast(null as tinyint), " +
+ "cast(null as smallint)," +
+ "cast(null as int)," +
+ "cast(null as bigint)," +
+ "cast(null as float)," +
+ "cast(null as double)," +
+ "cast(null as decimal)," +
+ "cast(null as boolean)," +
+ "cast(null as binary)," +
+ "cast(null as string)," +
+ "cast(null as varchar(20))," +
+ "cast(null as char(20))," +
+ "cast(null as timestamp)," +
+ "cast(null as date)")
+
resultSetWithNulls.next()
- assert(resultSetWithNulls.getString(1) == null)
+
+ assert(resultSetWithNulls.getByte(1) == 0)
assert(resultSetWithNulls.wasNull())
- assert(resultSetWithNulls.getBigDecimal(2) == null)
+
+ assert(resultSetWithNulls.getShort(2) == 0)
assert(resultSetWithNulls.wasNull())
- assert(resultSetWithNulls.getDouble(3) == 0.0)
+
+ assert(resultSetWithNulls.getInt(3) == 0)
assert(resultSetWithNulls.wasNull())
- assert(resultSetWithNulls.getDate(4) == null)
+
+ assert(resultSetWithNulls.getLong(4) == 0)
assert(resultSetWithNulls.wasNull())
- assert(resultSetWithNulls.getString(5) == null)
+
+ assert(resultSetWithNulls.getFloat(5) == 0.0)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getDouble(6) == 0.0)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getBigDecimal(7) == null)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getBoolean(8) == false)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getBinaryStream(9) == null)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getString(10) == null)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getString(11) == null)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getString(12) == null)
assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getTimestamp(13) == null)
+ assert(resultSetWithNulls.wasNull())
+
+ assert(resultSetWithNulls.getDate(14) == null)
+ assert(resultSetWithNulls.wasNull())
+
assert(!resultSetWithNulls.next())
val complexTypesQuery = if (mapSupported) {
diff --git
a/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/DataType.java
b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/DataType.java
index ab8f665..1812eaf 100644
---
a/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/DataType.java
+++
b/thriftserver/session/src/main/java/org/apache/livy/thriftserver/session/DataType.java
@@ -30,6 +30,9 @@ public enum DataType {
FLOAT,
DOUBLE,
BINARY,
+ DECIMAL,
+ TIMESTAMP,
+ DATE,
STRING;
}
