This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b540cc538614 [SPARK-47611][SQL] Cleanup dead code in
MySQLDialect.getCatalystType
b540cc538614 is described below
commit b540cc538614c9808dc5e83a339ff52917fa0f37
Author: Kent Yao <[email protected]>
AuthorDate: Wed Mar 27 01:45:22 2024 -0700
[SPARK-47611][SQL] Cleanup dead code in MySQLDialect.getCatalystType
### What changes were proposed in this pull request?
This PR removes an unnecessary case-match branch for Types.BIT in
MySQLDialect.getCatalystType, this is a special case for Maria Connector/J and
can be handled in defaults as we have matched and handled Types.BIT&size > 1
before this.
Additionally, we add some new tests for this corner case and other
MySQL/Maria quirks
### Why are the changes needed?
code refactoring and test improvement
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
new tests
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #45734 from yaooqinn/SPARK-47611.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/jdbc/MySQLIntegrationSuite.scala | 32 ++++++++++++++++++++--
.../org/apache/spark/sql/jdbc/MySQLDialect.scala | 2 --
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 2 --
3 files changed, 30 insertions(+), 6 deletions(-)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
index 705957631601..10049169caa1 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala
@@ -64,10 +64,11 @@ class MySQLIntegrationSuite extends
DockerJDBCIntegrationSuite {
conn.prepareStatement("CREATE TABLE unsigned_numbers (" +
"tiny TINYINT UNSIGNED, small SMALLINT UNSIGNED, med MEDIUMINT
UNSIGNED," +
"nor INT UNSIGNED, big BIGINT UNSIGNED, deci DECIMAL(40,20) UNSIGNED," +
- "dbl DOUBLE UNSIGNED)").executeUpdate()
+ "dbl DOUBLE UNSIGNED, tiny1u TINYINT(1) UNSIGNED)").executeUpdate()
conn.prepareStatement("INSERT INTO unsigned_numbers VALUES (255, 65535,
16777215, 4294967295," +
- "9223372036854775808, 123456789012345.123456789012345,
1.0000000000000002)").executeUpdate()
+ "9223372036854775808, 123456789012345.123456789012345,
1.0000000000000002, 0)")
+ .executeUpdate()
conn.prepareStatement("CREATE TABLE dates (d DATE, t TIME, dt DATETIME, ts
TIMESTAMP, "
+ "yr YEAR)").executeUpdate()
@@ -150,6 +151,13 @@ class MySQLIntegrationSuite extends
DockerJDBCIntegrationSuite {
assert(rows.get(4).isInstanceOf[BigDecimal])
assert(rows.get(5).isInstanceOf[BigDecimal])
assert(rows.get(6).isInstanceOf[Double])
+ // Unlike MySQL, MariaDB seems not to distinguish signed and unsigned
tinyint(1).
+ val isMaria = jdbcUrl.indexOf("disableMariaDbDriver") == -1
+ if (isMaria) {
+ assert(rows.get(7).isInstanceOf[Boolean])
+ } else {
+ assert(rows.get(7).isInstanceOf[Short])
+ }
assert(rows.getShort(0) === 255)
assert(rows.getInt(1) === 65535)
assert(rows.getInt(2) === 16777215)
@@ -157,6 +165,11 @@ class MySQLIntegrationSuite extends
DockerJDBCIntegrationSuite {
assert(rows.getAs[BigDecimal](4).equals(new
BigDecimal("9223372036854775808")))
assert(rows.getAs[BigDecimal](5).equals(new
BigDecimal("123456789012345.12345678901234500000")))
assert(rows.getDouble(6) === 1.0000000000000002)
+ if (isMaria) {
+ assert(rows.getBoolean(7) === false)
+ } else {
+ assert(rows.getShort(7) === 0)
+ }
}
test("Date types") {
@@ -260,6 +273,21 @@ class MySQLIntegrationSuite extends
DockerJDBCIntegrationSuite {
test("SPARK-47478: all boolean synonyms read-write roundtrip") {
val df = sqlContext.read.jdbc(jdbcUrl, "bools", new Properties)
checkAnswer(df, Row(true, true, true))
+
+ val properties0 = new Properties()
+ properties0.setProperty("transformedBitIsBoolean", "false")
+ properties0.setProperty("tinyInt1isBit", "true")
+
+ checkAnswer(spark.read.jdbc(jdbcUrl, "bools", properties0), Row(true,
true, true))
+ val properties1 = new Properties()
+ properties1.setProperty("transformedBitIsBoolean", "true")
+ properties1.setProperty("tinyInt1isBit", "true")
+ checkAnswer(spark.read.jdbc(jdbcUrl, "bools", properties1), Row(true,
true, true))
+
+ val properties2 = new Properties()
+ properties2.setProperty("tinyInt1isBit", "false")
+ checkAnswer(sqlContext.read.jdbc(jdbcUrl, "bools", properties2), Row(1,
true, 1))
+
df.write.mode("append").jdbc(jdbcUrl, "bools", new Properties)
checkAnswer(df, Seq(Row(true, true, true), Row(true, true, true)))
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
index 2735abfe9c39..3c238dc8a837 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
@@ -103,8 +103,6 @@ private case class MySQLDialect() extends JdbcDialect with
SQLConfHelper {
// MySQL connector behaviour
md.putLong("binarylong", 1)
Some(LongType)
- case Types.BIT if "TINYINT".equalsIgnoreCase(typeName) =>
- Some(BooleanType)
case Types.VARCHAR if "TINYTEXT".equalsIgnoreCase(typeName) =>
// TINYTEXT is Types.VARCHAR(63) from mysql jdbc, but keep it AS-IS
for historical reason
Some(StringType)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 016d1977f327..28e86e8ca1a5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -934,8 +934,6 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
Some(LongType))
assert(metadata.build().contains("binarylong"))
assert(mySqlDialect.getCatalystType(java.sql.Types.VARBINARY, "BIT", 1,
metadata) == None)
- assert(mySqlDialect.getCatalystType(java.sql.Types.BIT, "TINYINT", 1,
metadata) ==
- Some(BooleanType))
assert(mySqlDialect.getCatalystType(java.sql.Types.TINYINT, "TINYINT", 1,
metadata) ==
Some(ByteType))
assert(mySqlDialect.getCatalystType(java.sql.Types.REAL, "FLOAT", 1,
metadata) ===
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
