cloud-fan commented on a change in pull request #31281:
URL: https://github.com/apache/spark/pull/31281#discussion_r563491086
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
##########
@@ -37,31 +37,109 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
assert(CharVarcharUtils.getRawType(f.metadata) == Some(dt))
}
- test("char type values should be padded: top-level columns") {
+ test("char type values should be padded or trimmed: top-level columns") {
withTable("t") {
sql(s"CREATE TABLE t(i STRING, c CHAR(5)) USING $format")
- sql("INSERT INTO t VALUES ('1', 'a')")
- checkAnswer(spark.table("t"), Row("1", "a" + " " * 4))
- checkColType(spark.table("t").schema(1), CharType(5))
+ (0 to 5).map(n => "a" + " " * n).foreach { v =>
+ sql(s"INSERT OVERWRITE t VALUES ('1', '$v')")
+ checkAnswer(spark.table("t"), Row("1", "a" + " " * 4))
+ checkColType(spark.table("t").schema(1), CharType(5))
+ }
sql("INSERT OVERWRITE t VALUES ('1', null)")
checkAnswer(spark.table("t"), Row("1", null))
+
+ val e = intercept[SparkException](sql("INSERT OVERWRITE t VALUES ('1',
'abcdef')"))
+ assert(e.getCause.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
}
}
- test("char type values should be padded: partitioned columns") {
+ test("char type values should be padded or trimmed: partitioned columns") {
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c CHAR(5)) USING $format PARTITIONED BY
(c)")
+ (0 to 5).map(n => "a" + " " * n).foreach { v =>
+ sql(s"INSERT OVERWRITE t VALUES ('1', '$v')")
+ checkAnswer(spark.table("t"), Row("1", "a" + " " * 4))
+ checkColType(spark.table("t").schema(1), CharType(5))
+ }
+ val e1 = intercept[SparkException](sql("INSERT OVERWRITE t VALUES ('1',
'abcdef')"))
+ assert(e1.getCause.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
+ }
+
withTable("t") {
sql(s"CREATE TABLE t(i STRING, c CHAR(5)) USING $format PARTITIONED BY
(c)")
- sql("INSERT INTO t VALUES ('1', 'a')")
- checkAnswer(spark.table("t"), Row("1", "a" + " " * 4))
- checkColType(spark.table("t").schema(1), CharType(5))
+ (0 to 5).map(n => "a" + " " * n).foreach { v =>
+ sql(s"INSERT INTO t VALUES ('1', '$v')")
+ checkAnswer(spark.table("t"), Row("1", "a" + " " * 4))
+ sql(s"ALTER TABLE t DROP PARTITION(c='$v')")
+ checkAnswer(spark.table("t"), Nil)
+ }
+
+ val e2 = intercept[RuntimeException](sql("ALTER TABLE t DROP
PARTITION(c='abcdef')"))
+ assert(e2.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
+
+ val e3 = intercept[RuntimeException](sql("ALTER TABLE t ADD
PARTITION(c='abcdef')"))
+ assert(e3.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
+
+ sql("INSERT OVERWRITE t VALUES ('1', null)")
+ checkAnswer(spark.table("t"), Row("1", null))
+ }
+ }
- sql("ALTER TABLE t DROP PARTITION(c='a')")
+ test("varchar type values length check: partitioned columns dynamic") {
+ (0 to 5).foreach { n =>
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c VARCHAR(5)) USING $format PARTITIONED
BY (c)")
+ val v = "a" + " " * n
+ sql(s"INSERT INTO t VALUES ('1', '$v')")
+ checkAnswer(spark.table("t"), Row("1", "a" + " " * math.min(n, 4)))
+ checkColType(spark.table("t").schema(1), VarcharType(5))
+ }
+ }
+
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c VARCHAR(5)) USING $format PARTITIONED
BY (c)")
+ val e = intercept[SparkException](sql("INSERT OVERWRITE t VALUES ('1',
'abcdef')"))
+ assert(e.getCause.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
+ }
+
+ (0 to 5).foreach { n =>
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c VARCHAR(5)) USING $format PARTITIONED
BY (c)")
+ val v = "a" + " " * n
+ sql(s"INSERT INTO t VALUES ('1', '$v')")
+ checkAnswer(spark.table("t"), Row("1", "a" + " " * math.min(n, 4)))
+ sql(s"ALTER TABLE t DROP PARTITION(c='$v')")
+ checkAnswer(spark.table("t"), Nil)
+ }
+ }
+
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c VARCHAR(5)) USING $format PARTITIONED
BY (c)")
+ val e = intercept[RuntimeException](sql("ALTER TABLE t DROP
PARTITION(c='abcdef')"))
+ assert(e.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
sql("INSERT OVERWRITE t VALUES ('1', null)")
checkAnswer(spark.table("t"), Row("1", null))
}
}
+ test("varchar type values length check: partitioned columns of other types")
{
+ withTable("t") {
+ sql(s"CREATE TABLE t(i STRING, c VARCHAR(5)) USING $format PARTITIONED
BY (c)")
+ Seq(1, 10, 100, 1000, 10000).foreach { v =>
+ sql(s"INSERT OVERWRITE t VALUES ('1', $v)")
+ sql(s"ALTER TABLE t DROP PARTITION(c=$v)")
+ checkAnswer(spark.table("t"), Nil)
+ }
+
+ val e1 = intercept[SparkException](sql(s"INSERT OVERWRITE t VALUES ('1',
100000)"))
+ assert(e1.getCause.getMessage.contains("Exceeds char/varchar type length
limitation: 5"))
+
+ val e2 = intercept[RuntimeException](sql("ALTER TABLE t DROP
PARTITION(c='100000')"))
Review comment:
this is not "other types"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
