This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 742d9fa [SPARK-33480][SQL][FOLLOWUP] do not expose user data in error
message
742d9fa is described below
commit 742d9fa581ef612445730fe42d3887723e561cb1
Author: Wenchen Fan <wenc...@databricks.com>
AuthorDate: Mon Dec 7 13:35:37 2020 -0800
[SPARK-33480][SQL][FOLLOWUP] do not expose user data in error message
### What changes were proposed in this pull request?
This is a followup of https://github.com/apache/spark/pull/30412. This PR
updates the error message of char/varchar table insertion length check, to not
expose user data.
### Why are the changes needed?
This is risky to expose user data in the error message, especially the
string data, as it may contain sensitive data.
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
updated tests
Closes #30653 from cloud-fan/minor2.
Authored-by: Wenchen Fan <wenc...@databricks.com>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
(cherry picked from commit c0874ba9f13b9802eef4418490020692e37652ba)
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
.../spark/sql/catalyst/util/CharVarcharUtils.scala | 6 ++---
.../apache/spark/sql/CharVarcharTestSuite.scala | 26 +++++++++++-----------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
index b551d96..e42e384 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala
@@ -183,9 +183,9 @@ object CharVarcharUtils extends Logging {
private def raiseError(expr: Expression, typeName: String, length: Int):
Expression = {
val errorMsg = Concat(Seq(
- Literal("input string '"),
- expr,
- Literal(s"' exceeds $typeName type length limitation: $length")))
+ Literal("input string of length "),
+ Cast(Length(expr), StringType),
+ Literal(s" exceeds $typeName type length limitation: $length")))
Cast(RaiseError(errorMsg), StringType)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
index fcd334b..b0f1198 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
@@ -190,7 +190,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(null))
val e = intercept[SparkException](sql("INSERT INTO t VALUES ('123456')"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -203,7 +203,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(1, null))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (1,
'123456')"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation:
5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
}
@@ -215,7 +215,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(Row(null)))
val e = intercept[SparkException](sql("INSERT INTO t SELECT
struct('123456')"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -226,7 +226,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(Seq(null)))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (array('a',
'123456'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -235,7 +235,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
sql(s"CREATE TABLE t(c MAP<$typeName(5), STRING>) USING $format")
val e = intercept[SparkException](sql("INSERT INTO t VALUES
(map('123456', 'a'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -246,7 +246,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(Map("a" -> null)))
val e = intercept[SparkException](sql("INSERT INTO t VALUES (map('a',
'123456'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -255,10 +255,10 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
sql(s"CREATE TABLE t(c MAP<$typeName(5), $typeName(5)>) USING $format")
val e1 = intercept[SparkException](sql("INSERT INTO t VALUES
(map('123456', 'a'))"))
assert(e1.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
val e2 = intercept[SparkException](sql("INSERT INTO t VALUES (map('a',
'123456'))"))
assert(e2.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -269,7 +269,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(Row(Seq(null))))
val e = intercept[SparkException](sql("INSERT INTO t SELECT
struct(array('123456'))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -280,7 +280,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(Seq(Row(null))))
val e = intercept[SparkException](sql("INSERT INTO t VALUES
(array(struct('123456')))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -291,7 +291,7 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row(Seq(Seq(null))))
val e = intercept[SparkException](sql("INSERT INTO t VALUES
(array(array('123456')))"))
assert(e.getCause.getMessage.contains(
- s"input string '123456' exceeds $typeName type length limitation: 5"))
+ s"input string of length 6 exceeds $typeName type length limitation:
5"))
}
}
@@ -313,10 +313,10 @@ trait CharVarcharTestSuite extends QueryTest with
SQLTestUtils {
checkAnswer(spark.table("t"), Row("1234 ", "1234"))
val e1 = intercept[SparkException](sql("INSERT INTO t VALUES (123456,
1)"))
assert(e1.getCause.getMessage.contains(
- "input string '123456' exceeds char type length limitation: 5"))
+ "input string of length 6 exceeds char type length limitation: 5"))
val e2 = intercept[SparkException](sql("INSERT INTO t VALUES (1,
123456)"))
assert(e2.getCause.getMessage.contains(
- "input string '123456' exceeds varchar type length limitation: 5"))
+ "input string of length 6 exceeds varchar type length limitation: 5"))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
