srielau commented on code in PR #59716:
URL: https://github.com/apache/spark/pull/59716#discussion_r4389651209
##########
connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/PostgresIntegrationSuite.scala:
##########
@@ -416,4 +416,27 @@ class PostgresIntegrationSuite extends
DockerJDBCIntegrationV2Suite with V2JDBCT
assert(rows.length === 1)
assert(rows(0).getString(0) === "amy")
}
+
+ test("SPARK-60479: CHAR read-side padding with NULL") {
+ val table = s"$catalogName.char_pad"
+ withTable(table) {
+ sql(s"CREATE TABLE $table (id INT, flag CHAR(5))")
+ sql(s"INSERT INTO $table VALUES (1, 'Y'), (2, NULL)")
Review Comment:
Postgres and SQL Server docker tables now include `(3, 'YES')`. Equality
must exclude it and `NOT IN ('Y', 'N')` must return it, so both under-matching
and over-matching remotes fail.
##########
connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/v2/OracleIntegrationSuite.scala:
##########
@@ -428,4 +428,36 @@ class OracleIntegrationSuite extends
DockerJDBCIntegrationV2Suite with V2JDBCTes
assert(rows11(0).getString(0) === "amy")
assert(rows11(1).getString(0) === "alex")
}
+
+ test("SPARK-60479: CHAR equality is pushed, including CJK") {
+ val tableName = caseConvert("char_pad")
+ val table = s"$catalogName.$tableName"
+ withTable(table) {
+ // CHAR(n CHAR) so CJK occupies one length unit, matching Spark CHAR
padding.
+ Using.resource(getConnection()) { conn =>
+ conn.prepareStatement(
+ s"CREATE TABLE $tableName (id NUMBER, flag CHAR(4
CHAR))").executeUpdate()
Review Comment:
Added a `CHAR(4 BYTE)` table with multibyte `ä` / `äb` plus `Y`, and
asserted pushed equality returns those rows. That is the
stored-shorter-than-Spark-literal case the `CHAR(4 CHAR)` table could not tell
from exact compare.
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala:
##########
@@ -1882,6 +1883,104 @@ class DataSourceV2StrategySuite extends
SharedSparkSession {
}
}
+ test("SPARK-60479: read-side CHAR padding StaticInvoke translates") {
+ val md = new MetadataBuilder()
+ .putString(CharVarcharUtils.CHAR_VARCHAR_TYPE_STRING_METADATA_KEY,
"char(1)")
+ .build()
+ val col = AttributeReference("flag", StringType, metadata = md)()
+ val padded = CharVarcharUtils.addPaddingForScan(col)
+ val eq = EqualTo(padded, Literal("Y"))
+ val result = new V2ExpressionBuilder(eq, isPredicate = true).build()
+ assert(result.isDefined, "readSidePadding should translate to a V2
predicate")
+ result.get match {
+ case p: Predicate if p.name() == "=" =>
+ p.children()(0) match {
+ case g: GeneralScalarExpression
+ if g.name() == CharVarcharUtils.V2_CHAR_READ_SIDE_PADDING =>
+ assert(g.children()(0) === FieldReference("flag"))
+ assert(g.children()(1) === LiteralValue(1, IntegerType))
+ assert(g.children()(2) ===
+ LiteralValue(UTF8String.fromString(" "), StringType))
+ case other =>
+ fail(s"expected CHAR_READ_SIDE_PADDING, got ${other.describe()}")
+ }
+ assert(p.children()(1) === LiteralValue(UTF8String.fromString("Y"),
StringType))
+ case other =>
+ fail(s"expected '=' predicate, got ${other.describe()}")
+ }
+
+ val isNull = new V2ExpressionBuilder(IsNull(padded), isPredicate =
true).build()
+ assert(isNull.isDefined)
+ isNull.get match {
+ case p: Predicate if p.name() == "IS_NULL" =>
+ assert(p.children()(0) === FieldReference("flag"))
+ case other => fail(s"expected IS_NULL on the raw column, got
${other.describe()}")
+ }
+
+ val nseq = new V2ExpressionBuilder(EqualNullSafe(padded, Literal("Y")),
isPredicate = true)
+ .build()
+ assert(nseq.isDefined)
+ val in = new V2ExpressionBuilder(In(padded, Seq(Literal("Y"))),
isPredicate = true).build()
+ assert(in.isDefined)
+ val neq = new V2ExpressionBuilder(Not(EqualTo(padded, Literal("Y"))),
isPredicate = true)
+ .build()
+ assert(neq.isDefined)
+ neq.get match {
+ case p: Predicate if p.name() == "<>" =>
Review Comment:
The `<>` unit test now asserts the `CHAR_READ_SIDE_PADDING` child and the
padded literal, same as `=`. H2 runs `flag <> 'Y'` with `checkPushedInfo` and
`checkAnswer`; the NULL row is not returned.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
