comphead commented on code in PR #6246:
URL: https://github.com/apache/datafusion-comet/pull/6246#discussion_r4608076648
##########
spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala:
##########
@@ -309,6 +310,173 @@ abstract class ParquetReadSuite extends CometTestBase {
}
}
+ test("spark.sql.parquet.binaryAsString uses native scan") {
+ withTempDir { dir =>
+ val path = new Path(dir.toURI.toString, "part-r-0.parquet")
+ val schema = MessageTypeParser.parseMessageType("""
+ |message root {
+ | optional binary value;
+ | optional int32 id;
+ |}
+ |""".stripMargin)
+ val writer = createParquetWriter(schema, path)
+ Seq(Binary.fromString("one"),
Binary.fromConstantByteArray(Array(0xff.toByte))).zipWithIndex
+ .foreach { case (value, id) =>
+ val record = new SimpleGroup(schema)
+ record.add(0, value)
+ record.add(1, id)
+ writer.write(record)
+ }
+ writer.close()
+
+ withSQLConf(SQLConf.PARQUET_BINARY_AS_STRING.key -> "false") {
+ val df = spark.read.parquet(path.toString)
+ assert(
+ df.schema === StructType(
+ Seq(StructField("value", BinaryType, true), StructField("id",
IntegerType, true))))
+ val (_, cometPlan) = checkSparkAnswer(df)
+ assert(
+ collect(cometPlan) { case _: CometNativeScanExec => true }.nonEmpty,
+ s"Expected native scan with binaryAsString
disabled:\n${cometPlan.treeString}")
+ }
+
+ withSQLConf(SQLConf.PARQUET_BINARY_AS_STRING.key -> "true") {
+ val df = spark.read.parquet(path.toString)
+ assert(
+ df.schema === StructType(
+ Seq(StructField("value", StringType, true), StructField("id",
IntegerType, true))))
+ val (_, cometPlan) = checkSparkAnswer(df)
+ assert(
+ collect(cometPlan) { case _: CometNativeScanExec => true }.nonEmpty,
+ s"Expected native scan with binaryAsString
enabled:\n${cometPlan.treeString}")
+
+ val projected = spark.read.parquet(path.toString).select("id")
+ val (_, projectedCometPlan) = checkSparkAnswer(projected)
+ assert(
+ collect(projectedCometPlan) { case _: CometNativeScanExec => true
}.nonEmpty,
+ s"Expected native scan when no string columns are
read:\n${projectedCometPlan.treeString}")
+ }
+ }
+ }
+
+ test("binaryAsString preserves fixed-length binary conversion errors") {
+ withTempDir { dir =>
+ val path = new Path(dir.toURI.toString, "fixed.parquet")
+ val schema = MessageTypeParser.parseMessageType(
+ "message root { optional fixed_len_byte_array(3) value; }")
+ val writer = createParquetWriter(schema, path)
+ val record = new SimpleGroup(schema)
+ record.add(0, Binary.fromString("one"))
+ writer.write(record)
+ writer.close()
+ Seq("false", "true").foreach { rowFilter =>
+ withSQLConf(
+ SQLConf.PARQUET_BINARY_AS_STRING.key -> "true",
+ CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key ->
rowFilter) {
+ withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
+ intercept[SparkException] {
+ spark.read.schema("value
string").parquet(path.toString).collect()
+ }
+ }
+ val df = spark.read.schema("value string").parquet(path.toString)
+ assert(collect(df.queryExecution.executedPlan) { case _:
CometNativeScanExec =>
+ true
+ }.nonEmpty)
+ val error = intercept[SparkException](df.collect())
+ assert(
+
causeChain(error).exists(_.isInstanceOf[SchemaColumnConvertNotSupportedException]))
+ checkSparkAnswerAndOperator(spark.read.schema("value
binary").parquet(path.toString))
+ }
+ }
+ }
+ }
+
+ // With binaryAsString, the native scan compares `=`, `IN` and `IS [NOT]
NULL` on the raw bytes
+ // and other string filters on the decoded values. Spark compares raw bytes,
and these queries
+ // match it. A query that tells a malformed value apart from U+FFFD doesn't,
as the
+ // compatibility guide describes.
+ test("spark.sql.parquet.binaryAsString filters, joins and top-k") {
+ withTempDir { dir =>
+ val schema = MessageTypeParser.parseMessageType("""
+ |message root {
+ | optional binary value;
+ | optional int32 id;
+ |}
+ |""".stripMargin)
+ // Two partition directories, each holding a valid value, the malformed
byte 0xff and a null.
+ val partitioned = new File(dir, "partitioned").getCanonicalPath
+ Seq(1 -> "one", 2 -> "two").foreach { case (p, valid) =>
+ val writer = createParquetWriter(schema, new
Path(s"$partitioned/p=$p/part-r-0.parquet"))
+ Seq(
+ Some(Binary.fromString(valid)),
+ Some(Binary.fromConstantByteArray(Array(0xff.toByte))),
+ None).zipWithIndex.foreach { case (value, id) =>
+ val record = new SimpleGroup(schema)
+ value.foreach(record.add(0, _))
+ record.add(1, p * 10 + id)
+ writer.write(record)
+ }
+ writer.close()
+ }
+ val annotated = new File(dir, "annotated").getCanonicalPath
+ Seq(Some("one"), Some("two"),
None).toDF("value").write.parquet(annotated)
+
+ def assertNativeScan(plan: SparkPlan): Unit =
+ assert(
+ collect(plan) { case _: CometNativeScanExec => true }.nonEmpty,
+ s"Expected a native scan:\n${plan.treeString}")
+
+ Seq("false", "true").foreach { rowFilter =>
+ withSQLConf(
+ SQLConf.PARQUET_BINARY_AS_STRING.key -> "true",
+ CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key ->
rowFilter) {
+ spark.read.parquet(partitioned).createOrReplaceTempView("t")
+ spark.read.parquet(annotated).createOrReplaceTempView("annotated")
+ Seq(
+ "SELECT * FROM t WHERE value = 'one'",
+ "SELECT * FROM t WHERE value IN ('one', 'two')",
+ "SELECT * FROM t WHERE value NOT IN ('one', 'two')",
+ "SELECT * FROM t WHERE value IS NULL",
+ "SELECT * FROM t WHERE value IS NOT NULL",
+ "SELECT * FROM t WHERE value > 'o'",
+ "SELECT * FROM t WHERE value LIKE 'o%'",
+ "SELECT * FROM t WHERE value = 'one' OR id = 21",
+ "SELECT * FROM annotated WHERE value = 'one'",
+ "SELECT * FROM annotated WHERE value < 'two'",
+ "SELECT * FROM annotated WHERE value LIKE 't%'",
+ "SELECT * FROM annotated WHERE value NOT IN ('one')",
+ "SELECT * FROM annotated WHERE value IS NULL").foreach { query =>
Review Comment:
These five queries read a Spark-written table. I expect they can be an SQL
file test with `-- Config: spark.sql.parquet.binaryAsString=true` and `--
ConfigMatrix: spark.comet.parquet.rowFilterPushdown.enabled=false,true`, as
`float_comparisons.sql` does, compared against Spark by
`CometSqlFileTestSuite`. The unannotated queries and the
`pushdown_rows_matched` assertions need a raw Parquet writer, so they would
stay here.
##########
native/core/src/parquet/schema_adapter.rs:
##########
@@ -1409,6 +1447,100 @@ impl SparkPhysicalExprAdapter {
.data()
}
+ /// A `binaryAsString` scan reads string columns as Binary and decodes
them to strings with a
+ /// Spark cast, in its output projection and in the filters it pushes. For
a file that
+ /// annotates such a column as a string, the default adapter makes that
+ /// `CAST(CAST(s AS BINARY) AS STRING)`. arrow-rs has already validated an
annotated column's
Review Comment:
arrow-rs validates UTF-8 only when the converted type is `UTF8`
(`validate_utf8 = desc.converted_type() == ConvertedType::UTF8` in
`array_reader/byte_array.rs`, parquet 59.3.2). It maps a JSON-annotated
`BINARY` to `Utf8` without that check (`schema/primitive.rs`), and Spark reads
JSON as `StringType`. For such a file the cast pair used to decode with
replacement, and this rule now returns the bytes unvalidated. That matches the
setting-off behavior, so it is not a regression against `branch-1.2`, but this
comment and the `spark_configs_support.md` entry say an annotated column is
already validated. Could both be corrected for JSON? I have not run a
JSON-annotated file through the scan.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]