viirya commented on code in PR #6821:
URL: https://github.com/apache/datafusion-comet/pull/6821#discussion_r4233248376
##########
spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala:
##########
@@ -1340,6 +1340,122 @@ class CometParquetWriterSuite extends
CometParquetWriterTestBase {
}
}
+ test("a native overwrite of a cached path recaches the data") {
+ withTempPath { dir =>
+ val target = new File(dir, "target").getAbsolutePath
+ val source = new File(dir, "source").getAbsolutePath
+ withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
+ spark.range(0, 5).write.parquet(target)
+ spark.range(100, 103).write.parquet(source)
+ }
+ val cached = spark.read.parquet(target).cache()
+ try {
+ assert(cached.count() == 5)
+ withNativeWriter {
+ assertHasCometNativeWriteExec(
+ captureWritePlan(
+
spark.read.parquet(source).write.mode(SaveMode.Overwrite).parquet(target)))
+ }
+ // Spark recaches every cached plan that reads the output path once
the write commits.
+ checkAnswer(cached, (100L until 103L).map(Row(_)))
+ } finally {
+ cached.unpersist()
+ }
+ }
+ }
+
+ test("a native write rejects case-insensitively duplicate column names") {
+ withTempPath { dir =>
+ val source = new File(dir, "source").getAbsolutePath
+ val target = new File(dir, "target")
+ withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
+ spark.range(3).selectExpr("id AS a", "id AS b").write.parquet(source)
+ }
+ withNativeWriter {
+ val e = intercept[AnalysisException] {
Review Comment:
Good catch, thanks. Added `captureFailedWritePlan` to
`CometParquetWriterTestBase`, which keeps the plan from `onFailure` as well,
and the test now asserts the failed write carried a native write before
checking the error. With the support level forced to `Unsupported` it now fails
with "found 0" native write operators, and it still fails without the new check
in `CometNativeWriteExec`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
