HeartSaVioR commented on code in PR #49654:
URL: https://github.com/apache/spark/pull/49654#discussion_r1949086634
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala:
##########
@@ -456,11 +461,11 @@ case class DataSource(
// 3. It's OK that the output path doesn't exist yet;
val allPaths = paths ++ caseInsensitiveOptions.get("path")
val outputPath = if (allPaths.length == 1) {
- val path = new Path(allPaths.head)
- val fs = path.getFileSystem(newHadoopConfiguration())
- path.makeQualified(fs.getUri, fs.getWorkingDirectory)
- } else {
+ makeQualified(new Path(allPaths.head))
+ } else if (allPaths.length > 1) {
throw QueryExecutionErrors.multiplePathsSpecifiedError(allPaths)
+ } else {
Review Comment:
nit: Let's not include another fix - please have a separate PR for fixing
other stuff.
##########
R/pkg/tests/fulltests/test_sparkSQL.R:
##########
@@ -4000,7 +4000,7 @@ test_that("Call DataFrameWriter.save() API in Java
without path and check argume
# DataFrameWriter.save() without path.
expect_error(write.df(df, source = "csv"),
paste("Error in save :
org.apache.spark.SparkIllegalArgumentException:",
- "Expected exactly one path to be specified"))
+ "'path' is not specified."))
Review Comment:
Why this change is needed for this PR? I guess you'll revert this change
once you separate the fix in batch writer out.
##########
sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala:
##########
@@ -292,35 +293,48 @@ abstract class FileStreamSinkSuite extends StreamTest {
test("parquet") {
testFormat(None) // should not throw error as default format parquet when
not specified
testFormat(Some("parquet"))
+ testFormat(None, relativizeOutputPath = true)
+ testFormat(Some("parquet"), relativizeOutputPath = true)
}
test("orc") {
testFormat(Some("orc"))
+ testFormat(Some("orc"), relativizeOutputPath = true)
}
test("text") {
testFormat(Some("text"))
+ testFormat(Some("text"), relativizeOutputPath = true)
}
test("json") {
testFormat(Some("json"))
+ testFormat(Some("json"), relativizeOutputPath = true)
}
- def testFormat(format: Option[String]): Unit = {
- val inputData = MemoryStream[Int]
+ def testFormat(format: Option[String], relativizeOutputPath: Boolean =
false): Unit = {
+ val inputData = MemoryStream[String] // text format only supports String
val ds = inputData.toDS()
- val outputDir = Utils.createTempDir(namePrefix =
"stream.output").getCanonicalPath
+ val tempDir = Utils.createTempDir(namePrefix = "stream.output");
Review Comment:
nit: remove `;` as it's unnecessary
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
