dongjoon-hyun commented on a change in pull request #35509:
URL: https://github.com/apache/spark/pull/35509#discussion_r807192879
##########
File path:
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala
##########
@@ -65,4 +72,57 @@ class KubernetesUtilsSuite extends SparkFunSuite {
assert(sparkPodWithNoContainerName.pod.getSpec.getHostname == HOST)
assert(sparkPodWithNoContainerName.container.getName == null)
}
+
+ test("SPARK-38201: check uploadFileToHadoopCompatibleFS with different
delSrc and overwrite") {
+ withTempDir { srcDir =>
+ withTempDir { destDir =>
+ val upload =
PrivateMethod[Unit](Symbol("uploadFileToHadoopCompatibleFS"))
+ val fileName = "test.txt"
+ val srcFile = new File(srcDir, fileName)
+ val src = new Path(srcFile.getAbsolutePath)
+ val dest = new Path(destDir.getAbsolutePath, fileName)
+ val fs = src.getFileSystem(new Configuration())
+
+ def checkUploadException(delSrc: Boolean, overwrite: Boolean): Unit = {
+ val message = intercept[SparkException] {
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, delSrc,
overwrite))
+ }.getMessage
+ assert(message.contains("Error uploading file"))
+ }
+
+ def appendFileAndUpload(content: String, delSrc: Boolean, overwrite:
Boolean): Unit = {
+ FileUtils.write(srcFile, content, StandardCharsets.UTF_8, true)
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, delSrc,
overwrite))
+ }
+
+ // Write a new file, upload file with delSrc = false and overwrite =
true.
+ // Upload successful and record the `fileLength`.
+ appendFileAndUpload("init-content", delSrc = false, overwrite = true)
+ val firstLength = fs.getFileStatus(dest).getLen
+
+ // Append the file, upload file with delSrc = false and overwrite =
true.
+ // Upload succeeded but `fileLength` changed.
+ appendFileAndUpload("append-content", delSrc = false, overwrite = true)
+ val secondLength = fs.getFileStatus(dest).getLen
+ assert(firstLength < secondLength)
+
+ // Upload file with delSrc = false and overwrite = false.
+ // Upload failed because dest exists.
+ checkUploadException(delSrc = false, overwrite = false)
Review comment:
The last comment is not addressed still.
- https://github.com/apache/spark/pull/35509#discussion_r806611191
##########
File path:
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala
##########
@@ -65,4 +72,57 @@ class KubernetesUtilsSuite extends SparkFunSuite {
assert(sparkPodWithNoContainerName.pod.getSpec.getHostname == HOST)
assert(sparkPodWithNoContainerName.container.getName == null)
}
+
+ test("SPARK-38201: check uploadFileToHadoopCompatibleFS with different
delSrc and overwrite") {
+ withTempDir { srcDir =>
+ withTempDir { destDir =>
+ val upload =
PrivateMethod[Unit](Symbol("uploadFileToHadoopCompatibleFS"))
+ val fileName = "test.txt"
+ val srcFile = new File(srcDir, fileName)
+ val src = new Path(srcFile.getAbsolutePath)
+ val dest = new Path(destDir.getAbsolutePath, fileName)
+ val fs = src.getFileSystem(new Configuration())
+
+ def checkUploadException(delSrc: Boolean, overwrite: Boolean): Unit = {
+ val message = intercept[SparkException] {
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, delSrc,
overwrite))
+ }.getMessage
+ assert(message.contains("Error uploading file"))
+ }
+
+ def appendFileAndUpload(content: String, delSrc: Boolean, overwrite:
Boolean): Unit = {
+ FileUtils.write(srcFile, content, StandardCharsets.UTF_8, true)
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, delSrc,
overwrite))
+ }
+
+ // Write a new file, upload file with delSrc = false and overwrite =
true.
+ // Upload successful and record the `fileLength`.
+ appendFileAndUpload("init-content", delSrc = false, overwrite = true)
+ val firstLength = fs.getFileStatus(dest).getLen
+
+ // Append the file, upload file with delSrc = false and overwrite =
true.
+ // Upload succeeded but `fileLength` changed.
+ appendFileAndUpload("append-content", delSrc = false, overwrite = true)
+ val secondLength = fs.getFileStatus(dest).getLen
+ assert(firstLength < secondLength)
+
+ // Upload file with delSrc = false and overwrite = false.
+ // Upload failed because dest exists.
+ checkUploadException(delSrc = false, overwrite = false)
Review comment:
We need to check the file is identical after this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
