dongjoon-hyun commented on a change in pull request #35509:
URL: https://github.com/apache/spark/pull/35509#discussion_r806611191
##########
File path:
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesUtilsSuite.scala
##########
@@ -65,4 +72,61 @@ class KubernetesUtilsSuite extends SparkFunSuite {
assert(sparkPodWithNoContainerName.pod.getSpec.getHostname == HOST)
assert(sparkPodWithNoContainerName.container.getName == null)
}
+
+ test("SPARK-38201: check uploadFileToHadoopCompatibleFS with different
delSrc and overwrite") {
+
+ def checkUploadFailed(f: () => Unit): Unit = {
+ val message = intercept[SparkException] {
+ f
+ }.getMessage
+ assert(message.contains("Error uploading file"))
+ }
+
+ withTempDir { srcDir =>
+ withTempDir { destDir =>
+ val upload =
PrivateMethod[Unit](Symbol("uploadFileToHadoopCompatibleFS"))
+ val fileName = "test.txt"
+ val srcFile = new File(srcDir, fileName)
+ val src = new Path(srcFile.getAbsolutePath)
+ val dest = new Path(destDir.getAbsolutePath, fileName)
+ val fs = src.getFileSystem(new Configuration())
+ // Write a new file, upload file with delSrc = false and overwrite =
true.
+ // Upload successful and record the `fileLength`.
+ FileUtils.write(srcFile, "init-content", StandardCharsets.UTF_8)
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, false, true))
+ val firstLength = fs.getFileStatus(dest).getLen
+
+ // Append the file, upload file with delSrc = false and overwrite =
true.
+ // Upload succeeded but `fileLength` changed.
+ FileUtils.write(srcFile, "append-content", StandardCharsets.UTF_8,
true)
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, false, true))
+ val secondLength = fs.getFileStatus(dest).getLen
+ assert(firstLength < secondLength)
+
+ // Upload file with delSrc = false and overwrite = false.
+ // Upload failed because dest exists.
+ val message1 = intercept[SparkException] {
+ KubernetesUtils.invokePrivate(upload(src, dest, fs, false, false))
+ }.getMessage
+ assert(message1.contains("Error uploading file"))
Review comment:
It seems that we need to check that the uploaded file is not changed
here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
