skonto commented on a change in pull request #25870: [SPARK-27936][K8S] Support
python deps
URL: https://github.com/apache/spark/pull/25870#discussion_r344355922
##########
File path:
resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
##########
@@ -131,4 +133,41 @@ object Utils extends Logging {
s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
}
}
+
+ def getTestFileAbsolutePath(fileName: String, sparkHomeDir: Path): String = {
+ val filePathsFound = Files
+ .walk(sparkHomeDir)
+ .filter(Files.isRegularFile(_))
+ .filter((f: Path) => {f.toFile.getName == fileName})
+ // we should not have more than one here under current test build dir
+ // we only need one though
+ val filePath = filePathsFound
+ .iterator()
+ .asScala
+ .map(_.toAbsolutePath.toString)
+ .toArray
+ .headOption
+ filePath match {
+ case Some(file) => file
+ case _ => throw new SparkException(s"No valid $fileName file was found "
+
+ s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
+ }
+ }
+
+ def createZipFile(inFile: String, outFile: String): Unit = {
+ try {
+ val fileToZip = new File(inFile)
+ val fis = new FileInputStream(fileToZip)
+ val fos = new FileOutputStream(outFile)
+ val zipOut = new ZipOutputStream(fos)
+ val zipEntry = new ZipEntry(fileToZip.getName)
+ zipOut.putNextEntry(zipEntry)
+ IOUtils.copy(fis, zipOut)
+ IOUtils.closeQuietly(fis)
+ IOUtils.closeQuietly(zipOut)
+ } catch {
+ case e: Exception => log.error(s"Failed to zip file: $inFile"); throw e
+ }
+ }
}
+
Review comment:
@dongjoon-hyun did I ok :) Usually I add a \n at the end as an old practice:
https://unix.stackexchange.com/questions/18743/whats-the-point-in-adding-a-new-line-to-the-end-of-a-file
https://thoughtbot.com/blog/no-newline-at-end-of-file
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
