Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/22213#discussion_r216093949
--- Diff:
core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala ---
@@ -1144,6 +1144,48 @@ class SparkSubmitSuite
conf1.get(PY_FILES.key) should be (s"s3a://${pyFile.getAbsolutePath}")
conf1.get("spark.submit.pyFiles") should (startWith("/"))
}
+
+ test("handles natural line delimiters in --properties-file and --conf
uniformly") {
+ val delimKey = "spark.my.delimiter."
+ val LF = "\n"
+ val CR = "\r"
+
+ val leadingDelimKeyFromFile = s"${delimKey}leadingDelimKeyFromFile" ->
s"${LF}blah"
+ val trailingDelimKeyFromFile = s"${delimKey}trailingDelimKeyFromFile"
-> s"blah${CR}"
+ val infixDelimFromFile = s"${delimKey}infixDelimFromFile" ->
s"${CR}blah${LF}"
+ val nonDelimSpaceFromFile = s"${delimKey}nonDelimSpaceFromFile" -> "
blah\f"
+
+ val testProps = Seq(leadingDelimKeyFromFile, trailingDelimKeyFromFile,
infixDelimFromFile,
+ nonDelimSpaceFromFile)
+
+ val props = new java.util.Properties()
+ val propsFile = File.createTempFile("test-spark-conf", ".properties",
+ Utils.createTempDir())
+ val propsOutputStream = new FileOutputStream(propsFile)
+ try {
+ testProps.foreach { case (k, v) => props.put(k, v) }
+ props.store(propsOutputStream, "test whitespace")
+ } finally {
+ propsOutputStream.close()
+ }
+
+ val clArgs = Seq(
+ "--class", "org.SomeClass",
+ "--conf", s"${delimKey}=$LF",
+ "--conf", "spark.master=yarn",
+ "--properties-file", propsFile.getPath,
+ "thejar.jar")
+
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (_, _, conf, _) = submit.prepareSubmitEnvironment(appArgs)
+
+ Seq((delimKey -> LF), leadingDelimKeyFromFile,
trailingDelimKeyFromFile, infixDelimFromFile)
+ .foreach {
--- End diff --
nit: `case ... =>` stays with the opening brace.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
