Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/22213#discussion_r216093880
--- Diff:
core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala ---
@@ -1144,6 +1144,48 @@ class SparkSubmitSuite
conf1.get(PY_FILES.key) should be (s"s3a://${pyFile.getAbsolutePath}")
conf1.get("spark.submit.pyFiles") should (startWith("/"))
}
+
+ test("handles natural line delimiters in --properties-file and --conf
uniformly") {
+ val delimKey = "spark.my.delimiter."
+ val LF = "\n"
+ val CR = "\r"
+
+ val leadingDelimKeyFromFile = s"${delimKey}leadingDelimKeyFromFile" ->
s"${LF}blah"
+ val trailingDelimKeyFromFile = s"${delimKey}trailingDelimKeyFromFile"
-> s"blah${CR}"
+ val infixDelimFromFile = s"${delimKey}infixDelimFromFile" ->
s"${CR}blah${LF}"
+ val nonDelimSpaceFromFile = s"${delimKey}nonDelimSpaceFromFile" -> "
blah\f"
+
+ val testProps = Seq(leadingDelimKeyFromFile, trailingDelimKeyFromFile,
infixDelimFromFile,
+ nonDelimSpaceFromFile)
+
+ val props = new java.util.Properties()
+ val propsFile = File.createTempFile("test-spark-conf", ".properties",
+ Utils.createTempDir())
+ val propsOutputStream = new FileOutputStream(propsFile)
+ try {
+ testProps.foreach { case (k, v) => props.put(k, v) }
+ props.store(propsOutputStream, "test whitespace")
+ } finally {
+ propsOutputStream.close()
+ }
+
+ val clArgs = Seq(
+ "--class", "org.SomeClass",
+ "--conf", s"${delimKey}=$LF",
--- End diff --
Took a couple of minutes for me to notice this; could you make this similar
to the other properties (add a variable to hold its key/value) so that someone
reading L1182 knows easily where it comes from?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
