Repository: spark Updated Branches: refs/heads/master e3c7039b4 -> d6795c7a2
[SPARK-16515][SQL][FOLLOW-UP] Fix test `script` on OS X/Windows... ## Problem The current `sed` in `test_script.sh` is missing a `$`, leading to the failure of `script` test on OS X: ``` == Results == !== Correct Answer - 2 == == Spark Answer - 2 == ![x1_y1] [x1] ![x2_y2] [x2] ``` In addition, this `script` test would also fail on systems like Windows where we couldn't be able to invoke `bash` or `echo | sed`. ## What changes were proposed in this pull request? This patch - fixes `sed` in `test_script.sh` - adds command guards so that the `script` test would pass on systems like Windows ## How was this patch tested? - Jenkins - Manually verified tests pass on OS X Author: Liwei Lin <[email protected]> Closes #14280 from lw-lin/osx-sed. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d6795c7a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d6795c7a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d6795c7a Branch: refs/heads/master Commit: d6795c7a254b83d4ae4785f3add74981e5273c91 Parents: e3c7039 Author: Liwei Lin <[email protected]> Authored: Sun Jul 24 08:35:57 2016 +0100 Committer: Sean Owen <[email protected]> Committed: Sun Jul 24 08:35:57 2016 +0100 ---------------------------------------------------------------------- sql/hive/src/test/resources/test_script.sh | 2 +- .../sql/hive/execution/SQLQuerySuite.scala | 26 ++++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/d6795c7a/sql/hive/src/test/resources/test_script.sh ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/resources/test_script.sh b/sql/hive/src/test/resources/test_script.sh index ab998c4..eb0c50e 100755 --- a/sql/hive/src/test/resources/test_script.sh +++ b/sql/hive/src/test/resources/test_script.sh @@ -19,5 +19,5 @@ while read line do - echo "$line" | sed 's/\t/_/' + echo "$line" | sed $'s/\t/_/' done < /dev/stdin http://git-wip-us.apache.org/repos/asf/spark/blob/d6795c7a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 688260d..cba6aa5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution import java.sql.{Date, Timestamp} +import scala.sys.process.Process +import scala.util.Try + import org.apache.hadoop.fs.Path import org.apache.spark.sql._ @@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { import spark.implicits._ test("script") { - val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") - df.createOrReplaceTempView("script_table") - val query1 = sql( - """ - |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table - |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS - |(col1 STRING, col2 STRING)) script_test_table""".stripMargin) - checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil) + if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) { + val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") + df.createOrReplaceTempView("script_table") + val query1 = sql( + """ + |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table + |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS + |(col1 STRING, col2 STRING)) script_test_table""".stripMargin) + checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil) + } + // else skip this test } test("UDTF") { @@ -1766,4 +1772,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } } + + def testCommandAvailable(command: String): Boolean = { + Try(Process(command) !!).isSuccess + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
