This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 9227231 [SPARK-37871][TESTS] Use `python3` instead of `python` in BaseScriptTransformation tests 9227231 is described below commit 922723189e61d7228d6b5669d836ed86ef95e3ef Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Tue Jan 11 15:36:05 2022 -0800 [SPARK-37871][TESTS] Use `python3` instead of `python` in BaseScriptTransformation tests ### What changes were proposed in this pull request? This PR aims to use `python3` instead of `python` in `BaseScriptTransformation` tests. ### Why are the changes needed? Since Apache Spark deprecated `Python 2`, this PR aims to make it sure. In addition, sometimes `python3` or `python3.x` command exists. ``` [info] - SPARK-25158: Executor accidentally exit because ScriptTransformationWriterThread throw Exception *** FAILED *** (248 milliseconds) [info] "Job aborted due to stage failure: Task 0 in stage 2162.0 failed 1 times, most recent failure: Lost task 0.0 in stage 2162.0 (TID 2627) (0ac7628d09c6 executor driver): org.apache.spark.SparkException: Subprocess exited with status 127. Error: /bin/bash: python: command not found ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. Closes #35171 from dongjoon-hyun/SPARK-37871. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../execution/BaseScriptTransformationSuite.scala | 26 +++++++++++----------- .../sql/hive/execution/BigDataBenchmarkSuite.scala | 2 +- .../spark/sql/hive/execution/SQLQuerySuite.scala | 7 +++--- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala index 488a0fd..f774c45 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/BaseScriptTransformationSuite.scala @@ -109,7 +109,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU } test("SPARK-25990: TRANSFORM should handle different data types correctly") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) val scriptFilePath = copyAndGetResourceFile("test_script.py", ".py").getAbsoluteFile withTempView("v") { @@ -126,7 +126,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU |TRANSFORM(a, b, c, d, e) | ROW FORMAT DELIMITED | FIELDS TERMINATED BY '\t' - | USING 'python $scriptFilePath' AS (a, b, c, d, e) + | USING 'python3 $scriptFilePath' AS (a, b, c, d, e) | ROW FORMAT DELIMITED | FIELDS TERMINATED BY '\t' |FROM v @@ -212,7 +212,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU def testBasicInputDataTypesWith(serde: ScriptTransformationIOSchema, testName: String): Unit = { test(s"SPARK-32400: TRANSFORM should support basic data types as input ($testName)") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) withTempView("v") { val df = Seq( (1, "1", 1.0f, 1.0, 11.toByte, BigDecimal(1.0), new Timestamp(1), @@ -251,7 +251,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU test("SPARK-32400: TRANSFORM should support more data types (interval, array, map, struct " + "and udt) as input (no serde)") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) withTempView("v") { val df = Seq( (new CalendarInterval(7, 1, 1000), Array(0, 1, 2), Map("a" -> 1), (1, 2), @@ -287,7 +287,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU } test("SPARK-32400: TRANSFORM should respect DATETIME_JAVA8API_ENABLED (no serde)") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) Array(false, true).foreach { java8AapiEnable => withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8AapiEnable.toString) { withTempView("v") { @@ -405,7 +405,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU """.stripMargin, s""" |SELECT TRANSFORM(a) - |USING 'python some_non_existent_file' AS (a) + |USING 'python3 some_non_existent_file' AS (a) |FROM VALUES (1) t(a) """.stripMargin).foreach { query => intercept[SparkException] { @@ -442,7 +442,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU } test("SPARK-31936: Script transform support ArrayType/MapType/StructType (no serde)") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) withTempView("v") { val df = Seq( (Array(0, 1, 2), Array(Array(0, 1), Array(2)), @@ -488,7 +488,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU } test("SPARK-33934: Add SparkFile's root dir to env property PATH") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) val scriptFilePath = copyAndGetResourceFile("test_script.py", ".py").getAbsoluteFile withTempView("v") { val df = Seq( @@ -498,7 +498,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18) df.createTempView("v") - // test 'python /path/to/script.py' with local file + // test 'python3 /path/to/script.py' with local file checkAnswer( sql( s""" @@ -506,7 +506,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU |TRANSFORM(a, b, c, d, e) | ROW FORMAT DELIMITED | FIELDS TERMINATED BY '\t' - | USING 'python $scriptFilePath' AS (a, b, c, d, e) + | USING 'python3 $scriptFilePath' AS (a, b, c, d, e) | ROW FORMAT DELIMITED | FIELDS TERMINATED BY '\t' |FROM v @@ -583,14 +583,14 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU 'd.cast("string"), 'e.cast("string")).collect()) - // test `python script.py` when file added + // test `python3 script.py` when file added checkAnswer( sql( s""" |SELECT TRANSFORM(a, b, c, d, e) | ROW FORMAT DELIMITED | FIELDS TERMINATED BY '\t' - | USING 'python ${scriptFilePath.getName}' AS (a, b, c, d, e) + | USING 'python3 ${scriptFilePath.getName}' AS (a, b, c, d, e) | ROW FORMAT DELIMITED | FIELDS TERMINATED BY '\t' |FROM v @@ -635,7 +635,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU } test("SPARK-36208: TRANSFORM should support ANSI interval (no serde)") { - assume(TestUtils.testCommandAvailable("python")) + assume(TestUtils.testCommandAvailable("python3")) withTempView("v") { val df = Seq( (Period.of(1, 2, 0), Duration.ofDays(1).plusHours(2).plusMinutes(3).plusSeconds(4)) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala index b495dae..591ac02a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala @@ -114,7 +114,7 @@ class BigDataBenchmarkSuite extends HiveComparisonTest { |DROP TABLE IF EXISTS url_counts_partial; |CREATE TABLE url_counts_partial AS | SELECT TRANSFORM (line) - | USING 'python target/url_count.py' as (sourcePage, + | USING 'python3 target/url_count.py' as (sourcePage, | destPage, count) from documents; |DROP TABLE IF EXISTS url_counts_total; |CREATE TABLE url_counts_total AS diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index ba362d9..ba8e6cd 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -1319,7 +1319,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi sql( s"""FROM( | FROM test SELECT TRANSFORM(a, b) - | USING 'python $scriptFilePath/scripts/test_transform.py "\t"' + | USING 'python3 $scriptFilePath/scripts/test_transform.py "\t"' | AS (c STRING, d STRING) |) t |SELECT c @@ -1341,7 +1341,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi |SELECT TRANSFORM(a, b) |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' |WITH SERDEPROPERTIES('field.delim' = '|') - |USING 'python $scriptFilePath/scripts/test_transform.py "|"' + |USING 'python3 $scriptFilePath/scripts/test_transform.py "|"' |AS (c STRING, d STRING) |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' |WITH SERDEPROPERTIES('field.delim' = '|') @@ -2440,6 +2440,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi test("SPARK-25158: " + "Executor accidentally exit because ScriptTransformationWriterThread throw Exception") { + assume(TestUtils.testCommandAvailable("python3")) withTempView("test") { val defaultUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler try { @@ -2459,7 +2460,7 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi val e = intercept[SparkException] { sql( s"""FROM test SELECT TRANSFORM(a) - |USING 'python $scriptFilePath/scripts/test_transform.py "\t"' + |USING 'python3 $scriptFilePath/scripts/test_transform.py "\t"' """.stripMargin).collect() } assert(e.getMessage.contains("Failed to produce data.")) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org