git commit: Updated scripts for auditing releases
Repository: spark Updated Branches: refs/heads/branch-1.0 c3b406518 -> 6541ca24b Updated scripts for auditing releases - Added script to automatically generate change list CHANGES.txt - Added test for verifying linking against maven distributions of `spark-sql` and `spark-hive` - Added SBT projects for testing functionality of `spark-sql` and `spark-hive` - Fixed issues in existing tests that might have come up because of changes in Spark 1.0 Author: Tathagata Das Closes #844 from tdas/update-dev-scripts and squashes the following commits: 25090ba [Tathagata Das] Added missing license e2e20b3 [Tathagata Das] Updated tests for auditing releases. (cherry picked from commit b2bdd0e505f1ae3d39c46139f17bd43779ece635) Signed-off-by: Tathagata Das Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6541ca24 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6541ca24 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6541ca24 Branch: refs/heads/branch-1.0 Commit: 6541ca24b2cf79d6914b9d0bcf7b51bff405369e Parents: c3b4065 Author: Tathagata Das Authored: Thu May 22 20:48:55 2014 -0700 Committer: Tathagata Das Committed: Thu May 22 20:49:25 2014 -0700 -- dev/audit-release/audit_release.py | 11 +- .../maven_app_core/src/main/java/SimpleApp.java | 1 + .../sbt_app_core/src/main/scala/SparkApp.scala | 2 +- dev/audit-release/sbt_app_hive/build.sbt| 29 +++ dev/audit-release/sbt_app_hive/data.txt | 9 + .../src/main/resources/hive-site.xml| 213 +++ .../sbt_app_hive/src/main/scala/HiveApp.scala | 57 + dev/audit-release/sbt_app_sql/build.sbt | 29 +++ .../sbt_app_sql/src/main/scala/SqlApp.scala | 57 + .../src/main/scala/StreamingApp.scala | 1 - dev/create-release/generate-changelist.py | 144 + 11 files changed, 547 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/audit_release.py -- diff --git a/dev/audit-release/audit_release.py b/dev/audit-release/audit_release.py index 4a816d4..8c7573b 100755 --- a/dev/audit-release/audit_release.py +++ b/dev/audit-release/audit_release.py @@ -93,9 +93,12 @@ original_dir = os.getcwd() # For each of these modules, we'll test an 'empty' application in sbt and # maven that links against them. This will catch issues with messed up # dependencies within those projects. -modules = ["spark-core", "spark-bagel", "spark-mllib", "spark-streaming", "spark-repl", - "spark-graphx", "spark-streaming-flume", "spark-streaming-kafka", - "spark-streaming-mqtt", "spark-streaming-twitter", "spark-streaming-zeromq"] +modules = [ + "spark-core", "spark-bagel", "spark-mllib", "spark-streaming", "spark-repl", + "spark-graphx", "spark-streaming-flume", "spark-streaming-kafka", + "spark-streaming-mqtt", "spark-streaming-twitter", "spark-streaming-zeromq", + "spark-catalyst", "spark-sql", "spark-hive" +] modules = map(lambda m: "%s_%s" % (m, SCALA_BINARY_VERSION), modules) # Check for directories that might interfere with tests @@ -122,7 +125,7 @@ for module in modules: os.chdir(original_dir) # SBT application tests -for app in ["sbt_app_core", "sbt_app_graphx", "sbt_app_streaming"]: +for app in ["sbt_app_core", "sbt_app_graphx", "sbt_app_streaming", "sbt_app_sql", "sbt_app_hive"]: os.chdir(app) ret = run_cmd("sbt clean run", exit_on_failure=False) test(ret == 0, "sbt application (%s)" % app) http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java -- diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java index 6b65dda..5217689 100644 --- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java +++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java @@ -37,5 +37,6 @@ public class SimpleApp { System.exit(-1); } System.out.println("Test succeeded"); + sc.stop(); } } http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala -- diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala index a89b0d7..77bbd16 100644 --- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala +++ b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala @@ -19,6 +19,7 @@ package main.scala import scala.util.Try +import org.apache.
git commit: Updated scripts for auditing releases
Repository: spark Updated Branches: refs/heads/master cce77457e -> b2bdd0e50 Updated scripts for auditing releases - Added script to automatically generate change list CHANGES.txt - Added test for verifying linking against maven distributions of `spark-sql` and `spark-hive` - Added SBT projects for testing functionality of `spark-sql` and `spark-hive` - Fixed issues in existing tests that might have come up because of changes in Spark 1.0 Author: Tathagata Das Closes #844 from tdas/update-dev-scripts and squashes the following commits: 25090ba [Tathagata Das] Added missing license e2e20b3 [Tathagata Das] Updated tests for auditing releases. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b2bdd0e5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b2bdd0e5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b2bdd0e5 Branch: refs/heads/master Commit: b2bdd0e505f1ae3d39c46139f17bd43779ece635 Parents: cce7745 Author: Tathagata Das Authored: Thu May 22 20:48:55 2014 -0700 Committer: Tathagata Das Committed: Thu May 22 20:48:55 2014 -0700 -- dev/audit-release/audit_release.py | 11 +- .../maven_app_core/src/main/java/SimpleApp.java | 1 + .../sbt_app_core/src/main/scala/SparkApp.scala | 2 +- dev/audit-release/sbt_app_hive/build.sbt| 29 +++ dev/audit-release/sbt_app_hive/data.txt | 9 + .../src/main/resources/hive-site.xml| 213 +++ .../sbt_app_hive/src/main/scala/HiveApp.scala | 57 + dev/audit-release/sbt_app_sql/build.sbt | 29 +++ .../sbt_app_sql/src/main/scala/SqlApp.scala | 57 + .../src/main/scala/StreamingApp.scala | 1 - dev/create-release/generate-changelist.py | 144 + 11 files changed, 547 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/audit_release.py -- diff --git a/dev/audit-release/audit_release.py b/dev/audit-release/audit_release.py index 4a816d4..8c7573b 100755 --- a/dev/audit-release/audit_release.py +++ b/dev/audit-release/audit_release.py @@ -93,9 +93,12 @@ original_dir = os.getcwd() # For each of these modules, we'll test an 'empty' application in sbt and # maven that links against them. This will catch issues with messed up # dependencies within those projects. -modules = ["spark-core", "spark-bagel", "spark-mllib", "spark-streaming", "spark-repl", - "spark-graphx", "spark-streaming-flume", "spark-streaming-kafka", - "spark-streaming-mqtt", "spark-streaming-twitter", "spark-streaming-zeromq"] +modules = [ + "spark-core", "spark-bagel", "spark-mllib", "spark-streaming", "spark-repl", + "spark-graphx", "spark-streaming-flume", "spark-streaming-kafka", + "spark-streaming-mqtt", "spark-streaming-twitter", "spark-streaming-zeromq", + "spark-catalyst", "spark-sql", "spark-hive" +] modules = map(lambda m: "%s_%s" % (m, SCALA_BINARY_VERSION), modules) # Check for directories that might interfere with tests @@ -122,7 +125,7 @@ for module in modules: os.chdir(original_dir) # SBT application tests -for app in ["sbt_app_core", "sbt_app_graphx", "sbt_app_streaming"]: +for app in ["sbt_app_core", "sbt_app_graphx", "sbt_app_streaming", "sbt_app_sql", "sbt_app_hive"]: os.chdir(app) ret = run_cmd("sbt clean run", exit_on_failure=False) test(ret == 0, "sbt application (%s)" % app) http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java -- diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java index 6b65dda..5217689 100644 --- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java +++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java @@ -37,5 +37,6 @@ public class SimpleApp { System.exit(-1); } System.out.println("Test succeeded"); + sc.stop(); } } http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala -- diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala index a89b0d7..77bbd16 100644 --- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala +++ b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala @@ -19,6 +19,7 @@ package main.scala import scala.util.Try +import org.apache.spark.SparkConf import org.apache.spark.SparkContext import org.apache.spark.SparkContext._ @@ -30,7 +31