git commit: Updated scripts for auditing releases
Repository: spark
Updated Branches:
refs/heads/master cce77457e - b2bdd0e50
Updated scripts for auditing releases
- Added script to automatically generate change list CHANGES.txt
- Added test for verifying linking against maven distributions of `spark-sql`
and `spark-hive`
- Added SBT projects for testing functionality of `spark-sql` and `spark-hive`
- Fixed issues in existing tests that might have come up because of changes in
Spark 1.0
Author: Tathagata Das tathagata.das1...@gmail.com
Closes #844 from tdas/update-dev-scripts and squashes the following commits:
25090ba [Tathagata Das] Added missing license
e2e20b3 [Tathagata Das] Updated tests for auditing releases.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b2bdd0e5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b2bdd0e5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b2bdd0e5
Branch: refs/heads/master
Commit: b2bdd0e505f1ae3d39c46139f17bd43779ece635
Parents: cce7745
Author: Tathagata Das tathagata.das1...@gmail.com
Authored: Thu May 22 20:48:55 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 20:48:55 2014 -0700
--
dev/audit-release/audit_release.py | 11 +-
.../maven_app_core/src/main/java/SimpleApp.java | 1 +
.../sbt_app_core/src/main/scala/SparkApp.scala | 2 +-
dev/audit-release/sbt_app_hive/build.sbt| 29 +++
dev/audit-release/sbt_app_hive/data.txt | 9 +
.../src/main/resources/hive-site.xml| 213 +++
.../sbt_app_hive/src/main/scala/HiveApp.scala | 57 +
dev/audit-release/sbt_app_sql/build.sbt | 29 +++
.../sbt_app_sql/src/main/scala/SqlApp.scala | 57 +
.../src/main/scala/StreamingApp.scala | 1 -
dev/create-release/generate-changelist.py | 144 +
11 files changed, 547 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/audit_release.py
--
diff --git a/dev/audit-release/audit_release.py
b/dev/audit-release/audit_release.py
index 4a816d4..8c7573b 100755
--- a/dev/audit-release/audit_release.py
+++ b/dev/audit-release/audit_release.py
@@ -93,9 +93,12 @@ original_dir = os.getcwd()
# For each of these modules, we'll test an 'empty' application in sbt and
# maven that links against them. This will catch issues with messed up
# dependencies within those projects.
-modules = [spark-core, spark-bagel, spark-mllib, spark-streaming,
spark-repl,
- spark-graphx, spark-streaming-flume, spark-streaming-kafka,
- spark-streaming-mqtt, spark-streaming-twitter,
spark-streaming-zeromq]
+modules = [
+ spark-core, spark-bagel, spark-mllib, spark-streaming, spark-repl,
+ spark-graphx, spark-streaming-flume, spark-streaming-kafka,
+ spark-streaming-mqtt, spark-streaming-twitter, spark-streaming-zeromq,
+ spark-catalyst, spark-sql, spark-hive
+]
modules = map(lambda m: %s_%s % (m, SCALA_BINARY_VERSION), modules)
# Check for directories that might interfere with tests
@@ -122,7 +125,7 @@ for module in modules:
os.chdir(original_dir)
# SBT application tests
-for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming]:
+for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming,
sbt_app_sql, sbt_app_hive]:
os.chdir(app)
ret = run_cmd(sbt clean run, exit_on_failure=False)
test(ret == 0, sbt application (%s) % app)
http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
--
diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
index 6b65dda..5217689 100644
--- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
+++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
@@ -37,5 +37,6 @@ public class SimpleApp {
System.exit(-1);
}
System.out.println(Test succeeded);
+ sc.stop();
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/b2bdd0e5/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
--
diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
index a89b0d7..77bbd16 100644
--- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
+++ b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
@@ -19,6 +19,7 @@ package main.scala
import scala.util.Try
+import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
@@
git commit: Updated scripts for auditing releases
Repository: spark
Updated Branches:
refs/heads/branch-1.0 c3b406518 - 6541ca24b
Updated scripts for auditing releases
- Added script to automatically generate change list CHANGES.txt
- Added test for verifying linking against maven distributions of `spark-sql`
and `spark-hive`
- Added SBT projects for testing functionality of `spark-sql` and `spark-hive`
- Fixed issues in existing tests that might have come up because of changes in
Spark 1.0
Author: Tathagata Das tathagata.das1...@gmail.com
Closes #844 from tdas/update-dev-scripts and squashes the following commits:
25090ba [Tathagata Das] Added missing license
e2e20b3 [Tathagata Das] Updated tests for auditing releases.
(cherry picked from commit b2bdd0e505f1ae3d39c46139f17bd43779ece635)
Signed-off-by: Tathagata Das tathagata.das1...@gmail.com
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6541ca24
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6541ca24
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6541ca24
Branch: refs/heads/branch-1.0
Commit: 6541ca24b2cf79d6914b9d0bcf7b51bff405369e
Parents: c3b4065
Author: Tathagata Das tathagata.das1...@gmail.com
Authored: Thu May 22 20:48:55 2014 -0700
Committer: Tathagata Das tathagata.das1...@gmail.com
Committed: Thu May 22 20:49:25 2014 -0700
--
dev/audit-release/audit_release.py | 11 +-
.../maven_app_core/src/main/java/SimpleApp.java | 1 +
.../sbt_app_core/src/main/scala/SparkApp.scala | 2 +-
dev/audit-release/sbt_app_hive/build.sbt| 29 +++
dev/audit-release/sbt_app_hive/data.txt | 9 +
.../src/main/resources/hive-site.xml| 213 +++
.../sbt_app_hive/src/main/scala/HiveApp.scala | 57 +
dev/audit-release/sbt_app_sql/build.sbt | 29 +++
.../sbt_app_sql/src/main/scala/SqlApp.scala | 57 +
.../src/main/scala/StreamingApp.scala | 1 -
dev/create-release/generate-changelist.py | 144 +
11 files changed, 547 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/audit_release.py
--
diff --git a/dev/audit-release/audit_release.py
b/dev/audit-release/audit_release.py
index 4a816d4..8c7573b 100755
--- a/dev/audit-release/audit_release.py
+++ b/dev/audit-release/audit_release.py
@@ -93,9 +93,12 @@ original_dir = os.getcwd()
# For each of these modules, we'll test an 'empty' application in sbt and
# maven that links against them. This will catch issues with messed up
# dependencies within those projects.
-modules = [spark-core, spark-bagel, spark-mllib, spark-streaming,
spark-repl,
- spark-graphx, spark-streaming-flume, spark-streaming-kafka,
- spark-streaming-mqtt, spark-streaming-twitter,
spark-streaming-zeromq]
+modules = [
+ spark-core, spark-bagel, spark-mllib, spark-streaming, spark-repl,
+ spark-graphx, spark-streaming-flume, spark-streaming-kafka,
+ spark-streaming-mqtt, spark-streaming-twitter, spark-streaming-zeromq,
+ spark-catalyst, spark-sql, spark-hive
+]
modules = map(lambda m: %s_%s % (m, SCALA_BINARY_VERSION), modules)
# Check for directories that might interfere with tests
@@ -122,7 +125,7 @@ for module in modules:
os.chdir(original_dir)
# SBT application tests
-for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming]:
+for app in [sbt_app_core, sbt_app_graphx, sbt_app_streaming,
sbt_app_sql, sbt_app_hive]:
os.chdir(app)
ret = run_cmd(sbt clean run, exit_on_failure=False)
test(ret == 0, sbt application (%s) % app)
http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
--
diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
index 6b65dda..5217689 100644
--- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
+++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java
@@ -37,5 +37,6 @@ public class SimpleApp {
System.exit(-1);
}
System.out.println(Test succeeded);
+ sc.stop();
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6541ca24/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
--
diff --git a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
index a89b0d7..77bbd16 100644
--- a/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
+++ b/dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala
@@ -19,6 +19,7 @@ package main.scala
import
