Repository: spark Updated Branches: refs/heads/master a9ecd0614 -> 94fc57afd
[SPARK-10300] [BUILD] [TESTS] Add support for test tags in run-tests.py. Author: Marcelo Vanzin <[email protected]> Closes #8775 from vanzin/SPARK-10300. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/94fc57af Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/94fc57af Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/94fc57af Branch: refs/heads/master Commit: 94fc57afdf8ac6be35f13956232b6cf58857d047 Parents: a9ecd06 Author: Marcelo Vanzin <[email protected]> Authored: Wed Oct 7 14:11:21 2015 -0700 Committer: Marcelo Vanzin <[email protected]> Committed: Wed Oct 7 14:11:21 2015 -0700 ---------------------------------------------------------------------- bagel/pom.xml | 4 ++ core/pom.xml | 14 ++---- dev/run-tests.py | 19 +++++++- dev/sparktestsupport/modules.py | 24 +++++++++- external/flume-sink/pom.xml | 4 ++ external/flume/pom.xml | 10 +--- external/kafka/pom.xml | 10 +--- external/mqtt/pom.xml | 14 ++---- external/twitter/pom.xml | 10 +--- external/zeromq/pom.xml | 10 +--- extras/java8-tests/pom.xml | 10 +--- extras/kinesis-asl/pom.xml | 5 +- graphx/pom.xml | 4 ++ launcher/pom.xml | 10 ++-- mllib/pom.xml | 14 ++---- network/common/pom.xml | 14 ++---- network/shuffle/pom.xml | 10 +--- network/yarn/pom.xml | 4 ++ pom.xml | 24 +++++++++- project/SparkBuild.scala | 19 ++++++-- repl/pom.xml | 4 ++ sql/catalyst/pom.xml | 4 ++ sql/core/pom.xml | 9 ++-- sql/hive-thriftserver/pom.xml | 4 ++ .../hive/execution/HiveCompatibilitySuite.scala | 2 + sql/hive/pom.xml | 9 ++-- .../spark/sql/hive/client/VersionsSuite.scala | 2 + streaming/pom.xml | 14 ++---- tags/pom.xml | 50 ++++++++++++++++++++ .../org/apache/spark/tags/ExtendedHiveTest.java | 26 ++++++++++ .../org/apache/spark/tags/ExtendedYarnTest.java | 26 ++++++++++ unsafe/pom.xml | 10 +--- yarn/pom.xml | 4 ++ .../spark/deploy/yarn/YarnClusterSuite.scala | 2 + .../yarn/YarnShuffleIntegrationSuite.scala | 2 + 35 files changed, 267 insertions(+), 134 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/bagel/pom.xml ---------------------------------------------------------------------- diff --git a/bagel/pom.xml b/bagel/pom.xml index 3baf8d4..672e946 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -52,6 +52,10 @@ <artifactId>scalacheck_${scala.binary.version}</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index e31d90f..c0af98a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -332,16 +332,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-test</artifactId> <scope>test</scope> @@ -362,6 +352,10 @@ <artifactId>py4j</artifactId> <version>0.8.2.1</version> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/dev/run-tests.py ---------------------------------------------------------------------- diff --git a/dev/run-tests.py b/dev/run-tests.py index d8b22e1..1a81658 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -118,6 +118,14 @@ def determine_modules_to_test(changed_modules): return modules_to_test.union(set(changed_modules)) +def determine_tags_to_exclude(changed_modules): + tags = [] + for m in modules.all_modules: + if m not in changed_modules: + tags += m.test_tags + return tags + + # ------------------------------------------------------------------------------------------------- # Functions for working with subprocesses and shell tools # ------------------------------------------------------------------------------------------------- @@ -369,6 +377,7 @@ def detect_binary_inop_with_mima(): def run_scala_tests_maven(test_profiles): mvn_test_goals = ["test", "--fail-at-end"] + profiles_and_goals = test_profiles + mvn_test_goals print("[info] Running Spark tests using Maven with these arguments: ", @@ -392,7 +401,7 @@ def run_scala_tests_sbt(test_modules, test_profiles): exec_sbt(profiles_and_goals) -def run_scala_tests(build_tool, hadoop_version, test_modules): +def run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags): """Function to properly execute all tests passed in as a set from the `determine_test_suites` function""" set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS") @@ -401,6 +410,10 @@ def run_scala_tests(build_tool, hadoop_version, test_modules): test_profiles = get_hadoop_profiles(hadoop_version) + \ list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules))) + + if excluded_tags: + test_profiles += ['-Dtest.exclude.tags=' + ",".join(excluded_tags)] + if build_tool == "maven": run_scala_tests_maven(test_profiles) else: @@ -500,8 +513,10 @@ def main(): target_branch = os.environ["ghprbTargetBranch"] changed_files = identify_changed_files_from_git_commits("HEAD", target_branch=target_branch) changed_modules = determine_modules_for_files(changed_files) + excluded_tags = determine_tags_to_exclude(changed_modules) if not changed_modules: changed_modules = [modules.root] + excluded_tags = [] print("[info] Found the following changed modules:", ", ".join(x.name for x in changed_modules)) @@ -541,7 +556,7 @@ def main(): detect_binary_inop_with_mima() # run the test suites - run_scala_tests(build_tool, hadoop_version, test_modules) + run_scala_tests(build_tool, hadoop_version, test_modules, excluded_tags) modules_with_python_tests = [m for m in test_modules if m.python_test_goals] if modules_with_python_tests: http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/dev/sparktestsupport/modules.py ---------------------------------------------------------------------- diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 346452f..d65547e 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -31,7 +31,7 @@ class Module(object): def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(), environ={}, sbt_test_goals=(), python_test_goals=(), blacklisted_python_implementations=(), - should_run_r_tests=False): + test_tags=(), should_run_r_tests=False): """ Define a new module. @@ -50,6 +50,8 @@ class Module(object): :param blacklisted_python_implementations: A set of Python implementations that are not supported by this module's Python components. The values in this set should match strings returned by Python's `platform.python_implementation()`. + :param test_tags A set of tags that will be excluded when running unit tests if the module + is not explicitly changed. :param should_run_r_tests: If true, changes in this module will trigger all R tests. """ self.name = name @@ -60,6 +62,7 @@ class Module(object): self.environ = environ self.python_test_goals = python_test_goals self.blacklisted_python_implementations = blacklisted_python_implementations + self.test_tags = test_tags self.should_run_r_tests = should_run_r_tests self.dependent_modules = set() @@ -85,6 +88,9 @@ sql = Module( "catalyst/test", "sql/test", "hive/test", + ], + test_tags=[ + "org.apache.spark.tags.ExtendedHiveTest" ] ) @@ -398,6 +404,22 @@ ec2 = Module( ) +yarn = Module( + name="yarn", + dependencies=[], + source_file_regexes=[ + "yarn/", + "network/yarn/", + ], + sbt_test_goals=[ + "yarn/test", + "network-yarn/test", + ], + test_tags=[ + "org.apache.spark.tags.ExtendedYarnTest" + ] +) + # The root module is a dummy module which is used to run all of the tests. # No other modules should directly depend on this module. root = Module( http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/external/flume-sink/pom.xml ---------------------------------------------------------------------- diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index d7c2ac4..75113ff 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -90,6 +90,10 @@ <version>3.4.0.Final</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/external/flume/pom.xml ---------------------------------------------------------------------- diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 132062f..57f8360 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -67,14 +67,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/external/kafka/pom.xml ---------------------------------------------------------------------- diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index 05abd9e..79258c1 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -87,14 +87,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/external/mqtt/pom.xml ---------------------------------------------------------------------- diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index 05e6338..59fba8b 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -59,21 +59,15 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.activemq</groupId> <artifactId>activemq-core</artifactId> <version>5.7.0</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/external/twitter/pom.xml ---------------------------------------------------------------------- diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 244ad58..4c22ec8 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -59,14 +59,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/external/zeromq/pom.xml ---------------------------------------------------------------------- diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index 171df86..02d6b81 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -58,14 +58,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/extras/java8-tests/pom.xml ---------------------------------------------------------------------- diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index 81794a8..4ce90e7 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -59,14 +59,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/extras/kinesis-asl/pom.xml ---------------------------------------------------------------------- diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index 6dd8ff6..ef72d97 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -75,9 +75,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/graphx/pom.xml ---------------------------------------------------------------------- diff --git a/graphx/pom.xml b/graphx/pom.xml index 202fc19..987b831 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -66,6 +66,10 @@ <artifactId>scalacheck_${scala.binary.version}</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/launcher/pom.xml ---------------------------------------------------------------------- diff --git a/launcher/pom.xml b/launcher/pom.xml index ed38e66..d595d74 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -43,11 +43,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -63,6 +58,11 @@ <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> + <!-- Not needed by the test code, but referenced by SparkSubmit which is used by the tests. --> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/mllib/pom.xml ---------------------------------------------------------------------- diff --git a/mllib/pom.xml b/mllib/pom.xml index 22c0c60..7013912 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -95,16 +95,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -131,6 +121,10 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <profiles> <profile> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/network/common/pom.xml ---------------------------------------------------------------------- diff --git a/network/common/pom.xml b/network/common/pom.xml index 1cc054a..9af6cc5 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -65,21 +65,15 @@ <!-- Test dependencies --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/network/shuffle/pom.xml ---------------------------------------------------------------------- diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index 7a66c96..70ba5cb 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -79,14 +79,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>log4j</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/network/yarn/pom.xml ---------------------------------------------------------------------- diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index e745180..541ed9a 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -44,6 +44,10 @@ <artifactId>spark-network-shuffle_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> <!-- Provided dependencies --> <dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index d04ed1e..445e65c 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,7 @@ </mailingLists> <modules> + <module>tags</module> <module>core</module> <module>bagel</module> <!-- Deprecated --> <module>graphx</module> @@ -181,6 +182,7 @@ <libthrift.version>0.9.2</libthrift.version> <test.java.home>${java.home}</test.java.home> + <test.exclude.tags></test.exclude.tags> <!-- Dependency scopes that can be overridden by enabling certain profiles. These profiles are @@ -339,10 +341,26 @@ <artifactId>scalatest_${scala.binary.version}</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.novocode</groupId> + <artifactId>junit-interface</artifactId> + <scope>test</scope> + </dependency> </dependencies> <dependencyManagement> <dependencies> <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> <version>${chill.version}</version> @@ -742,7 +760,7 @@ <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> - <version>4.10</version> + <version>4.11</version> <scope>test</scope> </dependency> <dependency> @@ -760,7 +778,7 @@ <dependency> <groupId>com.novocode</groupId> <artifactId>junit-interface</artifactId> - <version>0.10</version> + <version>0.11</version> <scope>test</scope> </dependency> <dependency> @@ -1915,6 +1933,7 @@ <test.src.tables>src</test.src.tables> </systemProperties> <failIfNoTests>false</failIfNoTests> + <excludedGroups>${test.exclude.tags}</excludedGroups> </configuration> </plugin> <!-- Scalatest runs all Scala tests --> @@ -1952,6 +1971,7 @@ <!-- Needed by sql/hive tests. --> <test.src.tables>__not_used__</test.src.tables> </systemProperties> + <tagsToExclude>${test.exclude.tags}</tagsToExclude> </configuration> <executions> <execution> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/project/SparkBuild.scala ---------------------------------------------------------------------- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 901cfa5..1339980 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -35,11 +35,11 @@ object BuildCommons { val allProjects@Seq(bagel, catalyst, core, graphx, hive, hiveThriftServer, mllib, repl, sql, networkCommon, networkShuffle, streaming, streamingFlumeSink, streamingFlume, streamingKafka, - streamingMqtt, streamingTwitter, streamingZeromq, launcher, unsafe) = + streamingMqtt, streamingTwitter, streamingZeromq, launcher, unsafe, testTags) = Seq("bagel", "catalyst", "core", "graphx", "hive", "hive-thriftserver", "mllib", "repl", "sql", "network-common", "network-shuffle", "streaming", "streaming-flume-sink", "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter", - "streaming-zeromq", "launcher", "unsafe").map(ProjectRef(buildLocation, _)) + "streaming-zeromq", "launcher", "unsafe", "test-tags").map(ProjectRef(buildLocation, _)) val optionallyEnabledProjects@Seq(yarn, yarnStable, java8Tests, sparkGangliaLgpl, streamingKinesisAsl) = Seq("yarn", "yarn-stable", "java8-tests", "ganglia-lgpl", @@ -202,7 +202,7 @@ object SparkBuild extends PomBuild { (allProjects ++ optionallyEnabledProjects).foreach(enable(TestSettings.settings)) allProjects.filterNot(x => Seq(spark, hive, hiveThriftServer, catalyst, repl, - networkCommon, networkShuffle, networkYarn, unsafe).contains(x)).foreach { + networkCommon, networkShuffle, networkYarn, unsafe, testTags).contains(x)).foreach { x => enable(MimaBuild.mimaSettings(sparkHome, x))(x) } @@ -567,11 +567,20 @@ object TestSettings { javaOptions in Test ++= "-Xmx3g -Xss4096k -XX:PermSize=128M -XX:MaxNewSize=256m -XX:MaxPermSize=1g" .split(" ").toSeq, javaOptions += "-Xmx3g", + // Exclude tags defined in a system property + testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, + sys.props.get("test.exclude.tags").map { tags => + tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq + }.getOrElse(Nil): _*), + testOptions in Test += Tests.Argument(TestFrameworks.JUnit, + sys.props.get("test.exclude.tags").map { tags => + Seq("--exclude-categories=" + tags) + }.getOrElse(Nil): _*), // Show full stack trace and duration in test cases. testOptions in Test += Tests.Argument("-oDF"), - testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), + testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), // Enable Junit testing. - libraryDependencies += "com.novocode" % "junit-interface" % "0.9" % "test", + libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test", // Only allow one test at a time, even across projects, since they run in the same JVM parallelExecution in Test := false, // Make sure the test temp directory exists. http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/repl/pom.xml ---------------------------------------------------------------------- diff --git a/repl/pom.xml b/repl/pom.xml index 5cf416a..fb0a0e1 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -91,6 +91,10 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. --> <dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/sql/catalyst/pom.xml ---------------------------------------------------------------------- diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 6cfd53e..61d6fc6 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -55,6 +55,10 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> <artifactId>spark-unsafe_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/sql/core/pom.xml ---------------------------------------------------------------------- diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 465aa3a..c96855e 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -61,6 +61,10 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> + <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-column</artifactId> </dependency> @@ -74,11 +78,6 @@ <version>${fasterxml.jackson.version}</version> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/sql/hive-thriftserver/pom.xml ---------------------------------------------------------------------- diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index f7fe085..b5b2143 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -93,6 +93,10 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala index ab309e0..8f29fa9 100644 --- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala +++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala @@ -25,10 +25,12 @@ import org.scalatest.BeforeAndAfter import org.apache.spark.sql.SQLConf import org.apache.spark.sql.hive.test.TestHive +import org.apache.spark.tags.ExtendedHiveTest /** * Runs the test cases that are included in the hive distribution. */ +@ExtendedHiveTest class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter { // TODO: bundle in jar files... get from classpath private lazy val hiveQueryDir = TestHive.getHiveFile( http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/sql/hive/pom.xml ---------------------------------------------------------------------- diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index ac67fe5..d96f3e2 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -58,6 +58,10 @@ <artifactId>spark-sql_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> <!-- <dependency> <groupId>com.google.guava</groupId> @@ -161,11 +165,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.binary.version}</artifactId> <type>test-jar</type> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala index f0bb770..2da22ec 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.{Logging, SparkFunSuite} import org.apache.spark.sql.catalyst.expressions.{NamedExpression, Literal, AttributeReference, EqualTo} import org.apache.spark.sql.catalyst.util.quietly import org.apache.spark.sql.types.IntegerType +import org.apache.spark.tags.ExtendedHiveTest import org.apache.spark.util.Utils /** @@ -32,6 +33,7 @@ import org.apache.spark.util.Utils * sure that reflective calls are not throwing NoSuchMethod error, but the actually functionality * is not fully tested. */ +@ExtendedHiveTest class VersionsSuite extends SparkFunSuite with Logging { // Do not use a temp path here to speed up subsequent executions of the unit test during http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/streaming/pom.xml ---------------------------------------------------------------------- diff --git a/streaming/pom.xml b/streaming/pom.xml index 5cc9001..145c8a7 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -47,6 +47,10 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. --> <dependency> @@ -85,20 +89,10 @@ <scope>test</scope> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.seleniumhq.selenium</groupId> <artifactId>selenium-java</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/tags/pom.xml ---------------------------------------------------------------------- diff --git a/tags/pom.xml b/tags/pom.xml new file mode 100644 index 0000000..ca93722 --- /dev/null +++ b/tags/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.spark</groupId> + <artifactId>spark-parent_2.10</artifactId> + <version>1.6.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_2.10</artifactId> + <packaging>jar</packaging> + <name>Spark Project Test Tags</name> + <url>http://spark.apache.org/</url> + <properties> + <sbt.project.name>test-tags</sbt.project.name> + </properties> + + <dependencies> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.binary.version}</artifactId> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> + <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> + </build> +</project> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/tags/src/main/java/org/apache/spark/tags/ExtendedHiveTest.java ---------------------------------------------------------------------- diff --git a/tags/src/main/java/org/apache/spark/tags/ExtendedHiveTest.java b/tags/src/main/java/org/apache/spark/tags/ExtendedHiveTest.java new file mode 100644 index 0000000..1b0c416 --- /dev/null +++ b/tags/src/main/java/org/apache/spark/tags/ExtendedHiveTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.tags; + +import java.lang.annotation.*; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface ExtendedHiveTest { } http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/tags/src/main/java/org/apache/spark/tags/ExtendedYarnTest.java ---------------------------------------------------------------------- diff --git a/tags/src/main/java/org/apache/spark/tags/ExtendedYarnTest.java b/tags/src/main/java/org/apache/spark/tags/ExtendedYarnTest.java new file mode 100644 index 0000000..2a631bf --- /dev/null +++ b/tags/src/main/java/org/apache/spark/tags/ExtendedYarnTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.tags; + +import java.lang.annotation.*; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +public @interface ExtendedYarnTest { } http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/unsafe/pom.xml ---------------------------------------------------------------------- diff --git a/unsafe/pom.xml b/unsafe/pom.xml index 066abe9..caf1f77 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -56,14 +56,8 @@ <!-- Test dependencies --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.novocode</groupId> - <artifactId>junit-interface</artifactId> - <scope>test</scope> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>org.mockito</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/yarn/pom.xml ---------------------------------------------------------------------- diff --git a/yarn/pom.xml b/yarn/pom.xml index d8e4a4b..3eadacb 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -52,6 +52,10 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-api</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala ---------------------------------------------------------------------- diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index b5a42fd..f1601cd 100644 --- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.launcher.TestClasspathBuilder import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationStart, SparkListenerExecutorAdded} import org.apache.spark.scheduler.cluster.ExecutorInfo +import org.apache.spark.tags.ExtendedYarnTest import org.apache.spark.util.Utils /** @@ -39,6 +40,7 @@ import org.apache.spark.util.Utils * applications, and require the Spark assembly to be built before they can be successfully * run. */ +@ExtendedYarnTest class YarnClusterSuite extends BaseYarnClusterSuite { override def newYarnConfig(): YarnConfiguration = new YarnConfiguration() http://git-wip-us.apache.org/repos/asf/spark/blob/94fc57af/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala ---------------------------------------------------------------------- diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala index 8d9c9b3..a85e577 100644 --- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala +++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnShuffleIntegrationSuite.scala @@ -28,10 +28,12 @@ import org.scalatest.Matchers import org.apache.spark._ import org.apache.spark.network.shuffle.ShuffleTestAccessor import org.apache.spark.network.yarn.{YarnShuffleService, YarnTestAccessor} +import org.apache.spark.tags.ExtendedYarnTest /** * Integration test for the external shuffle service with a yarn mini-cluster */ +@ExtendedYarnTest class YarnShuffleIntegrationSuite extends BaseYarnClusterSuite { override def newYarnConfig(): YarnConfiguration = { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
