Updated Branches: refs/heads/master a787428ef -> 79e6c896b
CRUNCH-225: Added support for building using Scala 2.10 and 2.9. Also removed unused build.sbt file Signed-off-by: Micah Whitacre <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/79e6c896 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/79e6c896 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/79e6c896 Branch: refs/heads/master Commit: 79e6c896bbe0dd5fb57b5358d3f0600657804bb9 Parents: a787428 Author: Micah Whitacre <[email protected]> Authored: Wed Nov 13 20:22:58 2013 -0600 Committer: Micah Whitacre <[email protected]> Committed: Wed Nov 13 20:29:03 2013 -0600 ---------------------------------------------------------------------- crunch-scrunch/build.sbt | 50 -------------------- crunch-scrunch/pom.xml | 46 +++++++++++++++--- .../org/apache/crunch/scrunch/CogroupTest.scala | 2 +- .../org/apache/crunch/scrunch/CrunchSuite.scala | 39 +++++++++++++++ .../org/apache/crunch/scrunch/JoinTest.scala | 2 +- .../apache/crunch/scrunch/PCollectionTest.scala | 2 +- .../org/apache/crunch/scrunch/PTableTest.scala | 2 +- .../crunch/scrunch/PageRankClassTest.scala | 2 +- .../apache/crunch/scrunch/PageRankTest.scala | 2 +- .../crunch/scrunch/PipelineAppClosureTest.scala | 2 +- .../apache/crunch/scrunch/PipelineAppTest.scala | 2 +- .../org/apache/crunch/scrunch/TopTest.scala | 2 +- .../org/apache/crunch/scrunch/UnionTest.scala | 2 +- .../apache/crunch/scrunch/WordCountTest.scala | 2 +- .../interpreter/InterpreterJarTest.scala | 3 +- .../org/apache/crunch/test/TemporaryPath.java | 8 ++++ pom.xml | 6 --- 17 files changed, 100 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/build.sbt ---------------------------------------------------------------------- diff --git a/crunch-scrunch/build.sbt b/crunch-scrunch/build.sbt deleted file mode 100644 index 21b52ce..0000000 --- a/crunch-scrunch/build.sbt +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -name := "scrunch" - -version := "0.3.0" - -scalaVersion := "2.9.2" - -resolvers ++= Seq( - "Local Maven Repository" at "file://"+Path.userHome.absolutePath+"/.m2/repository", -) - -libraryDependencies ++= Seq( - "org.apache.crunch" % "crunch" % "0.3.0" excludeAll( - ExclusionRule(organization = "com.sun.jdmk"), - ExclusionRule(organization = "com.sun.jmx"), - ExclusionRule(organization = "javax.jms") - ), - "org.apache.hadoop" % "hadoop-client" % "1.0.3" % "provided" excludeAll( - ExclusionRule(organization = "com.sun.jdmk"), - ExclusionRule(organization = "com.sun.jmx"), - ExclusionRule(organization = "javax.jms") - ), - "org.apache.hbase" % "hbase" % "0.90.4" % "provided" excludeAll( - ExclusionRule(organization = "org.apache.hadoop"), - ExclusionRule(organization = "commons-logging"), - ExclusionRule(organization = "com.google.guava"), - ExclusionRule(organization = "log4j"), - ExclusionRule(organization = "org.slf4j") - ), - "junit" % "junit" % "4.8.1" % "test", - "org.scalatest" % "scalatest_2.9.2" % "1.7.2" % "test" -) - -parallelExecution in Test := false http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/pom.xml ---------------------------------------------------------------------- diff --git a/crunch-scrunch/pom.xml b/crunch-scrunch/pom.xml index 10c73c9..b6d7b54 100644 --- a/crunch-scrunch/pom.xml +++ b/crunch-scrunch/pom.xml @@ -60,11 +60,6 @@ under the License. <artifactId>slf4j-log4j12</artifactId> </dependency> <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.version}</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.crunch</groupId> <artifactId>crunch-test</artifactId> <scope>test</scope> @@ -209,6 +204,45 @@ under the License. </plugins> </build> </profile> + <profile> + <id>scala-2.9.2</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <scala.version>2.9.2</scala.version> + <scalatest.version>1.7.2</scalatest.version> + </properties> + <dependencies> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.version}</artifactId> + </dependency> + </dependencies> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.version}</artifactId> + <version>${scalatest.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + </profile> + <profile> + <id>scala-2.10</id> + <properties> + <scala.version>2.10.2</scala.version> + <scalatest.version>1.9.2</scalatest.version> + </properties> + <dependencies> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_2.10</artifactId> + <version>${scalatest.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + </profile> </profiles> - </project> http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala index 170b8c9..40df05b 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala @@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class CogroupTest extends CrunchTestSupport with JUnitSuite { +class CogroupTest extends CrunchSuite { lazy val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration) def wordCount(fileName: String) = { http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala new file mode 100644 index 0000000..37a4fe2 --- /dev/null +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.scrunch + +import org.scalatest.junit.JUnitSuite +import org.junit.{After, Before, Rule} +import org.apache.crunch.test.TemporaryPath + +class CrunchSuite extends JUnitSuite{ + + val tempDir = new TemporaryPath("crunch.tmp.dir", "hadoop.tmp.dir"); + + def getFolder() = { + tempDir + } + + @Before def initialize() { + tempDir.create() + } + + @After def cleanup() { + tempDir.delete() + } +} http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala index 0770ba3..bd640eb 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala @@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class JoinTest extends CrunchTestSupport with JUnitSuite { +class JoinTest extends CrunchSuite { lazy val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration) def wordCount(fileName: String) = { http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala index 94ac917..6d42ed6 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala @@ -27,7 +27,7 @@ import org.scalatest.junit.JUnitSuite /** * Tests functionality of Scala PCollection. */ -class PCollectionTest extends CrunchTestSupport with JUnitSuite { +class PCollectionTest extends CrunchSuite { // Number of lines in the Shakespeare data set. val linesInShakespeare: Int = 3667 http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala index cd644b1..35b2a8d 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala @@ -27,7 +27,7 @@ import org.scalatest.junit.JUnitSuite /** * Tests functionality of Scala PTable. */ -class PTableTest extends CrunchTestSupport with JUnitSuite { +class PTableTest extends CrunchSuite { /** * Gets a PCollection containing the lines from the tens data set. http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala index 847d65b..55e7783 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala @@ -61,7 +61,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Flo } } -class PageRankClassTest extends CrunchTestSupport with JUnitSuite { +class PageRankClassTest extends CrunchSuite { lazy val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration) http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala index 02e3d82..833fce8 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala @@ -49,7 +49,7 @@ class CachingPageRankFn extends DoFn[P[String, (Float, Float, List[String])], P[ } } -class PageRankTest extends CrunchTestSupport with JUnitSuite { +class PageRankTest extends CrunchSuite { lazy val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration) def initialInput(fileName: String) = { http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala index e0702e4..4666c3c 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala @@ -27,7 +27,7 @@ import _root_.org.junit.Test * Test that verifies that a Scala PipelineApp can properly send some side data as part of a * function closure. */ -class PipelineClosureAppTest extends CrunchTestSupport with JUnitSuite { +class PipelineClosureAppTest extends CrunchSuite { /** * A simple pipeline application that divides each element of a PCollection of numbers by http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala index db49c92..fffd529 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala @@ -39,7 +39,7 @@ object WordCount extends PipelineApp { } } -class PipelineAppTest extends CrunchTestSupport with JUnitSuite { +class PipelineAppTest extends CrunchSuite { @Test def run { val args = new Array[String](3) args(0) = tempDir.copyResourceFileName("shakes.txt") http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala index 0c7a992..ca728e8 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala @@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class TopTest extends CrunchTestSupport with JUnitSuite { +class TopTest extends CrunchSuite { @Test def topInMem { val ptable = Mem.tableOf(("foo", 17), ("bar", 29), ("baz", 1729)) http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala index e8b3fab..a507e37 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala @@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class UnionTest extends CrunchTestSupport with JUnitSuite { +class UnionTest extends CrunchSuite { lazy val pipeline = Pipeline.mapReduce[UnionTest](tempDir.getDefaultConfiguration) def wordCount(col: PCollection[String]) = { http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala index a2560d1..2cc1457 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala @@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport import org.scalatest.junit.JUnitSuite import _root_.org.junit.Test -class WordCountTest extends CrunchTestSupport with JUnitSuite { +class WordCountTest extends CrunchSuite { @Test def wordCount { val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration) val input = tempDir.copyResourceFileName("shakes.txt") http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala index 48aae00..5ebc303 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala @@ -29,11 +29,12 @@ import org.junit.Assert.assertNotNull import org.junit.Test import org.apache.crunch.test.CrunchTestSupport import org.scalatest.junit.JUnitSuite +import org.apache.crunch.scrunch.CrunchSuite /** * Tests creating jars from a {@link scala.tools.nsc.io.VirtualDirectory}. */ -class InterpreterJarTest extends CrunchTestSupport with JUnitSuite { +class InterpreterJarTest extends CrunchSuite { /** * Tests transforming a virtual directory into a temporary jar file. http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java ---------------------------------------------------------------------- diff --git a/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java b/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java index 3cd175f..1d25fc7 100644 --- a/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java +++ b/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java @@ -59,6 +59,14 @@ public final class TemporaryPath extends ExternalResource { } } + public void create() throws Throwable { + tmp.create(); + } + + public void delete() { + tmp.delete(); + } + @Override public Statement apply(Statement base, Description description) { return tmp.apply(base, description); http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 338b4f3..7e288d3 100644 --- a/pom.xml +++ b/pom.xml @@ -352,12 +352,6 @@ under the License. <artifactId>jline</artifactId> <version>${scala.version}</version> </dependency> - - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.version}</artifactId> - <version>${scalatest.version}</version> - </dependency> </dependencies> </dependencyManagement>
