Repository: crunch Updated Branches: refs/heads/master ded504eb1 -> 0e7e0d0b0
CRUNCH-444: Generate Tuple5..Tuple22 and generalized product Aggregators for Scrunch Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/0e7e0d0b Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/0e7e0d0b Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/0e7e0d0b Branch: refs/heads/master Commit: 0e7e0d0b079c2ff51dbcd483402802917ac79f8f Parents: ded504e Author: Josh Wills <[email protected]> Authored: Tue Jul 15 20:04:26 2014 -0700 Committer: Josh Wills <[email protected]> Committed: Thu Jul 17 12:12:45 2014 -0700 ---------------------------------------------------------------------- .../src/codegen/make_tuple_conversions.py | 51 +++++++ .../src/codegen/make_tuple_ptypefamily.py | 60 +++++++++ .../scrunch/AggregatorsIntegrationTest.scala | 41 ++++++ .../org/apache/crunch/scrunch/Aggregators.scala | 83 ++++++------ .../org/apache/crunch/scrunch/Conversions.scala | 2 +- .../scrunch/GeneratedTupleConversions.scala | 129 ++++++++++++++++++ .../scrunch/GeneratedTuplePTypeFamily.scala | 135 +++++++++++++++++++ .../org/apache/crunch/scrunch/PTypeFamily.scala | 20 ++- .../org/apache/crunch/scrunch/TupleNTest.scala | 32 +++++ 9 files changed, 505 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/codegen/make_tuple_conversions.py ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/codegen/make_tuple_conversions.py b/crunch-scrunch/src/codegen/make_tuple_conversions.py new file mode 100755 index 0000000..114ff1e --- /dev/null +++ b/crunch-scrunch/src/codegen/make_tuple_conversions.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +print """/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */""" +print "package org.apache.crunch.scrunch\n" +print "trait GeneratedTupleConversions {" +for j in range(5, 23): + lets = letters[0: j] + types = ", ".join(lets) + pth = ", ".join(["%s: PTypeH" % x for x in lets]) + print " implicit def tuple%d[%s] = new PTypeH[(%s)] {" % (j, pth, types) + print " def get(ptf: PTypeFamily) = {" + implicits = ", ".join(["implicitly[PTypeH[%s]].get(ptf)" % x for x in lets]) + print " ptf.tuple%d(%s)" % (j, implicits) + print " }" + print " }\n" +print "}" http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py new file mode 100755 index 0000000..fc7b989 --- /dev/null +++ b/crunch-scrunch/src/codegen/make_tuple_ptypefamily.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + +print """/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */""" +print "package org.apache.crunch.scrunch\n" +print "import org.apache.crunch.TupleN" +print "import org.apache.crunch.types.PType\n" + +print "trait GeneratedTuplePTypeFamily extends BasePTypeFamily {" +print " import GeneratedTupleHelper._\n" +for j in range(5, 23): + lets = letters[0:j] + types = ", ".join(lets) + args = ", ".join(["p%d: PType[%s]" % (x, l) for (x, l) in enumerate(lets)]) + print " def tuple%d[%s](%s) = {" % (j, types, args) + + inout = ",".join(["t.get(%d).asInstanceOf[%s]" % (x, l) for (x, l) in enumerate(lets)]) + print " val in = (t: TupleN) => (%s)" % inout + + outin = ", ".join(lets) + outout = ", ".join(["t._%d" % (1 + x) for x in range(j)]) + print " val out = (t: (%s)) => tupleN(%s)" % (outin, outout) + derout = ", ".join(["p%d" % x for x in range(j)]) + print " derived(classOf[(%s)], in, out, ptf.tuples(%s))" % (outin, derout) + print " }\n" +print "}\n" http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala new file mode 100644 index 0000000..94a6e12 --- /dev/null +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/AggregatorsIntegrationTest.scala @@ -0,0 +1,41 @@ +/* + * * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.crunch.scrunch + +import org.junit.Test + +class AggregatorsIntegrationTest extends CrunchSuite { + @Test def productAggregators { + val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration) + val input = tempDir.copyResourceFileName("shakes.txt") + + val fcc = pipeline.read(From.textFile(input)) + .flatMap(_.toLowerCase.split("\\s+")) + .filter(!_.isEmpty()) + .map(word => (word.slice(0, 1), (1L, word.length))) + .groupByKey + .combineValues(Aggregators.product[(Long, Int)](Aggregators.sum[Long], Aggregators.max[Int])) + .materialize + assert(fcc.exists(_ == ("w", (1404, 12)))) + + pipeline.done + } +} http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala index 238a01f..8b0f6b0 100644 --- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Aggregators.scala @@ -22,19 +22,19 @@ package org.apache.crunch.scrunch import org.apache.crunch.fn.{Aggregators => JAgg} import org.apache.crunch._ -import com.google.common.collect.ImmutableList +import com.google.common.collect.{UnmodifiableIterator, ImmutableList} import org.apache.hadoop.conf.Configuration import java.lang.{Iterable => JIterable} import scala.collection.JavaConversions import com.twitter.algebird.Monoid +import scala.reflect.ClassTag +import java.util /** * Scrunch versions of the common Aggregator types from Crunch. */ object Aggregators { - import scala.math.Ordering._ - def sum[T: Monoid]: Aggregator[T] = sumUsing(implicitly[Monoid[T]]) def sumUsing[T](m: Monoid[T]): Aggregator[T] = new SimpleAggregator[T] { @@ -176,14 +176,14 @@ object Aggregators { * Apply separate aggregators to each component of a {@link Tuple2}. */ def pair[V1, V2](a1: Aggregator[V1], a2: Aggregator[V2]): Aggregator[(V1, V2)] = { - return new Aggregators.PairAggregator[V1, V2](a1, a2) + new Aggregators.PairAggregator[V1, V2](a1, a2) } /** * Apply separate aggregators to each component of a {@link Tuple3}. */ def trip[V1, V2, V3](a1: Aggregator[V1], a2: Aggregator[V2], a3: Aggregator[V3]): Aggregator[(V1, V2, V3)] = { - return new Aggregators.TripAggregator[V1, V2, V3](a1, a2, a3) + new Aggregators.TripAggregator[V1, V2, V3](a1, a2, a3) } /** @@ -191,7 +191,15 @@ object Aggregators { */ def quad[V1, V2, V3, V4](a1: Aggregator[V1], a2: Aggregator[V2], a3: Aggregator[V3], a4: Aggregator[V4]) : Aggregator[(V1, V2, V3, V4)] = { - return new Aggregators.QuadAggregator[V1, V2, V3, V4](a1, a2, a3, a4) + new Aggregators.QuadAggregator[V1, V2, V3, V4](a1, a2, a3, a4) + } + + /** + * Apply separate aggregators to each component of a {@link Product} subclass. + */ + def product[T <: Product : ClassTag](aggregators: Aggregator[_]*): Aggregator[T] = { + new Aggregators.ProductAggregator[T](Array(aggregators : _*), + implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]) } /** @@ -202,20 +210,22 @@ object Aggregators { } } - private abstract class ProductAggregator[T <: Product](aggs: Array[Aggregator[_]]) extends Aggregator[T] { + private class ProductAggregator[T <: Product](val aggs: Array[Aggregator[_]], val clazz: Class[T]) + extends Aggregator[T] { + def initialize(configuration: Configuration) { for (a <- aggs) { a.initialize(configuration) } } - def reset { + def reset() { for (a <- aggs) { - a.reset + a.reset() } } - protected def updateTuple(t: T) { + override def update(t: T) { var i: Int = 0 while (i < aggs.length) { aggs(i).asInstanceOf[Aggregator[Any]].update(t.productElement(i)) @@ -223,49 +233,40 @@ object Aggregators { } } - protected def results[T](index: Int): Iterable[T] = { - return JavaConversions.iterableAsScalaIterable(aggs(index).results.asInstanceOf[JIterable[T]]) + override def results: JIterable[T] = { + val res = aggs.map(_.results()).map(JavaConversions.iterableAsScalaIterable(_).toList) + return new JIterable[T] { + override def iterator(): util.Iterator[T] = { + return new AggIterator[T](res, clazz) + } + } } } - private class PairAggregator[A, B](val a1: Aggregator[A], val a2: Aggregator[B]) - extends ProductAggregator[(A, B)](Array(a1, a2)) { + private class AggIterator[T](val results: Array[List[_]], val clazz: Class[T]) extends UnmodifiableIterator[T] { + var offset = 0 + val maxoffset = results.map(_.size).min + val construct = clazz.getConstructors.find(_.getParameterTypes.length == results.length) - def update(value: (A, B)) { - updateTuple(value) - } + override def hasNext: Boolean = offset < maxoffset - def results: JIterable[(A, B)] = { - return JavaConversions.asJavaIterable(results[A](0).zip(results[B](1))) + override def next(): T = { + val refs = results.map(x => x(offset)).asInstanceOf[Array[AnyRef]] + offset = offset + 1 + construct.get.newInstance(refs : _*).asInstanceOf[T] } } - private class TripAggregator[A, B, C](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C]) - extends ProductAggregator[(A, B, C)](Array(a1, a2, a3)) { - def update(value: (A, B, C)) { - updateTuple(value) - } + private class PairAggregator[A, B](val a1: Aggregator[A], val a2: Aggregator[B]) + extends ProductAggregator[(A, B)](Array(a1, a2), classOf[(A, B)]) { + } - def results: JIterable[(A, B, C)] = { - return JavaConversions.asJavaIterable( - results[A](0).zip(results[B](1)).zip(results[C](2)) - .map(x => (x._1._1, x._1._2, x._2))) - } + private class TripAggregator[A, B, C](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C]) + extends ProductAggregator[(A, B, C)](Array(a1, a2, a3), classOf[(A, B, C)]) { } private class QuadAggregator[A, B, C, D](val a1: Aggregator[A], val a2: Aggregator[B], val a3: Aggregator[C], val a4: Aggregator[D]) - extends ProductAggregator[(A, B, C, D)](Array(a1, a2, a3, a4)) { - - def update(value: (A, B, C, D)) { - updateTuple(value) - } - - def results: JIterable[(A, B, C, D)] = { - return JavaConversions.asJavaIterable( - (results[A](0).zip(results[B](1))).zip(results[C](2).zip(results[D](3))) - .map(x => (x._1._1, x._1._2, x._2._1, x._2._2))) - } + extends ProductAggregator[(A, B, C, D)](Array(a1, a2, a3, a4), classOf[(A, B, C, D)]) { } - } http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala index 05baebf..ffd85c6 100644 --- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Conversions.scala @@ -92,7 +92,7 @@ trait LowPriorityPTypeH { } } -object PTypeH extends LowPriorityPTypeH { +object PTypeH extends GeneratedTupleConversions with LowPriorityPTypeH { implicit val longs = new PTypeH[Long] { def get(ptf: PTypeFamily) = ptf.longs } implicit val jlongs = new PTypeH[java.lang.Long] { def get(ptf: PTypeFamily) = ptf.jlongs } http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala new file mode 100644 index 0000000..e2218c5 --- /dev/null +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTupleConversions.scala @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.scrunch + +trait GeneratedTupleConversions { + implicit def tuple5[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH] = new PTypeH[(A, B, C, D, E)] { + def get(ptf: PTypeFamily) = { + ptf.tuple5(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf)) + } + } + + implicit def tuple6[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH] = new PTypeH[(A, B, C, D, E, F)] { + def get(ptf: PTypeFamily) = { + ptf.tuple6(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf)) + } + } + + implicit def tuple7[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH] = new PTypeH[(A, B, C, D, E, F, G)] { + def get(ptf: PTypeFamily) = { + ptf.tuple7(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf)) + } + } + + implicit def tuple8[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H)] { + def get(ptf: PTypeFamily) = { + ptf.tuple8(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf)) + } + } + + implicit def tuple9[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I)] { + def get(ptf: PTypeFamily) = { + ptf.tuple9(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf)) + } + } + + implicit def tuple10[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J)] { + def get(ptf: PTypeFamily) = { + ptf.tuple10(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf)) + } + } + + implicit def tuple11[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K)] { + def get(ptf: PTypeFamily) = { + ptf.tuple11(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf)) + } + } + + implicit def tuple12[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L)] { + def get(ptf: PTypeFamily) = { + ptf.tuple12(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf)) + } + } + + implicit def tuple13[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M)] { + def get(ptf: PTypeFamily) = { + ptf.tuple13(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf)) + } + } + + implicit def tuple14[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)] { + def get(ptf: PTypeFamily) = { + ptf.tuple14(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf)) + } + } + + implicit def tuple15[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)] { + def get(ptf: PTypeFamily) = { + ptf.tuple15(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf)) + } + } + + implicit def tuple16[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)] { + def get(ptf: PTypeFamily) = { + ptf.tuple16(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf)) + } + } + + implicit def tuple17[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)] { + def get(ptf: PTypeFamily) = { + ptf.tuple17(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf)) + } + } + + implicit def tuple18[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)] { + def get(ptf: PTypeFamily) = { + ptf.tuple18(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf)) + } + } + + implicit def tuple19[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)] { + def get(ptf: PTypeFamily) = { + ptf.tuple19(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf)) + } + } + + implicit def tuple20[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)] { + def get(ptf: PTypeFamily) = { + ptf.tuple20(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf)) + } + } + + implicit def tuple21[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH, U: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)] { + def get(ptf: PTypeFamily) = { + ptf.tuple21(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf), implicitly[PTypeH[U]].get(ptf)) + } + } + + implicit def tuple22[A: PTypeH, B: PTypeH, C: PTypeH, D: PTypeH, E: PTypeH, F: PTypeH, G: PTypeH, H: PTypeH, I: PTypeH, J: PTypeH, K: PTypeH, L: PTypeH, M: PTypeH, N: PTypeH, O: PTypeH, P: PTypeH, Q: PTypeH, R: PTypeH, S: PTypeH, T: PTypeH, U: PTypeH, V: PTypeH] = new PTypeH[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)] { + def get(ptf: PTypeFamily) = { + ptf.tuple22(implicitly[PTypeH[A]].get(ptf), implicitly[PTypeH[B]].get(ptf), implicitly[PTypeH[C]].get(ptf), implicitly[PTypeH[D]].get(ptf), implicitly[PTypeH[E]].get(ptf), implicitly[PTypeH[F]].get(ptf), implicitly[PTypeH[G]].get(ptf), implicitly[PTypeH[H]].get(ptf), implicitly[PTypeH[I]].get(ptf), implicitly[PTypeH[J]].get(ptf), implicitly[PTypeH[K]].get(ptf), implicitly[PTypeH[L]].get(ptf), implicitly[PTypeH[M]].get(ptf), implicitly[PTypeH[N]].get(ptf), implicitly[PTypeH[O]].get(ptf), implicitly[PTypeH[P]].get(ptf), implicitly[PTypeH[Q]].get(ptf), implicitly[PTypeH[R]].get(ptf), implicitly[PTypeH[S]].get(ptf), implicitly[PTypeH[T]].get(ptf), implicitly[PTypeH[U]].get(ptf), implicitly[PTypeH[V]].get(ptf)) + } + } + +} http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala new file mode 100644 index 0000000..34a424f --- /dev/null +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/GeneratedTuplePTypeFamily.scala @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.scrunch + +import org.apache.crunch.TupleN +import org.apache.crunch.types.PType + +trait GeneratedTuplePTypeFamily extends BasePTypeFamily { + import GeneratedTupleHelper._ + + def tuple5[A, B, C, D, E](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E]) + val out = (t: (A, B, C, D, E)) => tupleN(t._1, t._2, t._3, t._4, t._5) + derived(classOf[(A, B, C, D, E)], in, out, ptf.tuples(p0, p1, p2, p3, p4)) + } + + def tuple6[A, B, C, D, E, F](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F]) + val out = (t: (A, B, C, D, E, F)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6) + derived(classOf[(A, B, C, D, E, F)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5)) + } + + def tuple7[A, B, C, D, E, F, G](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G]) + val out = (t: (A, B, C, D, E, F, G)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7) + derived(classOf[(A, B, C, D, E, F, G)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6)) + } + + def tuple8[A, B, C, D, E, F, G, H](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H]) + val out = (t: (A, B, C, D, E, F, G, H)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8) + derived(classOf[(A, B, C, D, E, F, G, H)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7)) + } + + def tuple9[A, B, C, D, E, F, G, H, I](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I]) + val out = (t: (A, B, C, D, E, F, G, H, I)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9) + derived(classOf[(A, B, C, D, E, F, G, H, I)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8)) + } + + def tuple10[A, B, C, D, E, F, G, H, I, J](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J]) + val out = (t: (A, B, C, D, E, F, G, H, I, J)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10) + derived(classOf[(A, B, C, D, E, F, G, H, I, J)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9)) + } + + def tuple11[A, B, C, D, E, F, G, H, I, J, K](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10)) + } + + def tuple12[A, B, C, D, E, F, G, H, I, J, K, L](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11)) + } + + def tuple13[A, B, C, D, E, F, G, H, I, J, K, L, M](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12)) + } + + def tuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13)) + } + + def tuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14)) + } + + def tuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15)) + } + + def tuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16)) + } + + def tuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17)) + } + + def tuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18)) + } + + def tuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19)) + } + + def tuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T], p20: PType[U]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T],t.get(20).asInstanceOf[U]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20, t._21) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20)) + } + + def tuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](p0: PType[A], p1: PType[B], p2: PType[C], p3: PType[D], p4: PType[E], p5: PType[F], p6: PType[G], p7: PType[H], p8: PType[I], p9: PType[J], p10: PType[K], p11: PType[L], p12: PType[M], p13: PType[N], p14: PType[O], p15: PType[P], p16: PType[Q], p17: PType[R], p18: PType[S], p19: PType[T], p20: PType[U], p21: PType[V]) = { + val in = (t: TupleN) => (t.get(0).asInstanceOf[A],t.get(1).asInstanceOf[B],t.get(2).asInstanceOf[C],t.get(3).asInstanceOf[D],t.get(4).asInstanceOf[E],t.get(5).asInstanceOf[F],t.get(6).asInstanceOf[G],t.get(7).asInstanceOf[H],t.get(8).asInstanceOf[I],t.get(9).asInstanceOf[J],t.get(10).asInstanceOf[K],t.get(11).asInstanceOf[L],t.get(12).asInstanceOf[M],t.get(13).asInstanceOf[N],t.get(14).asInstanceOf[O],t.get(15).asInstanceOf[P],t.get(16).asInstanceOf[Q],t.get(17).asInstanceOf[R],t.get(18).asInstanceOf[S],t.get(19).asInstanceOf[T],t.get(20).asInstanceOf[U],t.get(21).asInstanceOf[V]) + val out = (t: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)) => tupleN(t._1, t._2, t._3, t._4, t._5, t._6, t._7, t._8, t._9, t._10, t._11, t._12, t._13, t._14, t._15, t._16, t._17, t._18, t._19, t._20, t._21, t._22) + derived(classOf[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)], in, out, ptf.tuples(p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21)) + } + +} + http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala index bd03f6f..f77067a 100644 --- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/PTypeFamily.scala @@ -17,7 +17,7 @@ */ package org.apache.crunch.scrunch -import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, Union, MapFn} +import org.apache.crunch.{Pair => CPair, Tuple3 => CTuple3, Tuple4 => CTuple4, TupleN, Union, MapFn} import org.apache.crunch.types.{PType, PTypeFamily => PTF} import org.apache.crunch.types.writable.{WritableTypeFamily, Writables => CWritables} import org.apache.crunch.types.avro.{AvroType, AvroTypeFamily, Avros => CAvros} @@ -47,10 +47,22 @@ class TMapFn[S, T](val f: S => T, val pt: Option[PType[S]] = None, var init: Boo } } -trait PTypeFamily { +object GeneratedTupleHelper { + def tupleN(args: Any*) = { + TupleN.of(args.map(_.asInstanceOf[AnyRef]): _*) + } +} +trait BasePTypeFamily { def ptf: PTF + def derived[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = { + ptf.derived(cls, new TMapFn[S, T](in, Some(pt)), new TMapFn[T, S](out), pt) + } +} + +trait PTypeFamily extends GeneratedTuplePTypeFamily { + def writables[T <: Writable : ClassTag]: PType[T] def as[T](ptype: PType[T]) = ptf.as(ptype) @@ -61,10 +73,6 @@ trait PTypeFamily { def records[T: ClassTag] = ptf.records(implicitly[ClassTag[T]].runtimeClass) - def derived[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = { - ptf.derived(cls, new TMapFn[S, T](in, Some(pt)), new TMapFn[T, S](out), pt) - } - def derivedImmutable[S, T](cls: java.lang.Class[T], in: S => T, out: T => S, pt: PType[S]) = { ptf.derivedImmutable(cls, new TMapFn[S, T](in), new TMapFn[T, S](out), pt) } http://git-wip-us.apache.org/repos/asf/crunch/blob/0e7e0d0b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala new file mode 100644 index 0000000..640a72f --- /dev/null +++ b/crunch-scrunch/src/test/scala/org/apache/crunch/scrunch/TupleNTest.scala @@ -0,0 +1,32 @@ +/* + * * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.apache.crunch.scrunch + +import org.scalatest.junit.JUnitSuite +import org.junit.Test + +class TupleNTest extends JUnitSuite{ + @Test def testTupleN { + val pc = Mem.collectionOf((1, 2, "a", 3, "b"), (4, 5, "a", 6, "c")) + val res = pc.map(x => (x._3, x._4)).groupByKey.combineValues(Aggregators.sum[Int]).materialize + org.junit.Assert.assertEquals(List(("a", 9)), res.toList) + } +}
