Repository: crunch Updated Branches: refs/heads/master 567d25dd7 -> c135bba63
CRUNCH-406: Have From/At code for Avro fields read PTypes and do the internal casting themselves Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/c135bba6 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/c135bba6 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/c135bba6 Branch: refs/heads/master Commit: c135bba631c106f93aa2cb4f973bbff4262aba0d Parents: 567d25d Author: Josh Wills <[email protected]> Authored: Fri May 30 12:54:47 2014 -0700 Committer: Josh Wills <[email protected]> Committed: Sat May 31 14:43:10 2014 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/crunch/io/At.java | 12 +++++----- .../main/java/org/apache/crunch/io/From.java | 18 +++++++-------- .../scala/org/apache/crunch/scrunch/IO.scala | 24 ++++++++------------ 3 files changed, 24 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-core/src/main/java/org/apache/crunch/io/At.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/At.java b/crunch-core/src/main/java/org/apache/crunch/io/At.java index e60831a..9a8db2f 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/At.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/At.java @@ -131,22 +131,22 @@ public class At { * Creates a {@code SourceTarget<T>} instance from the Avro file(s) at the given path name. * * @param pathName The name of the path to the data on the filesystem - * @param avroType The {@code AvroType} for the Avro records + * @param ptype The {@code PType} for the Avro records * @return A new {@code SourceTarget<T>} instance */ - public static <T> SourceTarget<T> avroFile(String pathName, AvroType<T> avroType) { - return avroFile(new Path(pathName), avroType); + public static <T> SourceTarget<T> avroFile(String pathName, PType<T> ptype) { + return avroFile(new Path(pathName), ptype); } /** * Creates a {@code SourceTarget<T>} instance from the Avro file(s) at the given {@code Path}. * * @param path The {@code Path} to the data - * @param avroType The {@code AvroType} for the Avro records + * @param ptype The {@code PType} for the Avro records * @return A new {@code SourceTarget<T>} instance */ - public static <T> SourceTarget<T> avroFile(Path path, AvroType<T> avroType) { - return new AvroFileSourceTarget<T>(path, avroType); + public static <T> SourceTarget<T> avroFile(Path path, PType<T> ptype) { + return new AvroFileSourceTarget<T>(path, (AvroType<T>) ptype); } /** http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-core/src/main/java/org/apache/crunch/io/From.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/From.java b/crunch-core/src/main/java/org/apache/crunch/io/From.java index 14793a6..d72ddf4 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/From.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/From.java @@ -221,33 +221,33 @@ public class From { * Creates a {@code Source<T>} instance from the Avro file(s) at the given path name. * * @param pathName The name of the path to the data on the filesystem - * @param avroType The {@code AvroType} for the Avro records + * @param ptype The {@code AvroType} for the Avro records * @return A new {@code Source<T>} instance */ - public static <T> Source<T> avroFile(String pathName, AvroType<T> avroType) { - return avroFile(new Path(pathName), avroType); + public static <T> Source<T> avroFile(String pathName, PType<T> ptype) { + return avroFile(new Path(pathName), ptype); } /** * Creates a {@code Source<T>} instance from the Avro file(s) at the given {@code Path}. * * @param path The {@code Path} to the data - * @param avroType The {@code AvroType} for the Avro records + * @param ptype The {@code AvroType} for the Avro records * @return A new {@code Source<T>} instance */ - public static <T> Source<T> avroFile(Path path, AvroType<T> avroType) { - return new AvroFileSource<T>(path, avroType); + public static <T> Source<T> avroFile(Path path, PType<T> ptype) { + return new AvroFileSource<T>(path, (AvroType<T>) ptype); } /** * Creates a {@code Source<T>} instance from the Avro file(s) at the given {@code Path}s. * * @param paths A list of {@code Path}s to the data - * @param avroType The {@code AvroType} for the Avro records + * @param ptype The {@code PType} for the Avro records * @return A new {@code Source<T>} instance */ - public static <T> Source<T> avroFile(List<Path> paths, AvroType<T> avroType) { - return new AvroFileSource<T>(paths, avroType); + public static <T> Source<T> avroFile(List<Path> paths, PType<T> ptype) { + return new AvroFileSource<T>(paths, (AvroType<T>) ptype); } /** http://git-wip-us.apache.org/repos/asf/crunch/blob/c135bba6/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala index c4af454..d9d1acd 100644 --- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/IO.scala @@ -20,19 +20,13 @@ package org.apache.crunch.scrunch -import org.apache.crunch.io.{From => from, To => to, At => at, SequentialFileNamingScheme} +import org.apache.crunch.io.{From => from, To => to, At => at} import org.apache.hadoop.fs.Path import org.apache.hadoop.conf.Configuration import org.apache.crunch.types.PType import org.apache.hadoop.mapreduce.lib.input.FileInputFormat -import org.apache.crunch.types.avro.AvroType import org.apache.hadoop.io.Writable import org.apache.avro.specific.SpecificRecord -import org.apache.crunch.Target -import org.apache.crunch.io.impl.FileTargetImpl -import org.apache.crunch.io.avro.AvroFileTarget -import org.apache.crunch.io.seq.SeqFileTarget -import org.apache.crunch.io.text.TextFileTarget import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat object From { @@ -80,16 +74,16 @@ object From { from.avroFile(paths, avroClass) } - def avroFile[T](pathName: String, avroType: AvroType[T]) = { - from.avroFile(pathName, avroType) + def avroFile[T](pathName: String, ptype: PType[T]) = { + from.avroFile(pathName, ptype) } - def avroFile[T](path: Path, avroType: AvroType[T]) = { - from.avroFile(path, avroType) + def avroFile[T](path: Path, ptype: PType[T]) = { + from.avroFile(path, ptype) } - def avroFile[T](paths: List[Path], avroType: AvroType[T]) = { - from.avroFile(paths, avroType) + def avroFile[T](paths: List[Path], ptype: PType[T]) = { + from.avroFile(paths, ptype) } def avroFile(pathName: String) = { @@ -204,11 +198,11 @@ object At { at.avroFile(path, avroClass) } - def avroFile[T](pathName: String, avroType: AvroType[T]) = { + def avroFile[T](pathName: String, avroType: PType[T]) = { at.avroFile(pathName, avroType) } - def avroFile[T](path: Path, avroType: AvroType[T]) = { + def avroFile[T](path: Path, avroType: PType[T]) = { at.avroFile(path, avroType) }
