Updated Branches: refs/heads/master 1082111c7 -> dea3fd93e
CRUNCH-228: FileTargetImpl cuts off extensions of output files Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/dea3fd93 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/dea3fd93 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/dea3fd93 Branch: refs/heads/master Commit: dea3fd93eb54c68b7083fdbdb3f798aafb4d9e9a Parents: 1082111 Author: Dave Beech <[email protected]> Authored: Sun Jun 30 17:16:43 2013 +0100 Committer: Dave Beech <[email protected]> Committed: Sun Jun 30 17:16:43 2013 +0100 ---------------------------------------------------------------------- .../apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java | 8 ++++---- .../main/java/org/apache/crunch/io/impl/FileTargetImpl.java | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/dea3fd93/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java b/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java index cd7fe0b..c6ea4fa 100644 --- a/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/io/avro/trevni/TrevniKeyPipelineIT.java @@ -91,7 +91,7 @@ public class TrevniKeyPipelineIT implements Serializable { Person person = genericCollection.materialize().iterator().next(); - File trvFile = new File(outputFile, "part-m-00000-part-0.trv"); + File trvFile = new File(outputFile, "part-m-00000.trv-part-0.trv"); AvroColumnReader.Params params = new AvroColumnReader.Params(trvFile); params.setSchema(Person.SCHEMA$); @@ -125,8 +125,8 @@ public class TrevniKeyPipelineIT implements Serializable { Person person = genericCollection.materialize().iterator().next(); - File trv1File = new File(output1File, "part-m-00000-part-0.trv"); - File trv2File = new File(output2File, "part-m-00000-part-0.trv"); + File trv1File = new File(output1File, "part-m-00000.trv-part-0.trv"); + File trv2File = new File(output2File, "part-m-00000.trv-part-0.trv"); AvroColumnReader.Params params = new AvroColumnReader.Params(trv1File); params.setSchema(Person.SCHEMA$); @@ -178,7 +178,7 @@ public class TrevniKeyPipelineIT implements Serializable { assertThat(retrievedPerson, is(person)); - File trvFile = new File(outputFile, "part-m-00000-part-0.trv"); + File trvFile = new File(outputFile, "part-m-00000.trv-part-0.trv"); AvroColumnReader.Params params = new AvroColumnReader.Params(trvFile); params.setSchema(Person.SCHEMA$); http://git-wip-us.apache.org/repos/asf/crunch/blob/dea3fd93/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java index 50a1fd3..4d58830 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/impl/FileTargetImpl.java @@ -132,13 +132,14 @@ public class FileTargetImpl implements PathTarget { protected Path getDestFile(Configuration conf, Path src, Path dir, boolean mapOnlyJob) throws IOException { String outputFilename = null; + String sourceFilename = src.getName(); if (mapOnlyJob) { outputFilename = getFileNamingScheme().getMapOutputName(conf, dir); } else { - outputFilename = getFileNamingScheme().getReduceOutputName(conf, dir, extractPartitionNumber(src.getName())); + outputFilename = getFileNamingScheme().getReduceOutputName(conf, dir, extractPartitionNumber(sourceFilename)); } - if (src.getName().endsWith(org.apache.avro.mapred.AvroOutputFormat.EXT)) { - outputFilename += org.apache.avro.mapred.AvroOutputFormat.EXT; + if (sourceFilename.contains(".")) { + outputFilename += sourceFilename.substring(sourceFilename.indexOf(".")); } return new Path(dir, outputFilename); }
