Repository: crunch Updated Branches: refs/heads/master 55d73bf17 -> 6463da421
CRUNCH-590: Fix CSVInputFormat to work with S3 Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6463da42 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6463da42 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6463da42 Branch: refs/heads/master Commit: 6463da4218568d3d3fd82a33930f8b7cc57c1288 Parents: 55d73bf Author: Josh Wills <[email protected]> Authored: Tue Feb 2 11:05:04 2016 -0800 Committer: Josh Wills <[email protected]> Committed: Tue Feb 2 11:05:04 2016 -0800 ---------------------------------------------------------------------- .../main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/6463da42/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java b/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java index 2894686..2dc5e13 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/text/csv/CSVInputFormat.java @@ -92,10 +92,10 @@ public class CSVInputFormat extends FileInputFormat<LongWritable, Text> implemen final long splitSize = job.getConfiguration().getLong(CSVFileSource.INPUT_SPLIT_SIZE, 67108864); final List<InputSplit> splits = new ArrayList<InputSplit>(); final Path[] paths = FileUtil.stat2Paths(listStatus(job).toArray(new FileStatus[0])); - final FileSystem fileSystem = FileSystem.get(job.getConfiguration()); FSDataInputStream inputStream = null; try { for (final Path path : paths) { + FileSystem fileSystem = path.getFileSystem(job.getConfiguration()); inputStream = fileSystem.open(path); splits.addAll(getSplitsForFile(splitSize, fileSystem.getFileStatus(path).getLen(), path, inputStream)); } @@ -199,4 +199,4 @@ public class CSVInputFormat extends FileInputFormat<LongWritable, Text> implemen return splitsList; } -} \ No newline at end of file +}
