Updated Branches: refs/heads/master 41f01c037 -> 6536f4c90
CRUNCH-160: CrunchInputFormat needs to create a new Configuration object for each delegate InputFormat Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/6536f4c9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/6536f4c9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/6536f4c9 Branch: refs/heads/master Commit: 6536f4c909a1b8d952f1c4398195177659d1423d Parents: 41f01c0 Author: Josh Wills <[email protected]> Authored: Sun Feb 3 17:51:55 2013 -0800 Committer: Josh Wills <[email protected]> Committed: Sun Feb 3 17:51:55 2013 -0800 ---------------------------------------------------------------------- .../crunch/impl/mr/run/CrunchInputFormat.java | 8 +++----- 1 files changed, 3 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6536f4c9/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java b/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java index d39f67f..eb5dd8a 100644 --- a/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java +++ b/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java @@ -42,19 +42,17 @@ public class CrunchInputFormat<K, V> extends InputFormat<K, V> { @Override public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { List<InputSplit> splits = Lists.newArrayList(); - Configuration conf = job.getConfiguration(); + Configuration base = job.getConfiguration(); Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job); // First, build a map of InputFormats to Paths for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) { FormatBundle inputBundle = entry.getKey(); + Configuration conf = new Configuration(base); + inputBundle.configure(conf); Job jobCopy = new Job(conf); - inputBundle.configure(jobCopy.getConfiguration()); InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(), jobCopy.getConfiguration()); - if (format instanceof Configurable) { - ((Configurable) format).setConf(jobCopy.getConfiguration()); - } for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) { Integer nodeIndex = nodeEntry.getKey(); List<Path> paths = nodeEntry.getValue();
