Repository: crunch Updated Branches: refs/heads/master e176b6166 -> ca8ff16b6
CRUNCH-636: amend Make replication factor for temporary files configurable Signed-off-by: Josh Wills <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ca8ff16b Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ca8ff16b Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ca8ff16b Branch: refs/heads/master Commit: ca8ff16b6172be6cf2be1c185c41593ce4897c53 Parents: e176b61 Author: Attila Sasvari <[email protected]> Authored: Thu Mar 23 21:35:36 2017 +0100 Committer: Josh Wills <[email protected]> Committed: Thu Mar 23 16:22:11 2017 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java | 4 ++-- .../java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/ca8ff16b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java index d31bfad..7705d4a 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java @@ -329,8 +329,8 @@ class JobPrototype { return d; } - private void setInitialJobReplicationConfig(Configuration job, String defaultReplication) { - job.set(DFS_REPLICATION_INITIAL, defaultReplication); + private void setInitialJobReplicationConfig(Configuration jobConf, String defaultReplication) { + jobConf.set(DFS_REPLICATION_INITIAL, defaultReplication); } private static CrunchControlledJob.Hook getHook( http://git-wip-us.apache.org/repos/asf/crunch/blob/ca8ff16b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java index 44da01a..e9847c7 100644 --- a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java @@ -48,7 +48,6 @@ public class JobPrototypeTest { @Mock private Path mockPath; @Mock private FileTargetImpl mockTarget; @Mock private FileSystem mockFs; - @Mock private DoNode mockNode; @Mock private PGroupedTableImpl<String, String> mockPgroup; @Mock private Set<NodePath> mockInputs; private JobPrototype jobPrototypeUnderTest; @@ -86,7 +85,6 @@ public class JobPrototypeTest { @Test public void initialReplicationFactorUsedFromFileSystem() throws IOException { - testConfiguration = new Configuration(); HashMultimap<Target, NodePath> targetNodePaths = HashMultimap.create(); targetNodePaths.put(mockTarget, new NodePath()); doReturn(mockPath).when(mockTarget).getPath(); @@ -95,6 +93,7 @@ public class JobPrototypeTest { c.set("dfs.replication", TEST_INITIAL_DFS_REPLICATION); doReturn(c).when(mockFs).getConf(); jobPrototypeUnderTest.addReducePaths(targetNodePaths); + testConfiguration = new Configuration(false); jobPrototypeUnderTest.handleInitialReplication(testConfiguration); assertEquals(TEST_INITIAL_DFS_REPLICATION, testConfiguration.get("dfs.replication.initial")); @@ -102,7 +101,6 @@ public class JobPrototypeTest { @Test public void initialReplicationFactorUsedWhenItCannotBeRetrievedFromFileSystem() throws IOException { - testConfiguration = new Configuration(); HashMultimap<Target, NodePath> targetNodePaths = HashMultimap.create(); targetNodePaths.put(mockTarget, new NodePath()); doReturn(mockPath).when(mockTarget).getPath(); @@ -110,6 +108,7 @@ public class JobPrototypeTest { Configuration c = new Configuration(); c.set("dfs.replication", TEST_INITIAL_DFS_REPLICATION); jobPrototypeUnderTest.addReducePaths(targetNodePaths); + testConfiguration = new Configuration(false); jobPrototypeUnderTest.handleInitialReplication(testConfiguration); assertEquals("3", testConfiguration.get("dfs.replication.initial")); //default
