MAPREDUCE-6780. Add support for HDFS directory with erasure code policy to TeraGen and TeraSort. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bea004ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bea004ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bea004ea Branch: refs/heads/HDFS-7240 Commit: bea004eaeb7ba33bf324ef3e7065cfdd614d8198 Parents: ec0b707 Author: Kai Zheng <kai.zh...@intel.com> Authored: Sun Oct 9 15:33:26 2016 +0600 Committer: Kai Zheng <kai.zh...@intel.com> Committed: Sun Oct 9 15:33:26 2016 +0600 ---------------------------------------------------------------------- .../hadoop/examples/terasort/TeraGen.java | 3 +++ .../examples/terasort/TeraOutputFormat.java | 20 +++++++++++++++++--- .../hadoop/examples/terasort/TeraSort.java | 3 +++ 3 files changed, 23 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bea004ea/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java index 22fe344..7fbb22a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java @@ -246,6 +246,9 @@ public class TeraGen extends Configured implements Tool { private static void usage() throws IOException { System.err.println("teragen <num rows> <output dir>"); + System.err.println("If you want to generate data and store them as " + + "erasure code striping file, just make sure that the parent dir " + + "of <output dir> has erasure code policy set"); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/bea004ea/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java index fd3ea78..73c446d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java @@ -20,6 +20,8 @@ package org.apache.hadoop.examples.terasort; import java.io.IOException; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; @@ -40,6 +42,7 @@ import org.apache.hadoop.mapreduce.security.TokenCache; * An output format that writes the key and value appended together. */ public class TeraOutputFormat extends FileOutputFormat<Text,Text> { + private static final Log LOG = LogFactory.getLog(TeraOutputFormat.class); private OutputCommitter committer = null; /** @@ -74,10 +77,22 @@ public class TeraOutputFormat extends FileOutputFormat<Text,Text> { out.write(key.getBytes(), 0, key.getLength()); out.write(value.getBytes(), 0, value.getLength()); } - + public void close(TaskAttemptContext context) throws IOException { if (finalSync) { - out.hsync(); + try { + out.hsync(); + } catch (UnsupportedOperationException e) { + /* + * Currently, hsync operation on striping file with erasure code + * policy is not supported yet. So this is a workaround to make + * teragen and terasort to support directory with striping files. In + * future, if the hsync operation is supported on striping file, this + * workaround should be removed. + */ + LOG.info("Operation hsync is not supported so far on path with " + + "erasure code policy set"); + } } out.close(); } @@ -135,5 +150,4 @@ public class TeraOutputFormat extends FileOutputFormat<Text,Text> { } return committer; } - } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bea004ea/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java index 9beff3e..040d13f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java @@ -287,6 +287,9 @@ public class TeraSort extends Configured implements Tool { for (TeraSortConfigKeys teraSortConfigKeys : TeraSortConfigKeys.values()) { System.err.println(teraSortConfigKeys.toString()); } + System.err.println("If you want to store the output data as " + + "erasure code striping file, just make sure that the parent dir " + + "of <out> has erasure code policy set"); } public int run(String[] args) throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
