[49/50] [abbrv] hadoop git commit: HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69782710
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69782710
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69782710
Branch: refs/heads/HDFS-7285
Commit: 697827102014a90c1c64b76be170d940c78516bc
Parents: 9f5926d
Author: Kai Zheng kai.zh...@intel.com
Authored: Fri Mar 20 19:15:52 2015 +0800
Committer: Zhe Zhang z...@apache.org
Committed: Tue Mar 24 11:16:35 2015 -0700
--
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +
.../hadoop/fs/CommonConfigurationKeys.java | 15
.../erasurecode/coder/AbstractErasureCoder.java | 65 ++
.../coder/AbstractErasureDecoder.java | 6 +-
.../coder/AbstractErasureEncoder.java | 6 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 83 ++
.../io/erasurecode/coder/RSErasureEncoder.java | 47 ++
.../io/erasurecode/coder/XorErasureDecoder.java | 2 +-
.../io/erasurecode/coder/XorErasureEncoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 92
10 files changed, 315 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69782710/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index f566f0e..b69e69a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -26,3 +26,6 @@
HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai
Zheng
( Kai Zheng )
+
+HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
+( Kai Zheng )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69782710/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 7575496..70fea01 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -135,6 +135,21 @@ public class CommonConfigurationKeys extends
CommonConfigurationKeysPublic {
false;
/**
+ * Erasure Coding configuration family
+ */
+
+ /** Supported erasure codec classes */
+ public static final String IO_ERASURECODE_CODECS_KEY =
io.erasurecode.codecs;
+
+ /** Use XOR raw coder when possible for the RS codec */
+ public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY =
+ io.erasurecode.codec.rs.usexor;
+
+ /** Raw coder factory for the RS codec */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ io.erasurecode.codec.rs.rawcoder;
+
+ /**
* Service Authorization
*/
public static final String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69782710/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
index 8d3bc34..0e4de89 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
@@ -17,7 +17,12 @@
*/
package org.apache.hadoop.io.erasurecode.coder;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
/**
* A common class of basic facilities to be shared by encoder and decoder
@@ -31,6 +36,66 @@ public abstract class AbstractErasureCoder
private int numParityUnits;
private int chunkSize;
+ /**
+ * Create raw decoder using the factory specified by
[49/50] [abbrv] hadoop git commit: HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7da69bb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7da69bb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7da69bb3
Branch: refs/heads/HDFS-7285
Commit: 7da69bb30af30e02e5beebd0f5a505c879ba2efb
Parents: 18f0bac
Author: Kai Zheng kai.zh...@intel.com
Authored: Fri Mar 20 19:15:52 2015 +0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Mar 23 11:14:12 2015 -0700
--
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +
.../hadoop/fs/CommonConfigurationKeys.java | 15
.../erasurecode/coder/AbstractErasureCoder.java | 65 ++
.../coder/AbstractErasureDecoder.java | 6 +-
.../coder/AbstractErasureEncoder.java | 6 +-
.../io/erasurecode/coder/RSErasureDecoder.java | 83 ++
.../io/erasurecode/coder/RSErasureEncoder.java | 47 ++
.../io/erasurecode/coder/XorErasureDecoder.java | 2 +-
.../io/erasurecode/coder/XorErasureEncoder.java | 2 +-
.../erasurecode/coder/TestRSErasureCoder.java | 92
10 files changed, 315 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7da69bb3/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index f566f0e..b69e69a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -26,3 +26,6 @@
HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai
Zheng
( Kai Zheng )
+
+HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
+( Kai Zheng )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7da69bb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 7575496..70fea01 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -135,6 +135,21 @@ public class CommonConfigurationKeys extends
CommonConfigurationKeysPublic {
false;
/**
+ * Erasure Coding configuration family
+ */
+
+ /** Supported erasure codec classes */
+ public static final String IO_ERASURECODE_CODECS_KEY =
io.erasurecode.codecs;
+
+ /** Use XOR raw coder when possible for the RS codec */
+ public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY =
+ io.erasurecode.codec.rs.usexor;
+
+ /** Raw coder factory for the RS codec */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ io.erasurecode.codec.rs.rawcoder;
+
+ /**
* Service Authorization
*/
public static final String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7da69bb3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
index 8d3bc34..0e4de89 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
@@ -17,7 +17,12 @@
*/
package org.apache.hadoop.io.erasurecode.coder;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
/**
* A common class of basic facilities to be shared by encoder and decoder
@@ -31,6 +36,66 @@ public abstract class AbstractErasureCoder
private int numParityUnits;
private int chunkSize;
+ /**
+ * Create raw decoder using the factory specified by
