subject:"\[2\/2\] hadoop git commit\: HDFS\-13056. Expose file\-level composite CRCs in HDFS which are comparable across different instances\/layouts. Contributed by Dennis Huo."

[2/2] hadoop git commit: HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.

2018-04-16 Thread arp

HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable 
across different instances/layouts. Contributed by Dennis Huo.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/805e33b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/805e33b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/805e33b6

Branch: refs/heads/branch-3.1
Commit: 805e33b62c32ccaa5a9b1889331518967056ca66
Parents: 2911b54
Author: Xiao Chen 
Authored: Tue Apr 10 20:56:07 2018 -0700
Committer: Arpit Agarwal 
Committed: Mon Apr 16 11:58:10 2018 -0700

--
 .../hadoop/fs/CompositeCrcFileChecksum.java |  82 +
 .../main/java/org/apache/hadoop/fs/Options.java |  11 +
 .../org/apache/hadoop/util/CrcComposer.java | 187 ++
 .../java/org/apache/hadoop/util/CrcUtil.java| 220 +++
 .../org/apache/hadoop/util/DataChecksum.java|  18 +
 .../org/apache/hadoop/util/TestCrcComposer.java | 242 
 .../org/apache/hadoop/util/TestCrcUtil.java | 232 
 .../main/java/org/apache/hadoop/fs/Hdfs.java|   4 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  56 ++-
 .../hadoop/hdfs/DistributedFileSystem.java  |   5 +-
 .../apache/hadoop/hdfs/FileChecksumHelper.java  | 365 +--
 .../hdfs/client/HdfsClientConfigKeys.java   |   2 +
 .../hadoop/hdfs/client/impl/DfsClientConf.java  |  27 ++
 .../hdfs/protocol/BlockChecksumOptions.java |  54 +++
 .../hadoop/hdfs/protocol/BlockChecksumType.java |  30 ++
 .../datatransfer/DataTransferProtocol.java  |  12 +-
 .../hdfs/protocol/datatransfer/Sender.java  |  11 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |  44 +++
 .../src/main/proto/datatransfer.proto   |   7 +-
 .../src/main/proto/hdfs.proto   |  21 ++
 .../hdfs/protocol/datatransfer/Receiver.java|   8 +-
 .../server/datanode/BlockChecksumHelper.java| 289 +--
 .../hdfs/server/datanode/DataXceiver.java   |  26 +-
 ...dBlockChecksumCompositeCrcReconstructor.java |  80 
 ...StripedBlockChecksumMd5CrcReconstructor.java |  74 
 .../StripedBlockChecksumReconstructor.java  |  66 ++--
 .../erasurecode/StripedBlockReconstructor.java  |   1 +
 .../src/main/resources/hdfs-default.xml |  11 +
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  31 +-
 .../apache/hadoop/hdfs/TestFileChecksum.java| 101 -
 .../hdfs/TestFileChecksumCompositeCrc.java  |  47 +++
 .../hadoop/hdfs/protocolPB/TestPBHelper.java|  14 +
 .../hadoop/tools/mapred/TestCopyMapper.java | 173 +++--
 .../mapred/TestCopyMapperCompositeCrc.java  |  50 +++
 34 files changed, 2359 insertions(+), 242 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/805e33b6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
new file mode 100644
index 000..e1ed5cb
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.util.DataChecksum;
+
+/** Composite CRC. */
+@InterfaceAudience.LimitedPrivate({"HDFS"})
+@InterfaceStability.Unstable
+public class CompositeCrcFileChecksum extends FileChecksum {
+  public static final int LENGTH = Integer.SIZE / Byte.SIZE;
+
+  private int crc;

[2/2] hadoop git commit: HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.

2018-04-10 Thread xiao

HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable 
across different instances/layouts. Contributed by Dennis Huo.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c9cdad6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c9cdad6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c9cdad6

Branch: refs/heads/trunk
Commit: 7c9cdad6d04c98db5a83e2108219bf6e6c903daf
Parents: 6cc59a0
Author: Xiao Chen 
Authored: Tue Apr 10 20:56:07 2018 -0700
Committer: Xiao Chen 
Committed: Tue Apr 10 21:31:48 2018 -0700

--
 .../hadoop/fs/CompositeCrcFileChecksum.java |  82 +
 .../main/java/org/apache/hadoop/fs/Options.java |  11 +
 .../org/apache/hadoop/util/CrcComposer.java | 187 ++
 .../java/org/apache/hadoop/util/CrcUtil.java| 220 +++
 .../org/apache/hadoop/util/DataChecksum.java|  18 +
 .../org/apache/hadoop/util/TestCrcComposer.java | 242 
 .../org/apache/hadoop/util/TestCrcUtil.java | 232 
 .../main/java/org/apache/hadoop/fs/Hdfs.java|   4 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  56 ++-
 .../hadoop/hdfs/DistributedFileSystem.java  |   5 +-
 .../apache/hadoop/hdfs/FileChecksumHelper.java  | 365 +--
 .../hdfs/client/HdfsClientConfigKeys.java   |   2 +
 .../hadoop/hdfs/client/impl/DfsClientConf.java  |  27 ++
 .../hdfs/protocol/BlockChecksumOptions.java |  54 +++
 .../hadoop/hdfs/protocol/BlockChecksumType.java |  30 ++
 .../datatransfer/DataTransferProtocol.java  |  12 +-
 .../hdfs/protocol/datatransfer/Sender.java  |  11 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |  44 +++
 .../src/main/proto/datatransfer.proto   |   7 +-
 .../src/main/proto/hdfs.proto   |  21 ++
 .../hdfs/protocol/datatransfer/Receiver.java|   8 +-
 .../server/datanode/BlockChecksumHelper.java| 289 +--
 .../hdfs/server/datanode/DataXceiver.java   |  26 +-
 ...dBlockChecksumCompositeCrcReconstructor.java |  80 
 ...StripedBlockChecksumMd5CrcReconstructor.java |  74 
 .../StripedBlockChecksumReconstructor.java  |  66 ++--
 .../erasurecode/StripedBlockReconstructor.java  |   1 +
 .../src/main/resources/hdfs-default.xml |  11 +
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  31 +-
 .../apache/hadoop/hdfs/TestFileChecksum.java| 101 -
 .../hdfs/TestFileChecksumCompositeCrc.java  |  47 +++
 .../hadoop/hdfs/protocolPB/TestPBHelper.java|  14 +
 .../hadoop/tools/mapred/TestCopyMapper.java | 173 +++--
 .../mapred/TestCopyMapperCompositeCrc.java  |  50 +++
 34 files changed, 2359 insertions(+), 242 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c9cdad6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
new file mode 100644
index 000..e1ed5cb
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.util.DataChecksum;
+
+/** Composite CRC. */
+@InterfaceAudience.LimitedPrivate({"HDFS"})
+@InterfaceStability.Unstable
+public class CompositeCrcFileChecksum extends FileChecksum {
+  public static final int LENGTH = Integer.SIZE / Byte.SIZE;
+
+  private int crc;
+

[2/2] hadoop git commit: HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.

[2/2] hadoop git commit: HDFS-13056. Expose file-level composite CRCs in HDFS which are comparable across different instances/layouts. Contributed by Dennis Huo.

2 matches

Site Navigation

Mail list logo

Footer information