This is an automated email from the ASF dual-hosted git repository.
yiyang0203 pushed a commit to branch ozone-1.4
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/ozone-1.4 by this push:
new 9ad6c39b7c HDDS-9886. `hadoop fs -checksum` fails with
`NoClassDefFoundError` on Hadoop 2 (#5799)
9ad6c39b7c is described below
commit 9ad6c39b7c07dd2e78929eba05d91ff033ee7bf9
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Sat Dec 16 08:18:27 2023 +0100
HDDS-9886. `hadoop fs -checksum` fails with `NoClassDefFoundError` on
Hadoop 2 (#5799)
---
.../client/checksum/BaseFileChecksumHelper.java | 3 -
.../client/checksum/ECBlockChecksumComputer.java | 2 -
.../client/checksum/ECFileChecksumHelper.java | 1 -
.../checksum/ReplicatedBlockChecksumComputer.java | 2 -
.../checksum/ReplicatedFileChecksumHelper.java | 1 -
.../TestReplicatedBlockChecksumComputer.java | 2 -
.../client/checksum/CompositeCrcFileChecksum.java | 88 +++++++
.../hadoop/ozone/client/checksum/CrcComposer.java | 215 ++++++++++++++++
.../hadoop/ozone/client/checksum/CrcUtil.java | 278 +++++++++++++++++++++
.../hadoop/ozone/client/checksum/package-info.java | 23 ++
.../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 4 +-
.../src/main/smoketest/ozonefs/hadoopo3fs.robot | 2 +
hadoop-ozone/ozonefs-hadoop2/pom.xml | 19 +-
.../apache/hadoop/fs/TestOmKeyInfoWithHadoop2.java | 24 ++
hadoop-ozone/pom.xml | 6 +
.../ozone/shell/keys/TestChecksumKeyHandler.java | 4 +-
16 files changed, 658 insertions(+), 16 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
index bf52bbd478..2d83d88ed5 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/BaseFileChecksumHelper.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.ozone.client.checksum;
-import org.apache.hadoop.fs.CompositeCrcFileChecksum;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
@@ -34,8 +33,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
-import org.apache.hadoop.util.CrcComposer;
-import org.apache.hadoop.util.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
index 8b3d816ed6..e0b82bebc3 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECBlockChecksumComputer.java
@@ -23,8 +23,6 @@ import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.util.CrcComposer;
-import org.apache.hadoop.util.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECFileChecksumHelper.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECFileChecksumHelper.java
index fb3ae5d287..3da2368841 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECFileChecksumHelper.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ECFileChecksumHelper.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.CrcUtil;
import java.io.IOException;
import java.nio.ByteBuffer;
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedBlockChecksumComputer.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedBlockChecksumComputer.java
index 3efb57acc0..cf976e3bd3 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedBlockChecksumComputer.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedBlockChecksumComputer.java
@@ -21,8 +21,6 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.io.MD5Hash;
-import org.apache.hadoop.util.CrcComposer;
-import org.apache.hadoop.util.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.slf4j.Logger;
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
index c8443c83b5..09f9c7d037 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum/ReplicatedFileChecksumHelper.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.CrcUtil;
import java.io.IOException;
import java.nio.ByteBuffer;
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedBlockChecksumComputer.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedBlockChecksumComputer.java
index 18f07a4aba..5cf4401bae 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedBlockChecksumComputer.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/checksum/TestReplicatedBlockChecksumComputer.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.ozone.client.checksum;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.io.MD5Hash;
-import org.apache.hadoop.util.CrcComposer;
-import org.apache.hadoop.util.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.jupiter.api.Test;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CompositeCrcFileChecksum.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CompositeCrcFileChecksum.java
new file mode 100644
index 0000000000..461c29ca11
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CompositeCrcFileChecksum.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.client.checksum;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.util.DataChecksum;
+
+/** Composite CRC. */
[email protected]
[email protected]
+public class CompositeCrcFileChecksum extends FileChecksum {
+ public static final int LENGTH = Integer.SIZE / Byte.SIZE;
+
+ private int crc;
+ private DataChecksum.Type crcType;
+ private int bytesPerCrc;
+
+ /**
+ * Create a CompositeCrcFileChecksum.
+ *
+ * @param crc crc.
+ * @param crcType crcType.
+ * @param bytesPerCrc bytesPerCrc.
+ */
+ public CompositeCrcFileChecksum(
+ int crc, DataChecksum.Type crcType, int bytesPerCrc) {
+ this.crc = crc;
+ this.crcType = crcType;
+ this.bytesPerCrc = bytesPerCrc;
+ }
+
+ @Override
+ public String getAlgorithmName() {
+ return "COMPOSITE-" + crcType.name();
+ }
+
+ @Override
+ public int getLength() {
+ return LENGTH;
+ }
+
+ @Override
+ public byte[] getBytes() {
+ return CrcUtil.intToBytes(crc);
+ }
+
+ @Override
+ public ChecksumOpt getChecksumOpt() {
+ return new ChecksumOpt(crcType, bytesPerCrc);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ crc = in.readInt();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(crc);
+ }
+
+ @Override
+ public String toString() {
+ return getAlgorithmName() + ":" + String.format("0x%08x", crc);
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java
new file mode 100644
index 0000000000..a9d6ac9b27
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcComposer.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.client.checksum;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.util.DataChecksum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+/**
+ * Encapsulates logic for composing multiple CRCs into one or more combined
CRCs
+ * corresponding to concatenated underlying data ranges. Optimized for
composing
+ * a large number of CRCs that correspond to underlying chunks of data all of
+ * same size.
+ */
[email protected]
[email protected]
+public class CrcComposer {
+ private static final int CRC_SIZE_BYTES = 4;
+ private static final Logger LOG = LoggerFactory.getLogger(CrcComposer.class);
+
+ private final int crcPolynomial;
+ private final int precomputedMonomialForHint;
+ private final long bytesPerCrcHint;
+ private final long stripeLength;
+
+ private int curCompositeCrc = 0;
+ private long curPositionInStripe = 0;
+ private ByteArrayOutputStream digestOut = new ByteArrayOutputStream();
+
+ /**
+ * Returns a CrcComposer which will collapse all ingested CRCs into a single
+ * value.
+ *
+ * @param type type.
+ * @param bytesPerCrcHint bytesPerCrcHint.
+ * @throws IOException raised on errors performing I/O.
+ * @return a CrcComposer which will collapse all ingested CRCs into a single
value.
+ */
+ public static CrcComposer newCrcComposer(
+ DataChecksum.Type type, long bytesPerCrcHint)
+ throws IOException {
+ return newStripedCrcComposer(type, bytesPerCrcHint, Long.MAX_VALUE);
+ }
+
+ /**
+ * Returns a CrcComposer which will collapse CRCs for every combined
+ * underlying data size which aligns with the specified stripe boundary. For
+ * example, if "update" is called with 20 CRCs and bytesPerCrc == 5, and
+ * stripeLength == 10, then every two (10 / 5) consecutive CRCs will be
+ * combined with each other, yielding a list of 10 CRC "stripes" in the
+ * final digest, each corresponding to 10 underlying data bytes. Using
+ * a stripeLength greater than the total underlying data size is equivalent
+ * to using a non-striped CrcComposer.
+ *
+ * @param type type.
+ * @param bytesPerCrcHint bytesPerCrcHint.
+ * @param stripeLength stripeLength.
+ * @return a CrcComposer which will collapse CRCs for every combined.
+ * underlying data size which aligns with the specified stripe boundary.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public static CrcComposer newStripedCrcComposer(
+ DataChecksum.Type type, long bytesPerCrcHint, long stripeLength)
+ throws IOException {
+ int polynomial =
org.apache.hadoop.ozone.client.checksum.CrcUtil.getCrcPolynomialForType(type);
+ return new CrcComposer(
+ polynomial,
+
org.apache.hadoop.ozone.client.checksum.CrcUtil.getMonomial(bytesPerCrcHint,
polynomial),
+ bytesPerCrcHint,
+ stripeLength);
+ }
+
+ CrcComposer(
+ int crcPolynomial,
+ int precomputedMonomialForHint,
+ long bytesPerCrcHint,
+ long stripeLength) {
+ LOG.debug(
+ "crcPolynomial=0x{}, precomputedMonomialForHint=0x{}, "
+ + "bytesPerCrcHint={}, stripeLength={}",
+ Integer.toString(crcPolynomial, 16),
+ Integer.toString(precomputedMonomialForHint, 16),
+ bytesPerCrcHint,
+ stripeLength);
+ this.crcPolynomial = crcPolynomial;
+ this.precomputedMonomialForHint = precomputedMonomialForHint;
+ this.bytesPerCrcHint = bytesPerCrcHint;
+ this.stripeLength = stripeLength;
+ }
+
+ /**
+ * Composes length / CRC_SIZE_IN_BYTES more CRCs from crcBuffer, with
+ * each CRC expected to correspond to exactly {@code bytesPerCrc} underlying
+ * data bytes.
+ *
+ * @param crcBuffer crcBuffer.
+ * @param offset offset.
+ * @param length must be a multiple of the expected byte-size of a CRC.
+ * @param bytesPerCrc bytesPerCrc.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public void update(
+ byte[] crcBuffer, int offset, int length, long bytesPerCrc)
+ throws IOException {
+ if (length % CRC_SIZE_BYTES != 0) {
+ throw new IOException(String.format(
+ "Trying to update CRC from byte array with length '%d' at offset "
+ + "'%d' which is not a multiple of %d!",
+ length, offset, CRC_SIZE_BYTES));
+ }
+ int limit = offset + length;
+ while (offset < limit) {
+ int crcB =
org.apache.hadoop.ozone.client.checksum.CrcUtil.readInt(crcBuffer, offset);
+ update(crcB, bytesPerCrc);
+ offset += CRC_SIZE_BYTES;
+ }
+ }
+
+ /**
+ * Composes {@code numChecksumsToRead} additional CRCs into the current
digest
+ * out of {@code checksumIn}, with each CRC expected to correspond to exactly
+ * {@code bytesPerCrc} underlying data bytes.
+ *
+ * @param checksumIn checksumIn.
+ * @param numChecksumsToRead numChecksumsToRead.
+ * @param bytesPerCrc bytesPerCrc.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public void update(
+ DataInputStream checksumIn, long numChecksumsToRead, long bytesPerCrc)
+ throws IOException {
+ for (long i = 0; i < numChecksumsToRead; ++i) {
+ int crcB = checksumIn.readInt();
+ update(crcB, bytesPerCrc);
+ }
+ }
+
+ /**
+ * Updates with a single additional CRC which corresponds to an underlying
+ * data size of {@code bytesPerCrc}.
+ *
+ * @param crcB crcB.
+ * @param bytesPerCrc bytesPerCrc.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public void update(int crcB, long bytesPerCrc) throws IOException {
+ if (curCompositeCrc == 0) {
+ curCompositeCrc = crcB;
+ } else if (bytesPerCrc == bytesPerCrcHint) {
+ curCompositeCrc =
org.apache.hadoop.ozone.client.checksum.CrcUtil.composeWithMonomial(
+ curCompositeCrc, crcB, precomputedMonomialForHint, crcPolynomial);
+ } else {
+ curCompositeCrc =
org.apache.hadoop.ozone.client.checksum.CrcUtil.compose(
+ curCompositeCrc, crcB, bytesPerCrc, crcPolynomial);
+ }
+
+ curPositionInStripe += bytesPerCrc;
+
+ if (curPositionInStripe > stripeLength) {
+ throw new IOException(String.format(
+ "Current position in stripe '%d' after advancing by bytesPerCrc '%d'
"
+ + "exceeds stripeLength '%d' without stripe alignment.",
+ curPositionInStripe, bytesPerCrc, stripeLength));
+ } else if (curPositionInStripe == stripeLength) {
+ // Hit a stripe boundary; flush the curCompositeCrc and reset for next
+ // stripe.
+
digestOut.write(org.apache.hadoop.ozone.client.checksum.CrcUtil.intToBytes(curCompositeCrc),
0, CRC_SIZE_BYTES);
+ curCompositeCrc = 0;
+ curPositionInStripe = 0;
+ }
+ }
+
+ /**
+ * Returns byte representation of composed CRCs; if no stripeLength was
+ * specified, the digest should be of length equal to exactly one CRC.
+ * Otherwise, the number of CRCs in the returned array is equal to the
+ * total sum bytesPerCrc divided by stripeLength. If the sum of bytesPerCrc
+ * is not a multiple of stripeLength, then the last CRC in the array
+ * corresponds to totalLength % stripeLength underlying data bytes.
+ *
+ * @return byte representation of composed CRCs.
+ */
+ public byte[] digest() {
+ if (curPositionInStripe > 0) {
+ digestOut.write(CrcUtil.intToBytes(curCompositeCrc), 0, CRC_SIZE_BYTES);
+ curCompositeCrc = 0;
+ curPositionInStripe = 0;
+ }
+ byte[] digestValue = digestOut.toByteArray();
+ digestOut.reset();
+ return digestValue;
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java
new file mode 100644
index 0000000000..57ec23dc2b
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/CrcUtil.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.client.checksum;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.util.DataChecksum;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * This class provides utilities for working with CRCs.
+ */
[email protected]
[email protected]
+public final class CrcUtil {
+ public static final int MULTIPLICATIVE_IDENTITY = 0x80000000;
+ public static final int GZIP_POLYNOMIAL = 0xEDB88320;
+ public static final int CASTAGNOLI_POLYNOMIAL = 0x82F63B78;
+
+ /**
+ * Hide default constructor for a static utils class.
+ */
+ private CrcUtil() {
+ }
+
+ /**
+ * getCrcPolynomialForType.
+ *
+ * @param type type.
+ * @return the int representation of the polynomial associated with the
+ * CRC {@code type}, suitable for use with further CRC arithmetic.
+ * @throws IOException if there is no CRC polynomial applicable
+ * to the given {@code type}.
+ */
+ public static int getCrcPolynomialForType(DataChecksum.Type type) throws
IOException {
+ switch (type) {
+ case CRC32:
+ return GZIP_POLYNOMIAL;
+ case CRC32C:
+ return CASTAGNOLI_POLYNOMIAL;
+ default:
+ throw new IOException(
+ "No CRC polynomial could be associated with type: " + type);
+ }
+ }
+
+ /**
+ * Compute x^({@code lengthBytes} * 8) mod {@code mod}, where {@code mod} is
+ * in "reversed" (little-endian) format such that {@code mod & 1} represents
+ * x^31 and has an implicit term x^32.
+ *
+ * @param lengthBytes lengthBytes.
+ * @param mod mod.
+ * @return monomial.
+ */
+ public static int getMonomial(long lengthBytes, int mod) {
+ if (lengthBytes == 0) {
+ return MULTIPLICATIVE_IDENTITY;
+ } else if (lengthBytes < 0) {
+ throw new IllegalArgumentException(
+ "lengthBytes must be positive, got " + lengthBytes);
+ }
+
+ // Decompose into
+ // x^degree == x ^ SUM(bit[i] * 2^i) == PRODUCT(x ^ (bit[i] * 2^i))
+ // Generate each x^(2^i) by squaring.
+ // Since 'degree' is in 'bits', but we only need to support byte
+ // granularity we can begin with x^8.
+ int multiplier = MULTIPLICATIVE_IDENTITY >>> 8;
+ int product = MULTIPLICATIVE_IDENTITY;
+ long degree = lengthBytes;
+ while (degree > 0) {
+ if ((degree & 1) != 0) {
+ product = (product == MULTIPLICATIVE_IDENTITY) ? multiplier :
+ galoisFieldMultiply(product, multiplier, mod);
+ }
+ multiplier = galoisFieldMultiply(multiplier, multiplier, mod);
+ degree >>= 1;
+ }
+ return product;
+ }
+
+ /**
+ * composeWithMonomial.
+ *
+ * @param crcA crcA.
+ * @param crcB crcB.
+ * @param monomial Precomputed x^(lengthBInBytes * 8) mod {@code mod}
+ * @param mod mod.
+ * @return compose with monomial.
+ */
+ public static int composeWithMonomial(
+ int crcA, int crcB, int monomial, int mod) {
+ return galoisFieldMultiply(crcA, monomial, mod) ^ crcB;
+ }
+
+ /**
+ * compose.
+ *
+ * @param crcA crcA.
+ * @param crcB crcB.
+ * @param lengthB length of content corresponding to {@code crcB}, in bytes.
+ * @param mod mod.
+ * @return compose result.
+ */
+ public static int compose(int crcA, int crcB, long lengthB, int mod) {
+ int monomial = getMonomial(lengthB, mod);
+ return composeWithMonomial(crcA, crcB, monomial, mod);
+ }
+
+ /**
+ * @return 4-byte array holding the big-endian representation of
+ * {@code value}.
+ *
+ * @param value value.
+ */
+ public static byte[] intToBytes(int value) {
+ byte[] buf = new byte[4];
+ try {
+ writeInt(buf, 0, value);
+ } catch (IOException ioe) {
+ // Since this should only be able to occur from code bugs within this
+ // class rather than user input, we throw as a RuntimeException
+ // rather than requiring this method to declare throwing IOException
+ // for something the caller can't control.
+ throw new RuntimeException(ioe);
+ }
+ return buf;
+ }
+
+ /**
+ * Writes big-endian representation of {@code value} into {@code buf}
+ * starting at {@code offset}. buf.length must be greater than or
+ * equal to offset + 4.
+ *
+ * @param buf buf size.
+ * @param offset offset.
+ * @param value value.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public static void writeInt(byte[] buf, int offset, int value)
+ throws IOException {
+ if (offset + 4 > buf.length) {
+ throw new IOException(String.format(
+ "writeInt out of bounds: buf.length=%d, offset=%d",
+ buf.length, offset));
+ }
+ buf[offset + 0] = (byte)((value >>> 24) & 0xff);
+ buf[offset + 1] = (byte)((value >>> 16) & 0xff);
+ buf[offset + 2] = (byte)((value >>> 8) & 0xff);
+ buf[offset + 3] = (byte)(value & 0xff);
+ }
+
+ /**
+ * Reads 4-byte big-endian int value from {@code buf} starting at
+ * {@code offset}. buf.length must be greater than or equal to offset + 4.
+ *
+ * @param offset offset.
+ * @param buf buf.
+ * @return int.
+ * @throws IOException raised on errors performing I/O.
+ */
+ public static int readInt(byte[] buf, int offset)
+ throws IOException {
+ if (offset + 4 > buf.length) {
+ throw new IOException(String.format(
+ "readInt out of bounds: buf.length=%d, offset=%d",
+ buf.length, offset));
+ }
+ int value = ((buf[offset + 0] & 0xff) << 24) |
+ ((buf[offset + 1] & 0xff) << 16) |
+ ((buf[offset + 2] & 0xff) << 8) |
+ ((buf[offset + 3] & 0xff));
+ return value;
+ }
+
+ /**
+ * For use with debug statements; verifies bytes.length on creation,
+ * expecting it to represent exactly one CRC, and returns a hex
+ * formatted value.
+ *
+ * @param bytes bytes.
+ * @throws IOException raised on errors performing I/O.
+ * @return a list of hex formatted values.
+ */
+ public static String toSingleCrcString(final byte[] bytes)
+ throws IOException {
+ if (bytes.length != 4) {
+ throw new IOException((String.format(
+ "Unexpected byte[] length '%d' for single CRC. Contents: %s",
+ bytes.length, Arrays.toString(bytes))));
+ }
+ return String.format("0x%08x", readInt(bytes, 0));
+ }
+
+ /**
+ * For use with debug statements; verifies bytes.length on creation,
+ * expecting it to be divisible by CRC byte size, and returns a list of
+ * hex formatted values.
+ *
+ * @param bytes bytes.
+ * @throws IOException raised on errors performing I/O.
+ * @return a list of hex formatted values.
+ */
+ public static String toMultiCrcString(final byte[] bytes)
+ throws IOException {
+ if (bytes.length % 4 != 0) {
+ throw new IOException((String.format(
+ "Unexpected byte[] length '%d' not divisible by 4. Contents: %s",
+ bytes.length, Arrays.toString(bytes))));
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append('[');
+ for (int i = 0; i < bytes.length; i += 4) {
+ sb.append(String.format("0x%08x", readInt(bytes, i)));
+ if (i != bytes.length - 4) {
+ sb.append(", ");
+ }
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+
+ /**
+ * Galois field multiplication of {@code p} and {@code q} with the
+ * generator polynomial {@code m} as the modulus.
+ *
+ * @param m The little-endian polynomial to use as the modulus when
+ * multiplying p and q, with implicit "1" bit beyond the bottom bit.
+ */
+ private static int galoisFieldMultiply(int p, int q, int m) {
+ int summation = 0;
+
+ // Top bit is the x^0 place; each right-shift increments the degree of the
+ // current term.
+ int curTerm = MULTIPLICATIVE_IDENTITY;
+
+ // Iteratively multiply p by x mod m as we go to represent the q[i] term
+ // (of degree x^i) times p.
+ int px = p;
+
+ while (curTerm != 0) {
+ if ((q & curTerm) != 0) {
+ summation ^= px;
+ }
+
+ // Bottom bit represents highest degree since we're little-endian; before
+ // we multiply by "x" for the next term, check bottom bit to know whether
+ // the resulting px will thus have a term matching the implicit "1" term
+ // of "m" and thus will need to subtract "m" after mutiplying by "x".
+ boolean hasMaxDegree = ((px & 1) != 0);
+ px >>>= 1;
+ if (hasMaxDegree) {
+ px ^= m;
+ }
+ curTerm >>>= 1;
+ }
+ return summation;
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/package-info.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/package-info.java
new file mode 100644
index 0000000000..6dd2a9a992
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/checksum/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains checksum calculation shared between client and
+ * server code.
+ */
+package org.apache.hadoop.ozone.client.checksum;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
index 2b6a897cd4..9836379a59 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.protocolPB;
import com.google.protobuf.ByteString;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
-import org.apache.hadoop.fs.CompositeCrcFileChecksum;
+import org.apache.hadoop.ozone.client.checksum.CompositeCrcFileChecksum;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
@@ -47,7 +47,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRespo
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.ozone.client.checksum.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
index 51fab51d82..7fd25334e2 100644
--- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/hadoopo3fs.robot
@@ -38,3 +38,5 @@ Test hadoop dfs
Should contain ${result}
${PREFIX}-${random}
${result} = Execute hdfs dfs -cat
${dir}/${PREFIX}-${random}
Should contain ${result} This product
includes software developed
+ ${result} = Execute hdfs dfs -checksum
${dir}/${PREFIX}-${random}
+ Should contain ${result}
${PREFIX}-${random}
diff --git a/hadoop-ozone/ozonefs-hadoop2/pom.xml
b/hadoop-ozone/ozonefs-hadoop2/pom.xml
index 7a53718661..2e6a6408cb 100644
--- a/hadoop-ozone/ozonefs-hadoop2/pom.xml
+++ b/hadoop-ozone/ozonefs-hadoop2/pom.xml
@@ -26,7 +26,6 @@
<packaging>jar</packaging>
<version>1.4.0-SNAPSHOT</version>
<properties>
- <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so
far -->
<shaded.prefix>org.apache.hadoop.ozone.shaded</shaded.prefix>
</properties>
<dependencies>
@@ -97,6 +96,24 @@
<artifactId>slf4j-reload4j</artifactId>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-params</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git
a/hadoop-ozone/ozonefs-hadoop2/src/test/java/org/apache/hadoop/fs/TestOmKeyInfoWithHadoop2.java
b/hadoop-ozone/ozonefs-hadoop2/src/test/java/org/apache/hadoop/fs/TestOmKeyInfoWithHadoop2.java
new file mode 100644
index 0000000000..211226d941
--- /dev/null
+++
b/hadoop-ozone/ozonefs-hadoop2/src/test/java/org/apache/hadoop/fs/TestOmKeyInfoWithHadoop2.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.ozone.om.helpers.TestOmKeyInfo;
+
+class TestOmKeyInfoWithHadoop2 extends TestOmKeyInfo {
+ // test OmKeyInfo (including proto conversion) with Hadoop 2 on the classpath
+}
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index e2473a773d..eb7feb1d68 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -64,6 +64,12 @@
<artifactId>ozone-common</artifactId>
<version>${ozone.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-common</artifactId>
+ <version>${ozone.version}</version>
+ <type>test-jar</type>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-client</artifactId>
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
index 1279adcc8a..c55cb9f55a 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/shell/keys/TestChecksumKeyHandler.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.ozone.shell.keys;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.fs.CompositeCrcFileChecksum;
+import org.apache.hadoop.ozone.client.checksum.CompositeCrcFileChecksum;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -29,7 +29,7 @@ import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.shell.OzoneAddress;
-import org.apache.hadoop.util.CrcUtil;
+import org.apache.hadoop.ozone.client.checksum.CrcUtil;
import org.apache.hadoop.util.DataChecksum;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
