This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 66d90577c3 HDDS-8663. Support CodecBuffer for the remaining protobuf
v2 codecs in hadoop-hdds. (#4768)
66d90577c3 is described below
commit 66d90577c340a7b0a6abe444164d385c6e936eac
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Thu May 25 16:31:09 2023 +0800
HDDS-8663. Support CodecBuffer for the remaining protobuf v2 codecs in
hadoop-hdds. (#4768)
---
.../hadoop/hdds/scm/container/ContainerID.java | 3 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 123 +++------------------
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 30 ++++-
.../hadoop/hdds/scm/pipeline/PipelineID.java | 3 +-
.../hadoop/hdds/utils/db/DelegatedCodec.java | 38 +++++--
.../container/common/helpers/ChunkInfoList.java | 2 +-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 3 +-
.../metadata/DatanodeSchemaTwoDBDefinition.java | 3 +-
.../metadata/DeletedBlocksTransactionCodec.java | 46 --------
.../container/common/helpers/MoveDataNodePair.java | 41 ++++---
.../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +-
.../hdds/scm/metadata/ContainerInfoCodec.java | 47 --------
.../metadata/DeletedBlocksTransactionCodec.java | 55 ---------
.../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 ---------
.../hadoop/hdds/scm/metadata/PipelineCodec.java | 57 ----------
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 9 +-
.../hadoop/ozone/om/helpers/OmDBAccessIdInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmDBTenantState.java | 2 +-
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 5 +-
.../ozone/snapshot/SnapshotDiffReportOzone.java | 4 +-
.../ozone/recon/scm/ReconSCMDBDefinition.java | 3 +-
21 files changed, 127 insertions(+), 404 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
index b7ecd6b9c4..64e70db296 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java
@@ -37,7 +37,8 @@ import org.apache.hadoop.hdds.utils.db.LongCodec;
*/
public final class ContainerID implements Comparable<ContainerID> {
private static final Codec<ContainerID> CODEC = new DelegatedCodec<>(
- LongCodec.get(), ContainerID::valueOf, c -> c.id, true);
+ LongCodec.get(), ContainerID::valueOf, c -> c.id,
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<ContainerID> getCodec() {
return CODEC;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
index 67a0d433cc..3f6c2a7df9 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
@@ -17,12 +17,7 @@
*/
package org.apache.hadoop.hdds.scm.container;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
import java.time.Instant;
-import java.util.Arrays;
import java.util.Comparator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -30,6 +25,9 @@ import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
import org.apache.hadoop.util.Time;
import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -41,17 +39,23 @@ import org.apache.commons.lang3.builder.HashCodeBuilder;
/**
* Class wraps ozone container info.
*/
-public class ContainerInfo implements Comparator<ContainerInfo>,
- Comparable<ContainerInfo>, Externalizable {
+public final class ContainerInfo implements Comparable<ContainerInfo> {
+ private static final Comparator<ContainerInfo> COMPARATOR
+ = Comparator.comparingLong(info -> info.getLastUsed().toEpochMilli());
- private static final String SERIALIZATION_ERROR_MSG = "Java serialization
not"
- + " supported. Use protobuf instead.";
+ private static final Codec<ContainerInfo> CODEC = new DelegatedCodec<>(
+ Proto2Codec.get(HddsProtos.ContainerInfoProto.class),
+ ContainerInfo::fromProtobuf,
+ ContainerInfo::getProtobuf);
+ public static Codec<ContainerInfo> getCodec() {
+ return CODEC;
+ }
private HddsProtos.LifeCycleState state;
@JsonIgnore
- private PipelineID pipelineID;
- private ReplicationConfig replicationConfig;
+ private final PipelineID pipelineID;
+ private final ReplicationConfig replicationConfig;
/*
usedBytes is a volatile field. Writes and Reads of volatile long are atomic
and each read of a volatile will see the last write to that volatile by any
@@ -62,14 +66,14 @@ public class ContainerInfo implements
Comparator<ContainerInfo>,
private long numberOfKeys;
private Instant lastUsed;
// The wall-clock ms since the epoch at which the current state enters.
- private Instant stateEnterTime;
+ private final Instant stateEnterTime;
private String owner;
// This is JsonIgnored as originally this class held a long in instead of
// a containerID object. By emitting this in Json, it changes the JSON
output.
// Therefore the method getContainerID is annotated to return the original
// field and hence maintain the original output.
@JsonIgnore
- private ContainerID containerID;
+ private final ContainerID containerID;
// Delete Transaction Id is updated when new transaction for a container
// is stored in SCM delete Table.
// TODO: Replication Manager should consider deleteTransactionId so that
@@ -80,13 +84,6 @@ public class ContainerInfo implements
Comparator<ContainerInfo>,
// container replica should have the same sequenceId.
private long sequenceId;
- /**
- * Allows you to maintain private data on ContainerInfo. This is not
- * serialized via protobuf, just allows us to maintain some private data.
- */
- @JsonIgnore
- private byte[] data;
-
@SuppressWarnings("parameternumber")
ContainerInfo(
long containerID,
@@ -112,12 +109,6 @@ public class ContainerInfo implements
Comparator<ContainerInfo>,
this.replicationConfig = repConfig;
}
- /**
- * Needed for serialization findbugs.
- */
- public ContainerInfo() {
- }
-
public static ContainerInfo fromProtobuf(HddsProtos.ContainerInfoProto info)
{
ContainerInfo.Builder builder = new ContainerInfo.Builder();
final ReplicationConfig config = ReplicationConfig
@@ -334,26 +325,6 @@ public class ContainerInfo implements
Comparator<ContainerInfo>,
.toHashCode();
}
- /**
- * Compares its two arguments for order. Returns a negative integer, zero,
or
- * a positive integer as the first argument is less than, equal to, or
greater
- * than the second.<p>
- *
- * @param o1 the first object to be compared.
- * @param o2 the second object to be compared.
- * @return a negative integer, zero, or a positive integer as the first
- * argument is less than, equal to, or greater than the second.
- * @throws NullPointerException if an argument is null and this comparator
- * does not permit null arguments
- * @throws ClassCastException if the arguments' types prevent them from
- * being compared by this comparator.
- */
- @Override
- public int compare(ContainerInfo o1, ContainerInfo o2) {
- return Long.compare(
- o1.getLastUsed().toEpochMilli(), o2.getLastUsed().toEpochMilli());
- }
-
/**
* Compares this object with the specified object for order. Returns a
* negative integer, zero, or a positive integer as this object is less than,
@@ -368,65 +339,7 @@ public class ContainerInfo implements
Comparator<ContainerInfo>,
*/
@Override
public int compareTo(ContainerInfo o) {
- return this.compare(this, o);
- }
-
-
-
- /**
- * Returns private data that is set on this containerInfo.
- *
- * @return blob, the user can interpret it any way they like.
- */
- public byte[] getData() {
- if (this.data != null) {
- return Arrays.copyOf(this.data, this.data.length);
- } else {
- return null;
- }
- }
-
- /**
- * Set private data on ContainerInfo object.
- *
- * @param data -- private data.
- */
- public void setData(byte[] data) {
- if (data != null) {
- this.data = Arrays.copyOf(data, data.length);
- }
- }
-
- /**
- * Throws IOException as default java serialization is not supported. Use
- * serialization via protobuf instead.
- *
- * @param out the stream to write the object to
- * @throws IOException Includes any I/O exceptions that may occur
- * @serialData Overriding methods should use this tag to describe
- * the data layout of this Externalizable object.
- * List the sequence of element types and, if possible,
- * relate the element to a public/protected field and/or
- * method of this Externalizable class.
- */
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- throw new IOException(SERIALIZATION_ERROR_MSG);
- }
-
- /**
- * Throws IOException as default java serialization is not supported. Use
- * serialization via protobuf instead.
- *
- * @param in the stream to read data from in order to restore the object
- * @throws IOException if I/O errors occur
- * @throws ClassNotFoundException If the class for an object being
- * restored cannot be found.
- */
- @Override
- public void readExternal(ObjectInput in)
- throws IOException, ClassNotFoundException {
- throw new IOException(SERIALIZATION_ERROR_MSG);
+ return COMPARATOR.compare(this, o);
}
/**
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
index 06950f7f69..e198cae937 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -42,6 +42,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
+import org.apache.hadoop.ozone.ClientVersion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,12 +55,26 @@ import com.google.common.base.Preconditions;
* Represents a group of datanodes which store a container.
*/
public final class Pipeline {
+ /**
+ * This codec is inconsistent since
+ * deserialize(serialize(original)) does not equal to original
+ * -- the creation time may change.
+ */
+ private static final Codec<Pipeline> CODEC = new DelegatedCodec<>(
+ Proto2Codec.get(HddsProtos.Pipeline.class),
+ Pipeline::getFromProtobufSetCreationTimestamp,
+ p -> p.getProtobufMessage(ClientVersion.CURRENT_VERSION),
+ DelegatedCodec.CopyType.UNSUPPORTED);
+
+ public static Codec<Pipeline> getCodec() {
+ return CODEC;
+ }
private static final Logger LOG = LoggerFactory.getLogger(Pipeline.class);
private final PipelineID id;
private final ReplicationConfig replicationConfig;
- private PipelineState state;
+ private final PipelineState state;
private Map<DatanodeDetails, Long> nodeStatus;
private Map<DatanodeDetails, Integer> replicaIndexes;
// nodes with ordered distance to client
@@ -382,6 +400,14 @@ public final class Pipeline {
return builder.build();
}
+ static Pipeline getFromProtobufSetCreationTimestamp(
+ HddsProtos.Pipeline proto) throws UnknownPipelineStateException {
+ final Pipeline pipeline = getFromProtobuf(proto);
+ // When SCM is restarted, set Creation time with current time.
+ pipeline.setCreationTimestamp(Instant.now());
+ return pipeline;
+ }
+
public static Pipeline getFromProtobuf(HddsProtos.Pipeline pipeline)
throws UnknownPipelineStateException {
Preconditions.checkNotNull(pipeline, "Pipeline is null");
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
index f0cc5a381d..5ca3545626 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineID.java
@@ -33,7 +33,8 @@ import java.util.UUID;
*/
public final class PipelineID {
private static final Codec<PipelineID> CODEC = new DelegatedCodec<>(
- UuidCodec.get(), PipelineID::valueOf, c -> c.id, true);
+ UuidCodec.get(), PipelineID::valueOf, c -> c.id,
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<PipelineID> getCodec() {
return CODEC;
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
index a24436c362..ab78cfb59f 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/DelegatedCodec.java
@@ -30,10 +30,25 @@ import java.util.function.IntFunction;
* @param <DELEGATE> The object type of the {@link #delegate}.
*/
public class DelegatedCodec<T, DELEGATE> implements Codec<T> {
+ /** How to {@link #copyObject(Object)}? */
+ public enum CopyType {
+ /** Deep copy -- duplicate the underlying fields of the object. */
+ DEEP,
+ /** Shallow copy -- only duplicate the reference of the object. */
+ SHALLOW,
+ /**
+ * Copy is unsupported
+ * due to some reason such as the codec being inconsistent.
+ * <p>
+ * Consistency: deserialize(serialize(original)) equals to original.
+ */
+ UNSUPPORTED
+ }
+
private final Codec<DELEGATE> delegate;
private final CheckedFunction<DELEGATE, T, IOException> forward;
private final CheckedFunction<T, DELEGATE, IOException> backward;
- private final boolean shallowCopy;
+ private final CopyType copyType;
/**
* Construct a {@link Codec} using the given delegate.
@@ -41,24 +56,23 @@ public class DelegatedCodec<T, DELEGATE> implements
Codec<T> {
* @param delegate the delegate {@link Codec}
* @param forward a function to convert {@link DELEGATE} to {@link T}.
* @param backward a function to convert {@link T} back to {@link DELEGATE}.
- * @param shallowCopy Should it use shallow copy
- * in {@link #copyObject(Object)}?
+ * @param copyType How to {@link #copyObject(Object)}?
*/
public DelegatedCodec(Codec<DELEGATE> delegate,
CheckedFunction<DELEGATE, T, IOException> forward,
CheckedFunction<T, DELEGATE, IOException> backward,
- boolean shallowCopy) {
+ CopyType copyType) {
this.delegate = delegate;
this.forward = forward;
this.backward = backward;
- this.shallowCopy = shallowCopy;
+ this.copyType = copyType;
}
- /** The same as new DelegatedCodec(delegate, forward, backward, false). */
+ /** The same as new DelegatedCodec(delegate, forward, backward, DEEP). */
public DelegatedCodec(Codec<DELEGATE> delegate,
CheckedFunction<DELEGATE, T, IOException> forward,
CheckedFunction<T, DELEGATE, IOException> backward) {
- this(delegate, forward, backward, false);
+ this(delegate, forward, backward, CopyType.DEEP);
}
@Override
@@ -90,12 +104,18 @@ public class DelegatedCodec<T, DELEGATE> implements
Codec<T> {
@Override
public T copyObject(T message) {
- if (shallowCopy) {
+ if (copyType == CopyType.SHALLOW) {
return message;
- } else if (message instanceof CopyObject) {
+ } else if (copyType == CopyType.UNSUPPORTED) {
+ throw new UnsupportedOperationException();
+ }
+
+ if (message instanceof CopyObject) {
final CopyObject<T> casted = ((CopyObject<T>) message);
return casted.copyObject();
}
+
+ // Deep copy
try {
return forward.apply(delegate.copyObject(backward.apply(message)));
} catch (IOException e) {
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java
index b0aa9cb47c..0254dfe6bd 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfoList.java
@@ -36,7 +36,7 @@ public class ChunkInfoList {
Proto3Codec.get(ContainerProtos.ChunkInfoList.class),
ChunkInfoList::getFromProtoBuf,
ChunkInfoList::getProtoBufMessage,
- true);
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<ChunkInfoList> getCodec() {
return CODEC;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
index 630f2fed42..679478230d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
@@ -24,6 +24,7 @@ import
org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.FixedLengthStringUtils;
import org.apache.hadoop.hdds.utils.db.LongCodec;
import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
@@ -86,7 +87,7 @@ public class DatanodeSchemaThreeDBDefinition
String.class,
new FixedLengthStringCodec(),
DeletedBlocksTransaction.class,
- new DeletedBlocksTransactionCodec());
+ Proto2Codec.get(DeletedBlocksTransaction.class));
private static String separator = "";
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
index 2b6aff3e1d..a777139fb2 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.LongCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
@@ -69,7 +70,7 @@ public class DatanodeSchemaTwoDBDefinition extends
Long.class,
LongCodec.get(),
StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction.class,
- new DeletedBlocksTransactionCodec());
+ Proto2Codec.get(DeletedBlocksTransaction.class));
public DatanodeSchemaTwoDBDefinition(String dbPath,
ConfigurationSource config) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DeletedBlocksTransactionCodec.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DeletedBlocksTransactionCodec.java
deleted file mode 100644
index 90c26fe22b..0000000000
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DeletedBlocksTransactionCodec.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.metadata;
-
-import org.apache.hadoop.hdds.utils.db.Codec;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-
-import java.io.IOException;
-
-/**
- * Supports encoding and decoding {@link DeletedBlocksTransaction} objects.
- */
-public class DeletedBlocksTransactionCodec
- implements Codec<DeletedBlocksTransaction> {
-
- @Override public byte[] toPersistedFormat(
- DeletedBlocksTransaction deletedBlocksTransaction) {
- return deletedBlocksTransaction.toByteArray();
- }
-
- @Override public DeletedBlocksTransaction fromPersistedFormat(byte[] rawData)
- throws IOException {
- return DeletedBlocksTransaction.parseFrom(rawData);
- }
-
- @Override public DeletedBlocksTransaction copyObject(
- DeletedBlocksTransaction deletedBlocksTransaction) {
- throw new UnsupportedOperationException();
- }
-}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java
index 578134e64c..20de146813 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
@@ -14,21 +14,34 @@
* License for the specific language governing permissions and limitations
under
* the License.
*/
-
-
package org.apache.hadoop.hdds.scm.container.common.helpers;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.ratis.util.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.MoveDataNodePairProto;
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
+import org.apache.hadoop.ozone.ClientVersion;
-import java.io.IOException;
+import java.util.Objects;
/**
* MoveDataNodePair encapsulates the source and target
* datanodes of a move option.
+ * <p>
+ * This class is immutable.
*/
public class MoveDataNodePair {
+ private static final Codec<MoveDataNodePair> CODEC = new DelegatedCodec<>(
+ Proto2Codec.get(MoveDataNodePairProto.class),
+ MoveDataNodePair::getFromProtobuf,
+ pair -> pair.getProtobufMessage(ClientVersion.CURRENT_VERSION),
+ DelegatedCodec.CopyType.SHALLOW);
+
+ public static Codec<MoveDataNodePair> getCodec() {
+ return CODEC;
+ }
+
/**
* source datanode of current move option.
*/
@@ -52,18 +65,16 @@ public class MoveDataNodePair {
return src;
}
- public HddsProtos.MoveDataNodePairProto getProtobufMessage(int clientVersion)
- throws IOException {
- HddsProtos.MoveDataNodePairProto.Builder builder =
- HddsProtos.MoveDataNodePairProto.newBuilder()
- .setSrc(src.toProto(clientVersion))
- .setTgt(tgt.toProto(clientVersion));
- return builder.build();
+ public MoveDataNodePairProto getProtobufMessage(int clientVersion) {
+ return MoveDataNodePairProto.newBuilder()
+ .setSrc(src.toProto(clientVersion))
+ .setTgt(tgt.toProto(clientVersion))
+ .build();
}
public static MoveDataNodePair getFromProtobuf(
- HddsProtos.MoveDataNodePairProto mdnpp) {
- Preconditions.assertNotNull(mdnpp, "MoveDataNodePair is null");
+ MoveDataNodePairProto mdnpp) {
+ Objects.requireNonNull(mdnpp, "mdnpp == null");
DatanodeDetails src = DatanodeDetails.getFromProtoBuf(mdnpp.getSrc());
DatanodeDetails tgt = DatanodeDetails.getFromProtoBuf(mdnpp.getTgt());
return new MoveDataNodePair(src, tgt);
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java
index 274e0906ea..982b6ebc41 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/TransactionInfo.java
@@ -44,7 +44,7 @@ public final class TransactionInfo {
StringCodec.get(),
TransactionInfo::new,
TransactionInfo::generateTransactionInfo,
- true);
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<TransactionInfo> getCodec() {
return CODEC;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java
deleted file mode 100644
index 6b262150c6..0000000000
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/ContainerInfoCodec.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.scm.metadata;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.utils.db.Codec;
-
-/**
- * Codec to serialize / deserialize ContainerInfo.
- */
-public class ContainerInfoCodec implements Codec<ContainerInfo> {
-
- @Override
- public byte[] toPersistedFormat(ContainerInfo container) throws IOException {
- return container.getProtobuf().toByteArray();
- }
-
- @Override
- public ContainerInfo fromPersistedFormat(byte[] rawData) throws IOException {
- return ContainerInfo.fromProtobuf(
- ContainerInfoProto.PARSER.parseFrom(rawData));
- }
-
- @Override
- public ContainerInfo copyObject(ContainerInfo object) {
- throw new UnsupportedOperationException();
- }
-}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/DeletedBlocksTransactionCodec.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/DeletedBlocksTransactionCodec.java
deleted file mode 100644
index 3cc65d66d0..0000000000
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/DeletedBlocksTransactionCodec.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.metadata;
-
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import java.io.IOException;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.hdds.utils.db.Codec;
-
-/**
- * Codec for Persisting the DeletedBlocks.
- */
-public class DeletedBlocksTransactionCodec
- implements Codec<DeletedBlocksTransaction> {
- @Override
- public byte[] toPersistedFormat(DeletedBlocksTransaction object)
- throws IOException {
- return object.toByteArray();
- }
-
- @Override
- public DeletedBlocksTransaction fromPersistedFormat(byte[] rawData)
- throws IOException {
- try {
- return DeletedBlocksTransaction.parseFrom(rawData);
- } catch (InvalidProtocolBufferException e) {
- throw new IllegalArgumentException(
- "Can't convert rawBytes to DeletedBlocksTransaction.", e);
- }
- }
-
- @Override
- public DeletedBlocksTransaction copyObject(DeletedBlocksTransaction object) {
- return object;
- }
-}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/MoveDataNodePairCodec.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/MoveDataNodePairCodec.java
deleted file mode 100644
index a52639713a..0000000000
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/MoveDataNodePairCodec.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.scm.metadata;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.MoveDataNodePairProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers.MoveDataNodePair;
-import org.apache.hadoop.hdds.utils.db.Codec;
-import org.apache.hadoop.ozone.ClientVersion;
-
-import java.io.IOException;
-
-/**
- * Codec to serialize / deserialize MoveDataNodePair.
- */
-
-public class MoveDataNodePairCodec implements Codec<MoveDataNodePair> {
- @Override
- public byte[] toPersistedFormat(MoveDataNodePair mdnp)
- throws IOException {
- return mdnp
- .getProtobufMessage(ClientVersion.CURRENT_VERSION).toByteArray();
- }
-
- @Override
- public MoveDataNodePair fromPersistedFormat(byte[] rawData)
- throws IOException {
- MoveDataNodePairProto.Builder builder =
- MoveDataNodePairProto.newBuilder(
- MoveDataNodePairProto.PARSER.parseFrom(rawData));
- return MoveDataNodePair.getFromProtobuf(builder.build());
- }
-
- @Override
- public MoveDataNodePair copyObject(MoveDataNodePair object) {
- throw new UnsupportedOperationException();
- }
-}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java
deleted file mode 100644
index bc17b33c14..0000000000
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/PipelineCodec.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.scm.metadata;
-
-import java.io.IOException;
-import java.time.Instant;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.utils.db.Codec;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.ClientVersion;
-
-/**
- * Codec to serialize / deserialize Pipeline.
- */
-public class PipelineCodec implements Codec<Pipeline> {
-
- @Override
- public byte[] toPersistedFormat(Pipeline object) throws IOException {
- return object
- .getProtobufMessage(ClientVersion.CURRENT_VERSION).toByteArray();
- }
-
- @Override
- public Pipeline fromPersistedFormat(byte[] rawData) throws IOException {
- HddsProtos.Pipeline.Builder pipelineBuilder = HddsProtos.Pipeline
- .newBuilder(HddsProtos.Pipeline.PARSER.parseFrom(rawData));
- Pipeline pipeline = Pipeline.getFromProtobuf(pipelineBuilder.build());
- // When SCM is restarted, set Creation time with current time.
- pipeline.setCreationTimestamp(Instant.now());
- Preconditions.checkNotNull(pipeline);
- return pipeline;
- }
-
- @Override
- public Pipeline copyObject(Pipeline object) {
- throw new UnsupportedOperationException();
- }
-}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
index 3a067a4cb7..47a1b0259f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.utils.db.ByteStringCodec;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.DBDefinition;
import org.apache.hadoop.hdds.utils.db.LongCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
import org.apache.hadoop.hdds.utils.db.StringCodec;
/**
@@ -49,7 +50,7 @@ public class SCMDBDefinition implements DBDefinition {
Long.class,
LongCodec.get(),
DeletedBlocksTransaction.class,
- new DeletedBlocksTransactionCodec());
+ Proto2Codec.get(DeletedBlocksTransaction.class));
public static final DBColumnFamilyDefinition<BigInteger, X509Certificate>
VALID_CERTS =
@@ -98,7 +99,7 @@ public class SCMDBDefinition implements DBDefinition {
PipelineID.class,
PipelineID.getCodec(),
Pipeline.class,
- new PipelineCodec());
+ Pipeline.getCodec());
public static final DBColumnFamilyDefinition<ContainerID, ContainerInfo>
CONTAINERS =
@@ -107,7 +108,7 @@ public class SCMDBDefinition implements DBDefinition {
ContainerID.class,
ContainerID.getCodec(),
ContainerInfo.class,
- new ContainerInfoCodec());
+ ContainerInfo.getCodec());
public static final DBColumnFamilyDefinition<String, TransactionInfo>
TRANSACTIONINFO =
@@ -152,7 +153,7 @@ public class SCMDBDefinition implements DBDefinition {
ContainerID.class,
ContainerID.getCodec(),
MoveDataNodePair.class,
- new MoveDataNodePairCodec());
+ MoveDataNodePair.getCodec());
/**
* Stores miscellaneous SCM metadata, including upgrade finalization status
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java
index 8db9952781..53a1b7e291 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBAccessIdInfo.java
@@ -34,7 +34,7 @@ public final class OmDBAccessIdInfo {
Proto2Codec.get(ExtendedUserAccessIdInfo.class),
OmDBAccessIdInfo::getFromProtobuf,
OmDBAccessIdInfo::getProtobuf,
- true);
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<OmDBAccessIdInfo> getCodec() {
return CODEC;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java
index 95c9951d10..0c113aba85 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmDBTenantState.java
@@ -34,7 +34,7 @@ public final class OmDBTenantState implements
Comparable<OmDBTenantState> {
Proto2Codec.get(OzoneManagerProtocolProtos.TenantState.class),
OmDBTenantState::getFromProtobuf,
OmDBTenantState::getProtobuf,
- true);
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<OmDBTenantState> getCodec() {
return CODEC;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
index a6a4a64780..33d650a774 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
@@ -55,7 +55,10 @@ public final class SnapshotInfo implements Auditable {
Proto2Codec.get(OzoneManagerProtocolProtos.SnapshotInfo.class),
SnapshotInfo::getFromProtobuf,
SnapshotInfo::getProtobuf,
- true);
+ // FIXME: HDDS-8665 Deep copy will cause failures
+ // - TestOMSnapshotDeleteRequest NullPointerException
+ // - TestOMSnapshotPurgeRequestAndResponse AssertionFailedError
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<SnapshotInfo> getCodec() {
return CODEC;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
index 9b470e3bbc..03a1ed418e 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReportOzone.java
@@ -37,6 +37,8 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
/**
* Snapshot diff report.
+ * <p>
+ * This class is immutable.
*/
public class SnapshotDiffReportOzone
extends org.apache.hadoop.hdfs.protocol.SnapshotDiffReport {
@@ -45,7 +47,7 @@ public class SnapshotDiffReportOzone
Proto2Codec.get(DiffReportEntryProto.class),
SnapshotDiffReportOzone::fromProtobufDiffReportEntry,
SnapshotDiffReportOzone::toProtobufDiffReportEntry,
- true);
+ DelegatedCodec.CopyType.SHALLOW);
public static Codec<DiffReportEntry> getDiffReportEntryCodec() {
return CODEC;
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
index f66e3a7e1c..1ded306d2f 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
@@ -34,7 +34,8 @@ import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
*/
public class ReconSCMDBDefinition extends SCMDBDefinition {
private static final Codec<UUID> UUID_CODEC = new DelegatedCodec<>(
- StringCodec.get(), UUID::fromString, UUID::toString, true);
+ StringCodec.get(), UUID::fromString, UUID::toString,
+ DelegatedCodec.CopyType.SHALLOW);
public static final String RECON_SCM_DB_NAME = "recon-scm.db";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
