This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new daebde2 HDDS-3474. Create transactionInfo Table in OmMetadataManager.
(#855)
daebde2 is described below
commit daebde27f7e3a0958e01af2ca943f920bc834c61
Author: Bharat Viswanadham <[email protected]>
AuthorDate: Fri May 22 11:00:06 2020 -0700
HDDS-3474. Create transactionInfo Table in OmMetadataManager. (#855)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 3 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 21 +++-
.../ozone/om/codec/OMTransactionInfoCodec.java | 50 ++++++++
.../hadoop/ozone/om/ratis/OMTransactionInfo.java | 140 +++++++++++++++++++++
.../hadoop/ozone/om/TestOmMetadataManager.java | 25 ++++
.../ozone/om/codec/TestOMTransactionInfoCodec.java | 82 ++++++++++++
7 files changed, 324 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 8a57e65..b31f80d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -347,4 +347,8 @@ public final class OzoneConsts {
public static final String GDPR_ALGORITHM = "algorithm";
+ // Transaction Info
+ public static final String TRANSACTION_INFO_KEY = "#TRANSACTIONINFO";
+ public static final String TRANSACTION_INFO_SPLIT_KEY = "#";
+
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index e5432da..ce509ea 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.lock.OzoneManagerLock;
+import org.apache.hadoop.ozone.om.ratis.OMTransactionInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.UserVolumeInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -340,6 +341,8 @@ public interface OMMetadataManager {
*/
Table<String, S3SecretValue> getS3SecretTable();
+ Table<String, OMTransactionInfo> getTransactionInfoTable();
+
/**
* Returns number of rows in a table. This should not be used for very
* large tables.
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 54f1452..9bfbca0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.hdds.utils.db.cache.TableCacheImpl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
+import org.apache.hadoop.ozone.om.codec.OMTransactionInfoCodec;
import org.apache.hadoop.ozone.om.codec.OmBucketInfoCodec;
import org.apache.hadoop.ozone.om.codec.OmKeyInfoCodec;
import org.apache.hadoop.ozone.om.codec.OmMultipartKeyInfoCodec;
@@ -64,6 +65,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.lock.OzoneManagerLock;
+import org.apache.hadoop.ozone.om.ratis.OMTransactionInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.UserVolumeInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
@@ -118,6 +120,9 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
* |----------------------------------------------------------------------|
* | multipartInfoTable| /volumeName/bucketName/keyName/uploadId ->... |
* |----------------------------------------------------------------------|
+ * |----------------------------------------------------------------------|
+ * | transactionInfoTable | #TRANSACTIONINFO -> OMTransactionInfo |
+ * |----------------------------------------------------------------------|
*/
public static final String USER_TABLE = "userTable";
@@ -131,6 +136,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
public static final String S3_SECRET_TABLE = "s3SecretTable";
public static final String DELEGATION_TOKEN_TABLE = "dTokenTable";
public static final String PREFIX_TABLE = "prefixTable";
+ public static final String TRANSACTION_INFO_TABLE =
+ "transactionInfoTable";
private DBStore store;
@@ -148,6 +155,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
private Table s3SecretTable;
private Table dTokenTable;
private Table prefixTable;
+ private Table transactionInfoTable;
private boolean isRatisEnabled;
public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException {
@@ -282,6 +290,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
.addTable(DELEGATION_TOKEN_TABLE)
.addTable(S3_SECRET_TABLE)
.addTable(PREFIX_TABLE)
+ .addTable(TRANSACTION_INFO_TABLE)
.addCodec(OzoneTokenIdentifier.class, new TokenIdentifierCodec())
.addCodec(OmKeyInfo.class, new OmKeyInfoCodec())
.addCodec(RepeatedOmKeyInfo.class, new RepeatedOmKeyInfoCodec())
@@ -290,7 +299,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
.addCodec(UserVolumeInfo.class, new UserVolumeInfoCodec())
.addCodec(OmMultipartKeyInfo.class, new OmMultipartKeyInfoCodec())
.addCodec(S3SecretValue.class, new S3SecretValueCodec())
- .addCodec(OmPrefixInfo.class, new OmPrefixInfoCodec());
+ .addCodec(OmPrefixInfo.class, new OmPrefixInfoCodec())
+ .addCodec(OMTransactionInfo.class, new OMTransactionInfoCodec());
}
/**
@@ -346,6 +356,10 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
prefixTable = this.store.getTable(PREFIX_TABLE, String.class,
OmPrefixInfo.class);
checkTableStatus(prefixTable, PREFIX_TABLE);
+
+ transactionInfoTable = this.store.getTable(TRANSACTION_INFO_TABLE,
+ String.class, OMTransactionInfo.class);
+ checkTableStatus(transactionInfoTable, TRANSACTION_INFO_TABLE);
}
/**
@@ -1033,6 +1047,11 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
return s3SecretTable;
}
+ @Override
+ public Table<String, OMTransactionInfo> getTransactionInfoTable() {
+ return transactionInfoTable;
+ }
+
/**
* Update store used by subclass.
*
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMTransactionInfoCodec.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMTransactionInfoCodec.java
new file mode 100644
index 0000000..2cca220
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMTransactionInfoCodec.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om.codec;
+
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.ozone.om.ratis.OMTransactionInfo;
+
+import java.io.IOException;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Codec to convert {@link OMTransactionInfo} to byte array and from byte array
+ * to {@link OMTransactionInfo}.
+ */
+public class OMTransactionInfoCodec implements Codec<OMTransactionInfo> {
+ @Override
+ public byte[] toPersistedFormat(OMTransactionInfo object) throws IOException
{
+ checkNotNull(object, "Null object can't be converted to byte array.");
+ return object.convertToByteArray();
+ }
+
+ @Override
+ public OMTransactionInfo fromPersistedFormat(byte[] rawData)
+ throws IOException {
+ checkNotNull(rawData, "Null byte array can't be converted to " +
+ "real object.");
+ return OMTransactionInfo.getFromByteArray(rawData);
+ }
+
+ @Override
+ public OMTransactionInfo copyObject(OMTransactionInfo object) {
+ return object;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java
new file mode 100644
index 0000000..712659f
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OMTransactionInfo.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.ratis;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.StringUtils;
+
+import java.util.Objects;
+
+import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_SPLIT_KEY;
+
+/**
+ * TransactionInfo which is persisted to OM DB.
+ */
+public final class OMTransactionInfo {
+
+ private long currentTerm; // term associated with the ratis log index.
+ // Transaction index corresponds to ratis log index
+ private long transactionIndex;
+
+ private OMTransactionInfo(String transactionInfo) {
+ String[] tInfo =
+ transactionInfo.split(TRANSACTION_INFO_SPLIT_KEY);
+ Preconditions.checkState(tInfo.length==2,
+ "Incorrect TransactionInfo value");
+
+ currentTerm = Long.parseLong(tInfo[0]);
+ transactionIndex = Long.parseLong(tInfo[1]);
+ }
+
+ private OMTransactionInfo(long currentTerm, long transactionIndex) {
+ this.currentTerm = currentTerm;
+ this.transactionIndex = transactionIndex;
+ }
+
+ /**
+ * Get current term.
+ * @return currentTerm
+ */
+ public long getCurrentTerm() {
+ return currentTerm;
+ }
+
+ /**
+ * Get current transaction index.
+ * @return transactionIndex
+ */
+ public long getTransactionIndex() {
+ return transactionIndex;
+ }
+
+ /**
+ * Generate String form of transaction info which need to be persisted in OM
+ * DB finally in byte array.
+ * @return transaction info.
+ */
+ private String generateTransactionInfo() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(currentTerm);
+ stringBuilder.append(TRANSACTION_INFO_SPLIT_KEY);
+ stringBuilder.append(transactionIndex);
+
+ return stringBuilder.toString();
+ }
+
+ /**
+ * Convert OMTransactionInfo to byteArray to be persisted to OM DB.
+ * @return byte[]
+ */
+ public byte[] convertToByteArray() {
+ return StringUtils.string2Bytes(generateTransactionInfo());
+ }
+
+ /**
+ * Convert byte array persisted in DB to OMTransactionInfo.
+ * @param bytes
+ * @return OMTransactionInfo
+ */
+ public static OMTransactionInfo getFromByteArray(byte[] bytes) {
+ String tInfo = StringUtils.bytes2String(bytes);
+ return new OMTransactionInfo(tInfo);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ OMTransactionInfo that = (OMTransactionInfo) o;
+ return currentTerm == that.currentTerm &&
+ transactionIndex == that.transactionIndex;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(currentTerm, transactionIndex);
+ }
+
+ /**
+ * Builder to build {@link OMTransactionInfo}.
+ */
+ public static class Builder {
+ private long currentTerm = 0;
+ private long transactionIndex = -1;
+
+ public Builder setCurrentTerm(long term) {
+ this.currentTerm = term;
+ return this;
+ }
+
+ public Builder setTransactionIndex(long tIndex) {
+ this.transactionIndex = tIndex;
+ return this;
+ }
+
+ public OMTransactionInfo build() {
+ return new OMTransactionInfo(currentTerm, transactionIndex);
+ }
+
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 8975a24..054c97f 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.ratis.OMTransactionInfo;
import org.apache.hadoop.ozone.om.request.TestOMRequestUtils;
import org.junit.Assert;
import org.junit.Before;
@@ -35,6 +36,7 @@ import org.junit.rules.TemporaryFolder;
import java.util.List;
import java.util.TreeSet;
+import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
/**
@@ -58,6 +60,29 @@ public class TestOmMetadataManager {
}
@Test
+ public void testTransactionTable() throws Exception {
+ omMetadataManager.getTransactionInfoTable().put(TRANSACTION_INFO_KEY,
+ new OMTransactionInfo.Builder().setCurrentTerm(1)
+ .setTransactionIndex(100).build());
+
+ omMetadataManager.getTransactionInfoTable().put(TRANSACTION_INFO_KEY,
+ new OMTransactionInfo.Builder().setCurrentTerm(2)
+ .setTransactionIndex(200).build());
+
+ omMetadataManager.getTransactionInfoTable().put(TRANSACTION_INFO_KEY,
+ new OMTransactionInfo.Builder().setCurrentTerm(3)
+ .setTransactionIndex(250).build());
+
+ OMTransactionInfo omTransactionInfo =
+ omMetadataManager.getTransactionInfoTable().get(TRANSACTION_INFO_KEY);
+
+ Assert.assertEquals(3, omTransactionInfo.getCurrentTerm());
+ Assert.assertEquals(250, omTransactionInfo.getTransactionIndex());
+
+
+ }
+
+ @Test
public void testListVolumes() throws Exception {
String ownerName = "owner";
OmVolumeArgs.Builder argsBuilder = OmVolumeArgs.newBuilder()
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOMTransactionInfoCodec.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOMTransactionInfoCodec.java
new file mode 100644
index 0000000..a8ffd00
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/codec/TestOMTransactionInfoCodec.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om.codec;
+
+import org.apache.hadoop.ozone.om.ratis.OMTransactionInfo;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Class to test {@link OMTransactionInfoCodec}.
+ */
+public class TestOMTransactionInfoCodec {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+
+ private OMTransactionInfoCodec codec;
+
+ @Before
+ public void setUp() {
+ codec = new OMTransactionInfoCodec();
+ }
+ @Test
+ public void toAndFromPersistedFormat() throws Exception {
+ OMTransactionInfo omTransactionInfo =
+ new OMTransactionInfo.Builder().setTransactionIndex(100)
+ .setCurrentTerm(11).build();
+
+ OMTransactionInfo convertedTransactionInfo =
+ codec.fromPersistedFormat(codec.toPersistedFormat(omTransactionInfo));
+
+ Assert.assertEquals(omTransactionInfo, convertedTransactionInfo);
+
+ }
+ @Test
+ public void testCodecWithNullDataFromTable() throws Exception {
+ thrown.expect(NullPointerException.class);
+ codec.fromPersistedFormat(null);
+ }
+
+
+ @Test
+ public void testCodecWithNullDataFromUser() throws Exception {
+ thrown.expect(NullPointerException.class);
+ codec.toPersistedFormat(null);
+ }
+
+
+ @Test
+ public void testCodecWithIncorrectValues() throws Exception {
+ try {
+ codec.fromPersistedFormat("random".getBytes(StandardCharsets.UTF_8));
+ fail("testCodecWithIncorrectValues failed");
+ } catch (IllegalStateException ex) {
+ GenericTestUtils.assertExceptionContains("Incorrect TransactionInfo " +
+ "value", ex);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
