This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 00acf36d32 HDDS-8707. Avoid linear search in DBDefinition
implementations. (#4782)
00acf36d32 is described below
commit 00acf36d32ea8f452437d2333034ed15d20813ea
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Fri Jun 2 12:59:30 2023 +0800
HDDS-8707. Avoid linear search in DBDefinition implementations. (#4782)
Co-authored-by: Doroszlai, Attila <[email protected]>
---
.../hdds/datanode/metadata/CRLDBDefinition.java | 17 ++--
.../metadata/AbstractDatanodeDBDefinition.java | 11 +--
.../metadata/DatanodeSchemaOneDBDefinition.java | 20 ++++-
.../metadata/DatanodeSchemaThreeDBDefinition.java | 19 +++--
.../metadata/DatanodeSchemaTwoDBDefinition.java | 21 +++--
.../metadata/DatanodeStoreSchemaThreeImpl.java | 2 +-
.../apache/hadoop/hdds/utils/CollectionUtils.java | 92 ++++++++++++++++++++++
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 4 +-
.../hdds/utils/db/DBColumnFamilyDefinition.java | 26 +++++-
.../apache/hadoop/hdds/utils/db/DBDefinition.java | 77 +++++++++++++-----
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 6 --
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 5 --
.../hadoop/hdds/utils/TestCollectionUtils.java | 88 +++++++++++++++++++++
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 32 +++-----
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 4 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcService.java | 12 ++-
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 35 +++++---
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 14 ++--
.../hadoop/ozone/om/codec/OMDBDefinition.java | 44 ++++++++---
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 12 +--
.../apache/hadoop/ozone/om/TestOMDBDefinition.java | 6 +-
.../ozone/recon/scm/ReconSCMDBDefinition.java | 15 ++--
.../ozone/recon/spi/impl/ReconDBDefinition.java | 23 +++---
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 15 ++--
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 31 +++++---
.../org/apache/hadoop/ozone/debug/DBScanner.java | 16 +---
26 files changed, 465 insertions(+), 182 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java
index bdd53a3341..970f682d47 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.ozone.OzoneConsts;
import java.io.File;
+import java.util.Map;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
@@ -37,7 +38,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
/**
* Class defines the structure and types of the crl.db.
*/
-public class CRLDBDefinition implements DBDefinition {
+public class CRLDBDefinition extends DBDefinition.WithMap {
public static final DBColumnFamilyDefinition<Long, CRLInfo> PENDING_CRLS =
new DBColumnFamilyDefinition<>(
@@ -56,6 +57,15 @@ public class CRLDBDefinition implements DBDefinition {
Long.class,
LongCodec.get());
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ PENDING_CRLS, CRL_SEQUENCE_ID);
+
+ public CRLDBDefinition() {
+ // TODO: change it to singleton
+ super(COLUMN_FAMILIES);
+ }
+
@Override
public String getName() {
return OzoneConsts.DN_CRL_DB;
@@ -87,9 +97,4 @@ public class CRLDBDefinition implements DBDefinition {
+ File.separator
+ OzoneConsts.CRL_DB_DIRECTORY_NAME);
}
-
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {PENDING_CRLS, CRL_SEQUENCE_ID};
- }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeDBDefinition.java
index 49bcd8edd2..2c1c3c214d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/AbstractDatanodeDBDefinition.java
@@ -31,9 +31,8 @@ import java.io.File;
*/
public abstract class AbstractDatanodeDBDefinition implements DBDefinition {
- private File dbDir;
-
- private ConfigurationSource config;
+ private final File dbDir;
+ private final ConfigurationSource config;
/**
* @param dbPath The absolute path to the .db file corresponding to this
@@ -66,12 +65,6 @@ public abstract class AbstractDatanodeDBDefinition
implements DBDefinition {
return config;
}
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {getBlockDataColumnFamily(),
- getMetadataColumnFamily(), getDeletedBlocksColumnFamily()};
- }
-
public abstract DBColumnFamilyDefinition<String, BlockData>
getBlockDataColumnFamily();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java
index 62c617eb04..a002eef3f7 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaOneDBDefinition.java
@@ -21,9 +21,13 @@ import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.LongCodec;
+import org.apache.hadoop.hdds.utils.CollectionUtils;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
+import java.util.List;
+import java.util.Map;
+
import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY;
/**
@@ -70,6 +74,12 @@ public class DatanodeSchemaOneDBDefinition
ChunkInfoList.class,
SchemaOneChunkInfoListCodec.get());
+ private static final Map<String, List<DBColumnFamilyDefinition<?, ?>>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMultiMap(
+ BLOCK_DATA,
+ METADATA,
+ DELETED_BLOCKS);
+
public DatanodeSchemaOneDBDefinition(String dbPath,
ConfigurationSource config) {
super(dbPath, config);
@@ -93,8 +103,12 @@ public class DatanodeSchemaOneDBDefinition
}
@Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {getMetadataColumnFamily(),
- getDeletedBlocksColumnFamily(), getBlockDataColumnFamily()};
+ public List<DBColumnFamilyDefinition<?, ?>> getColumnFamilies(String name) {
+ return COLUMN_FAMILIES.get(name);
+ }
+
+ @Override
+ public Iterable<DBColumnFamilyDefinition<?, ?>> getColumnFamilies() {
+ return () -> CollectionUtils.newIterator(COLUMN_FAMILIES.values());
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
index 983dfb2c28..51b5e6c271 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaThreeDBDefinition.java
@@ -21,6 +21,7 @@ import com.google.common.primitives.Longs;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
import org.apache.hadoop.hdds.utils.db.FixedLengthStringUtils;
import org.apache.hadoop.hdds.utils.db.LongCodec;
import org.apache.hadoop.hdds.utils.db.FixedLengthStringCodec;
@@ -31,6 +32,8 @@ import
org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.common.utils.db.DatanodeDBProfile;
+import java.util.Map;
+
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
import static
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.HDDS_DEFAULT_DB_PROFILE;
@@ -52,7 +55,8 @@ import static
org.apache.hadoop.hdds.utils.db.DBStoreBuilder.HDDS_DEFAULT_DB_PRO
* utilize the "Prefix Seek" feature from Rocksdb to optimize seek.
*/
public class DatanodeSchemaThreeDBDefinition
- extends AbstractDatanodeDBDefinition {
+ extends AbstractDatanodeDBDefinition
+ implements DBDefinition.WithMapInterface {
public static final DBColumnFamilyDefinition<String, BlockData>
BLOCK_DATA =
new DBColumnFamilyDefinition<>(
@@ -91,6 +95,13 @@ public class DatanodeSchemaThreeDBDefinition
private static String separator = "";
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ BLOCK_DATA,
+ METADATA,
+ DELETED_BLOCKS,
+ DELETE_TRANSACTION);
+
public DatanodeSchemaThreeDBDefinition(String dbPath,
ConfigurationSource config) {
super(dbPath, config);
@@ -115,10 +126,8 @@ public class DatanodeSchemaThreeDBDefinition
}
@Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {getBlockDataColumnFamily(),
- getMetadataColumnFamily(), getDeletedBlocksColumnFamily(),
- getDeleteTransactionsColumnFamily()};
+ public Map<String, DBColumnFamilyDefinition<?, ?>> getMap() {
+ return COLUMN_FAMILIES;
}
@Override
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
index bf47207f13..50e181147e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeSchemaTwoDBDefinition.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.metadata;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
import org.apache.hadoop.hdds.utils.db.LongCodec;
import org.apache.hadoop.hdds.utils.db.Proto2Codec;
import org.apache.hadoop.hdds.utils.db.StringCodec;
@@ -28,13 +29,16 @@ import
org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
+import java.util.Map;
+
/**
* This class defines the RocksDB structure for datanodes following schema
* version 2, where the block data, metadata, and transactions which are to be
* deleted are put in their own separate column families.
*/
-public class DatanodeSchemaTwoDBDefinition extends
- AbstractDatanodeDBDefinition {
+public class DatanodeSchemaTwoDBDefinition
+ extends AbstractDatanodeDBDefinition
+ implements DBDefinition.WithMapInterface {
public static final DBColumnFamilyDefinition<String, BlockData>
BLOCK_DATA =
@@ -77,11 +81,16 @@ public class DatanodeSchemaTwoDBDefinition extends
super(dbPath, config);
}
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ BLOCK_DATA,
+ METADATA,
+ DELETED_BLOCKS,
+ DELETE_TRANSACTION);
+
@Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {getBlockDataColumnFamily(),
- getMetadataColumnFamily(), getDeletedBlocksColumnFamily(),
- getDeleteTransactionsColumnFamily()};
+ public Map<String, DBColumnFamilyDefinition<?, ?>> getMap() {
+ return COLUMN_FAMILIES;
}
@Override
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaThreeImpl.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaThreeImpl.java
index a89e7a8e45..43739d2a33 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaThreeImpl.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/metadata/DatanodeStoreSchemaThreeImpl.java
@@ -211,7 +211,7 @@ public class DatanodeStoreSchemaThreeImpl extends
AbstractDatanodeStore
options);
} else {
LOG.warn("Failed to find cf {} in DB {}", entry.getKey(),
- getDbDef().getDBLocation(null));
+ getDbDef().getClass());
}
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/CollectionUtils.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/CollectionUtils.java
new file mode 100644
index 0000000000..5d46bcc957
--- /dev/null
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/CollectionUtils.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.utils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/** Utility methods for Java Collections. */
+public interface CollectionUtils {
+ static <KEY, VALUE> Map<KEY, VALUE> newUnmodifiableMap(
+ List<VALUE> values, Function<VALUE, KEY> getKey,
+ Map<KEY, VALUE> existing) {
+ final Map<KEY, VALUE> map = new HashMap<>(existing);
+ for (VALUE v : values) {
+ final KEY key = getKey.apply(v);
+ final VALUE previous = map.put(key, v);
+ if (previous != null) {
+ throw new IllegalArgumentException("Already exists: " + key
+ + ", previous " + previous.getClass());
+ }
+ }
+ return Collections.unmodifiableMap(map);
+ }
+
+ static <KEY, VALUE> Map<KEY, List<VALUE>> newUnmodifiableMultiMap(
+ List<VALUE> values, Function<VALUE, KEY> getKey) {
+ final Map<KEY, List<VALUE>> map = new HashMap<>();
+ for (VALUE v : values) {
+ final KEY key = getKey.apply(v);
+ map.computeIfAbsent(key, k -> new ArrayList<>()).add(v);
+ }
+ return Collections.unmodifiableMap(map.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey,
+ e -> Collections.unmodifiableList(e.getValue()))));
+ }
+
+ static <T> Iterator<T> newIterator(Collection<List<T>> values) {
+ final Iterator<List<T>> listIterator = values.iterator();
+ return new Iterator<T>() {
+ private Iterator<T> i = Collections.emptyIterator();
+
+ private Iterator<T> nextIterator() {
+ if (i.hasNext()) {
+ return i;
+ }
+ while (listIterator.hasNext()) {
+ i = listIterator.next().iterator();
+ if (i.hasNext()) {
+ return i;
+ }
+ }
+ return Collections.emptyIterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return nextIterator().hasNext();
+ }
+
+ @Override
+ public T next() {
+ if (hasNext()) {
+ return i.next();
+ }
+ throw new NoSuchElementException();
+ }
+ };
+ }
+}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
index 4009cbec8c..1989f25802 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.hdds.utils;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Streams;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
@@ -284,7 +285,8 @@ public final class HAUtils {
public static Table<String, TransactionInfo> getTransactionInfoTable(
DBStore dbStore, DBDefinition definition) throws IOException {
- return Arrays.stream(definition.getColumnFamilies())
+ return (Table<String, TransactionInfo>)
+ Streams.stream(definition.getColumnFamilies())
.filter(t -> t.getValueType() == TransactionInfo.class).findFirst()
.get().getTable(dbStore);
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
index d1d1ac10a4..653182214b 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
@@ -18,9 +18,14 @@
*/
package org.apache.hadoop.hdds.utils.db;
+import org.apache.hadoop.hdds.utils.CollectionUtils;
import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
/**
* Class represents one single column table with the required codecs and types.
@@ -29,6 +34,23 @@ import java.io.IOException;
* @param <VALUE> they type of the value.
*/
public class DBColumnFamilyDefinition<KEY, VALUE> {
+ public static Map<String, DBColumnFamilyDefinition<?, ?>> newUnmodifiableMap(
+ DBColumnFamilyDefinition<?, ?>... families) {
+ return newUnmodifiableMap(Collections.emptyMap(), families);
+ }
+
+ public static Map<String, DBColumnFamilyDefinition<?, ?>> newUnmodifiableMap(
+ Map<String, DBColumnFamilyDefinition<?, ?>> existing,
+ DBColumnFamilyDefinition<?, ?>... families) {
+ return CollectionUtils.newUnmodifiableMap(Arrays.asList(families),
+ DBColumnFamilyDefinition::getName, existing);
+ }
+
+ public static Map<String, List<DBColumnFamilyDefinition<?, ?>>>
+ newUnmodifiableMultiMap(DBColumnFamilyDefinition<?, ?>... families) {
+ return CollectionUtils.newUnmodifiableMultiMap(Arrays.asList(families),
+ DBColumnFamilyDefinition::getName);
+ }
private final String tableName;
@@ -64,10 +86,6 @@ public class DBColumnFamilyDefinition<KEY, VALUE> {
return tableName;
}
- public String getTableName() {
- return tableName;
- }
-
public Class<KEY> getKeyType() {
return keyType;
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
index 6485beb328..968d62f0dd 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
@@ -24,9 +24,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.Arrays;
-import java.util.Optional;
-import java.util.function.Function;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
/**
* Simple interface to provide information to create a DBStore..
@@ -57,29 +58,65 @@ public interface DBDefinition {
/**
* @return The column families present in the DB.
*/
- DBColumnFamilyDefinition[] getColumnFamilies();
+ Iterable<DBColumnFamilyDefinition<?, ?>> getColumnFamilies();
/**
- * Get the key type class for the given table.
- * @param table table name
- * @return the class of key type of the given table wrapped in an
- * {@link Optional}
+ * @return The column families for the given name.
*/
- default Optional<Class> getKeyType(String table) {
- return Arrays.stream(getColumnFamilies()).filter(cf -> cf.getName().equals(
- table)).map((Function<DBColumnFamilyDefinition, Class>)
- DBColumnFamilyDefinition::getKeyType).findAny();
+ List<DBColumnFamilyDefinition<?, ?>> getColumnFamilies(String name);
+
+
+ /**
+ * @return The unique column family for the given name.
+ */
+ default DBColumnFamilyDefinition<?, ?> getColumnFamily(String name) {
+ final List<DBColumnFamilyDefinition<?, ?>> list = getColumnFamilies(name);
+ if (list == null || list.isEmpty()) {
+ return null;
+ }
+ if (list.size() > 1) {
+ throw new IllegalStateException("Multi-valued: The name " + name
+ + " maps to multiple values " + list + " in " + getName());
+ }
+ return list.get(0);
+ }
+
+ /**
+ * Define a {@link WithMapInterface#getMap()} method
+ * to implement {@link #getColumnFamily(String)}
+ * and {@link #getColumnFamilies()}.
+ */
+ interface WithMapInterface extends DBDefinition {
+ /** @return the underlying map. */
+ Map<String, DBColumnFamilyDefinition<?, ?>> getMap();
+
+ @Override
+ default Collection<DBColumnFamilyDefinition<?, ?>> getColumnFamilies() {
+ return getMap().values();
+ }
+
+ @Override
+ default List<DBColumnFamilyDefinition<?, ?>> getColumnFamilies(
+ String name) {
+ final DBColumnFamilyDefinition<?, ?> d = getMap().get(name);
+ return d != null ? Collections.singletonList(d) :
Collections.emptyList();
+ }
}
/**
- * Get the value type class for the given table.
- * @param table table name
- * @return the class of value type of the given table wrapped in an
- * {@link Optional}
+ * Provide constructors to initialize {@link #map}
+ * and use it to implement {@link #getMap()}.
*/
- default Optional<Class> getValueType(String table) {
- return Arrays.stream(getColumnFamilies()).filter(cf -> cf.getName().equals(
- table)).map((Function<DBColumnFamilyDefinition, Class>)
- DBColumnFamilyDefinition::getValueType).findAny();
+ abstract class WithMap implements WithMapInterface {
+ private final Map<String, DBColumnFamilyDefinition<?, ?>> map;
+
+ protected WithMap(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
+ this.map = map;
+ }
+
+ @Override
+ public final Map<String, DBColumnFamilyDefinition<?, ?>> getMap() {
+ return map;
+ }
}
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
index f44bc82897..f4b740a255 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java
@@ -180,12 +180,6 @@ public interface DBStore extends Closeable,
BatchOperationHandler {
*/
Map<Integer, String> getTableNames();
- /**
- * Get Codec registry.
- * @return codec registry.
- */
- CodecRegistry getCodecRegistry();
-
/**
* Get data written to DB since a specific sequence number.
* @param sequenceNumber
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
index 220c0b797a..d20fa13dda 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java
@@ -345,11 +345,6 @@ public class RDBStore implements DBStore {
return db.getExtraColumnFamilies();
}
- @Override
- public CodecRegistry getCodecRegistry() {
- return codecRegistry;
- }
-
@Override
public DBUpdatesWrapper getUpdatesSince(long sequenceNumber)
throws SequenceNumberNotFoundException {
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestCollectionUtils.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestCollectionUtils.java
new file mode 100644
index 0000000000..2fd773d878
--- /dev/null
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestCollectionUtils.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.utils;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+/** Test for {@link CollectionUtils}. */
+public class TestCollectionUtils {
+ @Test
+ void singleNonEmptyLists() {
+ List<List<String>> listOfLists = new ArrayList<>();
+ listOfLists.add(asList("a", "b"));
+ assertIteration(asList("a", "b"), listOfLists);
+ }
+
+ @Test
+ void allNonEmptyLists() {
+ List<List<String>> listOfLists = new ArrayList<>();
+ listOfLists.add(asList("a", "b"));
+ listOfLists.add(singletonList("c"));
+ listOfLists.add(singletonList("d"));
+ assertIteration(asList("a", "b", "c", "d"), listOfLists);
+ }
+
+ @Test
+ void singleEmptyLists() {
+ List<List<String>> listOfLists = new ArrayList<>();
+ listOfLists.add(emptyList());
+ assertIteration(emptyList(), listOfLists);
+ }
+
+ @Test
+ void someEmptyLists() {
+ List<List<String>> listOfLists = new ArrayList<>();
+ listOfLists.add(asList("a", "b"));
+ listOfLists.add(emptyList());
+ listOfLists.add(emptyList());
+ listOfLists.add(singletonList("c"));
+ listOfLists.add(singletonList("d"));
+ listOfLists.add(emptyList());
+
+ assertIteration(asList("a", "b", "c", "d"), listOfLists);
+ }
+
+ @Test
+ void allEmptyLists() {
+ List<List<String>> listOfLists = new ArrayList<>();
+ listOfLists.add(emptyList());
+ listOfLists.add(emptyList());
+
+ assertIteration(emptyList(), listOfLists);
+ }
+
+ @Test
+ void empty() {
+ assertIteration(emptyList(), emptyList());
+ }
+
+ private static <T> void assertIteration(List<T> expected,
+ List<List<T>> listOfLists) {
+ List<T> actual = new ArrayList<>();
+ CollectionUtils.newIterator(listOfLists).forEachRemaining(actual::add);
+ Assertions.assertEquals(expected, actual);
+ }
+}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
index 66efc36aa2..303dafde55 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
@@ -172,11 +172,11 @@ public class TestDBStoreBuilder {
}
String sampleTableName = "sampleTable";
- final DBDefinition sampleDB = new DBDefinition() {
-
- private final DBColumnFamilyDefinition<String, Long> sampleTable =
- new DBColumnFamilyDefinition<>(sampleTableName,
- String.class, StringCodec.get(), Long.class, LongCodec.get());
+ final DBColumnFamilyDefinition<String, Long> sampleTable =
+ new DBColumnFamilyDefinition<>(sampleTableName,
+ String.class, StringCodec.get(), Long.class, LongCodec.get());
+ final DBDefinition sampleDB = new DBDefinition.WithMap(
+ DBColumnFamilyDefinition.newUnmodifiableMap(sampleTable)) {
{
ManagedColumnFamilyOptions cfOptions = new
ManagedColumnFamilyOptions();
// reverse the default option for check
@@ -195,11 +195,6 @@ public class TestDBStoreBuilder {
return null;
}
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[]{sampleTable};
- }
-
@Override
public File getDBLocation(ConfigurationSource conf) {
return null;
@@ -249,13 +244,11 @@ public class TestDBStoreBuilder {
}
String sampleTableName = "sampleTable";
- final DBDefinition sampleDB = new DBDefinition() {
-
- private final DBColumnFamilyDefinition<String, Long> sampleTable =
- new DBColumnFamilyDefinition<>(sampleTableName, String.class,
- StringCodec.get(), Long.class, LongCodec.get());
-
-
+ final DBColumnFamilyDefinition<String, Long> sampleTable =
+ new DBColumnFamilyDefinition<>(sampleTableName, String.class,
+ StringCodec.get(), Long.class, LongCodec.get());
+ final DBDefinition sampleDB = new DBDefinition.WithMap(
+ DBColumnFamilyDefinition.newUnmodifiableMap(sampleTable)) {
@Override
public String getName() {
return "sampleDB";
@@ -266,11 +259,6 @@ public class TestDBStoreBuilder {
return null;
}
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[]{sampleTable};
- }
-
@Override
public File getDBLocation(ConfigurationSource conf) {
return null;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
index 8f0d281d20..7487c24378 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcProtocolService.java
@@ -48,8 +48,8 @@ public class InterSCMGrpcProtocolService {
private Server server;
private final AtomicBoolean isStarted = new AtomicBoolean(false);
- public InterSCMGrpcProtocolService(final ConfigurationSource conf,
- final StorageContainerManager scm) {
+ InterSCMGrpcProtocolService(final ConfigurationSource conf,
+ final StorageContainerManager scm) throws IOException {
Preconditions.checkNotNull(conf);
this.port = conf.getInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY,
ScmConfigKeys.OZONE_SCM_GRPC_PORT_DEFAULT);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
index e962ded0ed..bd4b56cd8c 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/InterSCMGrpcService.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.scm.ha;
import java.io.IOException;
-import java.util.Arrays;
import com.google.common.base.Preconditions;
@@ -27,6 +26,7 @@ import
org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolProtos.CopyDBCh
import org.apache.hadoop.hdds.protocol.scm.proto.InterSCMProtocolServiceGrpc;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.utils.HAUtils;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -49,10 +49,13 @@ public class InterSCMGrpcService extends
private final SCMDBCheckpointProvider provider;
private final StorageContainerManager scm;
+ private final Table<String, TransactionInfo> transactionInfoTable;
- public InterSCMGrpcService(final StorageContainerManager scm) {
+ InterSCMGrpcService(final StorageContainerManager scm) throws IOException {
Preconditions.checkNotNull(scm);
this.scm = scm;
+ this.transactionInfoTable = HAUtils.getTransactionInfoTable(
+ scm.getScmMetadataStore().getStore(), new SCMDBDefinition());
provider =
new SCMDBCheckpointProvider(scm.getScmMetadataStore().getStore());
}
@@ -62,11 +65,6 @@ public class InterSCMGrpcService extends
StreamObserver<CopyDBCheckpointResponseProto> responseObserver) {
try {
scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
- Table<String, TransactionInfo> transactionInfoTable =
- Arrays.stream(new SCMDBDefinition().getColumnFamilies())
- .filter(t -> t.getValueType() == TransactionInfo.class)
-
.findFirst().get().getTable(scm.getScmMetadataStore().getStore());
-
TransactionInfo transactionInfo =
transactionInfoTable.get(TRANSACTION_INFO_KEY);
Preconditions.checkNotNull(transactionInfo);
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
index 1caf8021e5..5217ddabb3 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/SCMDBDefinition.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.metadata;
import java.math.BigInteger;
import java.security.cert.X509Certificate;
+import java.util.Map;
import com.google.protobuf.ByteString;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
@@ -41,7 +42,14 @@ import org.apache.hadoop.hdds.utils.db.StringCodec;
/**
* Class defines the structure and types of the scm.db.
*/
-public class SCMDBDefinition implements DBDefinition {
+public class SCMDBDefinition extends DBDefinition.WithMap {
+ public SCMDBDefinition() {
+ this(COLUMN_FAMILIES);
+ }
+
+ protected SCMDBDefinition(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
+ super(map);
+ }
public static final DBColumnFamilyDefinition<Long, DeletedBlocksTransaction>
DELETED_BLOCKS =
@@ -176,6 +184,23 @@ public class SCMDBDefinition implements DBDefinition {
ByteString.class,
ByteStringCodec.getInstance());
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ CONTAINERS,
+ CRLS,
+ CRL_SEQUENCE_ID,
+ DELETED_BLOCKS,
+ META,
+ MOVE,
+ PIPELINES,
+ REVOKED_CERTS,
+ REVOKED_CERTS_V2,
+ SEQUENCE_ID,
+ STATEFUL_SERVICE_CONFIG,
+ TRANSACTIONINFO,
+ VALID_CERTS,
+ VALID_SCM_CERTS);
+
@Override
public String getName() {
return "scm.db";
@@ -185,12 +210,4 @@ public class SCMDBDefinition implements DBDefinition {
public String getLocationConfigKey() {
return ScmConfigKeys.OZONE_SCM_DB_DIRS;
}
-
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {DELETED_BLOCKS, VALID_CERTS,
- VALID_SCM_CERTS, REVOKED_CERTS, REVOKED_CERTS_V2, PIPELINES,
CONTAINERS,
- TRANSACTIONINFO, CRLS, CRL_SEQUENCE_ID, SEQUENCE_ID, MOVE, META,
- STATEFUL_SERVICE_CONFIG};
- }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 9c5ad7081a..fc215d2fb3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -53,8 +53,7 @@ import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo;
import org.apache.hadoop.hdds.utils.BackgroundService;
-import org.apache.hadoop.hdds.utils.db.CodecRegistry;
-import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -1699,16 +1698,15 @@ public class KeyManagerImpl implements KeyManager {
}
private String getNextGreaterString(String volumeName, String bucketName,
- String keyPrefix) throws IOException {
+ String keyPrefix) {
// Increment the last character of the string and return the new ozone key.
Preconditions.checkArgument(!Strings.isNullOrEmpty(keyPrefix),
"Key prefix is null or empty");
- CodecRegistry codecRegistry =
- ((RDBStore) metadataManager.getStore()).getCodecRegistry();
- byte[] keyPrefixInBytes = codecRegistry.asRawData(keyPrefix);
+ final StringCodec codec = StringCodec.get();
+ final byte[] keyPrefixInBytes = codec.toPersistedFormat(keyPrefix);
keyPrefixInBytes[keyPrefixInBytes.length - 1]++;
- String nextPrefix = codecRegistry.asObject(keyPrefixInBytes, String.class);
- return metadataManager.getOzoneKey(volumeName, bucketName, nextPrefix);
+ return metadataManager.getOzoneKey(volumeName, bucketName,
+ codec.fromPersistedFormat(keyPrefixInBytes));
}
private FileEncryptionInfo getFileEncryptionInfo(OmBucketInfo bucketInfo)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index 2acb964b55..0d17faf1fb 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -44,10 +44,12 @@ import
org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import
org.apache.hadoop.ozone.storage.proto.OzoneManagerStorageProtos.PersistedUserVolumeInfo;
+import java.util.Map;
+
/**
* Class defines the structure and types of the om.db.
*/
-public class OMDBDefinition implements DBDefinition {
+public class OMDBDefinition extends DBDefinition.WithMap {
public static final DBColumnFamilyDefinition<String, RepeatedOmKeyInfo>
DELETED_TABLE =
@@ -248,6 +250,34 @@ public class OMDBDefinition implements DBDefinition {
String.class, // path to key in prev snapshot's key(file)/dir Table.
StringCodec.get());
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ BUCKET_TABLE,
+ DELETED_DIR_TABLE,
+ DELETED_TABLE,
+ DIRECTORY_TABLE,
+ DTOKEN_TABLE,
+ FILE_TABLE,
+ KEY_TABLE,
+ META_TABLE,
+ MULTIPART_INFO_TABLE,
+ OPEN_FILE_TABLE,
+ OPEN_KEY_TABLE,
+ PREFIX_TABLE,
+ PRINCIPAL_TO_ACCESS_IDS_TABLE,
+ S3_SECRET_TABLE,
+ SNAPSHOT_INFO_TABLE,
+ SNAPSHOT_RENAMED_TABLE,
+ TENANT_ACCESS_ID_TABLE,
+ TENANT_STATE_TABLE,
+ TRANSACTION_INFO_TABLE,
+ USER_TABLE,
+ VOLUME_TABLE);
+
+ public OMDBDefinition() {
+ super(COLUMN_FAMILIES);
+ }
+
@Override
public String getName() {
return OzoneConsts.OM_DB_NAME;
@@ -257,17 +287,5 @@ public class OMDBDefinition implements DBDefinition {
public String getLocationConfigKey() {
return OMConfigKeys.OZONE_OM_DB_DIRS;
}
-
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {DELETED_TABLE, USER_TABLE,
- VOLUME_TABLE, OPEN_KEY_TABLE, KEY_TABLE,
- BUCKET_TABLE, MULTIPART_INFO_TABLE, PREFIX_TABLE, DTOKEN_TABLE,
- S3_SECRET_TABLE, TRANSACTION_INFO_TABLE, DIRECTORY_TABLE,
- FILE_TABLE, OPEN_FILE_TABLE, DELETED_DIR_TABLE, META_TABLE,
- TENANT_ACCESS_ID_TABLE,
- PRINCIPAL_TO_ACCESS_IDS_TABLE, TENANT_STATE_TABLE,
- SNAPSHOT_INFO_TABLE, SNAPSHOT_RENAMED_TABLE};
- }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
index e2b47292bc..e2ecf4081b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
@@ -479,14 +479,14 @@ public final class OzoneManagerDoubleBuffer {
CleanupTableInfo cleanupTableInfo =
responseClass.getAnnotation(CleanupTableInfo.class);
if (cleanupTableInfo != null) {
- String[] cleanupTables;
+ final List<String> cleanupTables;
if (cleanupTableInfo.cleanupAll()) {
- cleanupTables = Arrays
- .stream(new OMDBDefinition().getColumnFamilies())
- .map(DBColumnFamilyDefinition::getTableName)
- .toArray(String[]::new);
+ cleanupTables = new OMDBDefinition().getColumnFamilies()
+ .stream()
+ .map(DBColumnFamilyDefinition::getName)
+ .collect(Collectors.toList());
} else {
- cleanupTables = cleanupTableInfo.cleanupTables();
+ cleanupTables = Arrays.asList(cleanupTableInfo.cleanupTables());
}
for (String table : cleanupTables) {
cleanupEpochs.computeIfAbsent(table, list -> new ArrayList<>())
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
index 277969c5e9..66045a0d4f 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
@@ -48,9 +48,9 @@ public class TestOMDBDefinition {
OMDBDefinition dbDef = new OMDBDefinition();
// Get list of tables from DB Definitions
- DBColumnFamilyDefinition[] columnFamilyDefinitions =
- dbDef.getColumnFamilies();
- int countOmDefTables = columnFamilyDefinitions.length;
+ final Collection<DBColumnFamilyDefinition<?, ?>> columnFamilyDefinitions
+ = dbDef.getColumnFamilies();
+ final int countOmDefTables = columnFamilyDefinitions.size();
ArrayList<String> missingDBDefTables = new ArrayList<>();
// Get list of tables from the RocksDB Store
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
index 1ded306d2f..1ea2f7b131 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
@@ -18,9 +18,9 @@
*/
package org.apache.hadoop.ozone.recon.scm;
+import java.util.Map;
import java.util.UUID;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.db.Codec;
@@ -48,6 +48,14 @@ public class ReconSCMDBDefinition extends SCMDBDefinition {
DatanodeDetails.class,
DatanodeDetails.getCodec());
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ new SCMDBDefinition().getMap(), NODES);
+
+ public ReconSCMDBDefinition() {
+ super(COLUMN_FAMILIES);
+ }
+
@Override
public String getName() {
return RECON_SCM_DB_NAME;
@@ -57,9 +65,4 @@ public class ReconSCMDBDefinition extends SCMDBDefinition {
public String getLocationConfigKey() {
return ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
}
-
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return ArrayUtils.add(super.getColumnFamilies(), NODES);
- }
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java
index 59e51d6204..8cb3b4188e 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBDefinition.java
@@ -28,14 +28,17 @@ import org.apache.hadoop.ozone.recon.codec.NSSummaryCodec;
import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistoryList;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
+import java.util.Map;
+
/**
* RocksDB definition for the DB internal to Recon.
*/
-public class ReconDBDefinition implements DBDefinition {
+public class ReconDBDefinition extends DBDefinition.WithMap {
- private String dbName;
+ private final String dbName;
public ReconDBDefinition(String dbName) {
+ super(COLUMN_FAMILIES);
this.dbName = dbName;
}
@@ -93,6 +96,15 @@ public class ReconDBDefinition implements DBDefinition {
ContainerReplicaHistoryList.class,
ContainerReplicaHistoryList.getCodec());
+ private static final Map<String, DBColumnFamilyDefinition<?, ?>>
+ COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
+ CONTAINER_KEY,
+ CONTAINER_KEY_COUNT,
+ KEY_CONTAINER,
+ NAMESPACE_SUMMARY,
+ REPLICA_HISTORY,
+ REPLICA_HISTORY_V2);
+
@Override
public String getName() {
return dbName;
@@ -102,11 +114,4 @@ public class ReconDBDefinition implements DBDefinition {
public String getLocationConfigKey() {
return ReconServerConfigKeys.OZONE_RECON_DB_DIR;
}
-
- @Override
- public DBColumnFamilyDefinition[] getColumnFamilies() {
- return new DBColumnFamilyDefinition[] {
- CONTAINER_KEY, KEY_CONTAINER, CONTAINER_KEY_COUNT, REPLICA_HISTORY,
- NAMESPACE_SUMMARY, REPLICA_HISTORY_V2};
- }
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
index 4ee08863c6..ff17939010 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
@@ -27,13 +27,12 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
-import org.apache.hadoop.hdds.utils.db.CodecRegistry;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +46,6 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
LoggerFactory.getLogger(OMDBUpdatesHandler.class);
private Map<Integer, String> tablesNames;
- private CodecRegistry codecRegistry;
private OMMetadataManager omMetadataManager;
private List<OMDBUpdateEvent> omdbUpdateEvents = new ArrayList<>();
private Map<Object, OMDBUpdateEvent> omdbLatestUpdateEvents
@@ -57,7 +55,6 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
public OMDBUpdatesHandler(OMMetadataManager metadataManager) {
omMetadataManager = metadataManager;
tablesNames = metadataManager.getStore().getTableNames();
- codecRegistry = metadataManager.getStore().getCodecRegistry();
omdbDefinition = new OMDBDefinition();
}
@@ -102,14 +99,14 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
// When this table data will be needed, all events for this table will be
// saved using Object as key and new task will also retrieve using Object
// as key.
- Optional<Class> keyType = omdbDefinition.getKeyType(tableName);
- Optional<Class> valueType = omdbDefinition.getValueType(tableName);
- if (keyType.isPresent() && valueType.isPresent()) {
+ final DBColumnFamilyDefinition<?, ?> cf
+ = omdbDefinition.getColumnFamily(tableName);
+ if (cf != null) {
OMDBUpdateEvent.OMUpdateEventBuilder builder =
new OMDBUpdateEvent.OMUpdateEventBuilder<>();
builder.setTable(tableName);
builder.setAction(action);
- Object key = codecRegistry.asObject(keyBytes, keyType.get());
+ final Object key = cf.getKeyCodec().fromPersistedFormat(keyBytes);
builder.setKey(key);
// Put new
@@ -129,7 +126,7 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
}
if (action == PUT) {
- Object value = codecRegistry.asObject(valueBytes, valueType.get());
+ final Object value =
cf.getValueCodec().fromPersistedFormat(valueBytes);
builder.setValue(value);
// If a PUT operation happens on an existing Key, it is tagged
// as an "UPDATE" event.
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
index d03599fff3..5469d85c7b 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
@@ -302,20 +302,31 @@ public class TestOMDBUpdatesHandler {
@Test
public void testGetKeyType() throws IOException {
- assertEquals(String.class, omdbDefinition.getKeyType(
- omMetadataManager.getKeyTable(getBucketLayout()).getName()).get());
- assertEquals(OzoneTokenIdentifier.class, omdbDefinition.getKeyType(
- omMetadataManager.getDelegationTokenTable().getName()).get());
+ final String keyTable = omMetadataManager
+ .getKeyTable(getBucketLayout()).getName();
+ assertEquals(String.class,
+ omdbDefinition.getColumnFamily(keyTable).getKeyType());
+
+ final String delegationTokenTable = omMetadataManager
+ .getDelegationTokenTable().getName();
+ assertEquals(OzoneTokenIdentifier.class,
+ omdbDefinition.getColumnFamily(delegationTokenTable).getKeyType());
}
@Test
public void testGetValueType() throws IOException {
- assertEquals(OmKeyInfo.class, omdbDefinition.getValueType(
- omMetadataManager.getKeyTable(getBucketLayout()).getName()).get());
- assertEquals(OmVolumeArgs.class, omdbDefinition.getValueType(
- omMetadataManager.getVolumeTable().getName()).get());
- assertEquals(OmBucketInfo.class, omdbDefinition.getValueType(
- omMetadataManager.getBucketTable().getName()).get());
+ final String keyTable = omMetadataManager
+ .getKeyTable(getBucketLayout()).getName();
+ assertEquals(OmKeyInfo.class,
+ omdbDefinition.getColumnFamily(keyTable).getValueType());
+
+ final String volumeTable = omMetadataManager.getVolumeTable().getName();
+ assertEquals(OmVolumeArgs.class,
+ omdbDefinition.getColumnFamily(volumeTable).getValueType());
+
+ final String bucketTable = omMetadataManager.getBucketTable().getName();
+ assertEquals(OmBucketInfo.class,
+ omdbDefinition.getColumnFamily(bucketTable).getValueType());
}
@NotNull
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java
index 623651d0cc..9146abfbc9 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBScanner.java
@@ -49,9 +49,7 @@ import java.io.PrintWriter;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Callable;
@@ -307,20 +305,14 @@ public class DBScanner implements Callable<Void>,
SubcommandWithParent {
return false;
}
- Map<String, DBColumnFamilyDefinition> columnFamilyMap = new HashMap<>();
- for (DBColumnFamilyDefinition cfDef : dbDefinition.getColumnFamilies()) {
- LOG.info("Found table: {}", cfDef.getTableName());
- columnFamilyMap.put(cfDef.getTableName(), cfDef);
- }
- if (!columnFamilyMap.containsKey(tableName)) {
+ final DBColumnFamilyDefinition<?, ?> columnFamilyDefinition =
+ dbDefinition.getColumnFamily(tableName);
+ if (columnFamilyDefinition == null) {
err().print("Error: Table with name '" + tableName + "' not found");
return false;
}
-
- DBColumnFamilyDefinition columnFamilyDefinition =
- columnFamilyMap.get(tableName);
ColumnFamilyHandle columnFamilyHandle = getColumnFamilyHandle(
- columnFamilyDefinition.getTableName().getBytes(UTF_8),
+ columnFamilyDefinition.getName().getBytes(UTF_8),
columnFamilyHandleList);
if (columnFamilyHandle == null) {
throw new IllegalStateException("columnFamilyHandle is null");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
