This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8f7bf38bce4 HDDS-15730. Support more StatefulService types in
DBScanner (#10654)
8f7bf38bce4 is described below
commit 8f7bf38bce436cc511192ed329ad7f970d891a69
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Jul 13 05:10:48 2026 +0200
HDDS-15730. Support more StatefulService types in DBScanner (#10654)
---
.../scm/block/DeletedBlockLogStateManagerImpl.java | 6 ++-
.../SCMDeletedBlockTransactionStatusManager.java | 10 ++---
.../scm/container/balancer/ContainerBalancer.java | 9 +++-
.../apache/hadoop/hdds/scm/ha/StatefulService.java | 13 +++---
.../hdds/scm/ha/StatefulServiceDefinition.java | 45 +++++++++++++++++++
.../hdds/scm/security/RootCARotationManager.java | 6 ++-
.../apache/hadoop/ozone/debug/ldb/DBScanner.java | 51 +++++++++++++++-------
7 files changed, 108 insertions(+), 32 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
index 9d0a55fb54b..234b9d43403 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogStateManagerImpl.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.ha.SCMHADBTransactionBuffer;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServer;
+import org.apache.hadoop.hdds.scm.ha.StatefulServiceDefinition;
import
org.apache.hadoop.hdds.scm.ha.invoker.DeletedBlockLogStateManagerInvoker;
import org.apache.hadoop.hdds.utils.db.CodecException;
import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
@@ -52,7 +53,10 @@ public class DeletedBlockLogStateManagerImpl
private Table<String, ByteString> statefulConfigTable;
private final SCMHADBTransactionBuffer transactionBuffer;
private volatile Set<Long> deletingTxIDs;
- public static final String SERVICE_NAME =
DeletedBlockLogStateManager.class.getSimpleName();
+ private static final String SERVICE_NAME =
DeletedBlockLogStateManager.class.getSimpleName();
+
+ public static final
StatefulServiceDefinition<DeletedBlocksTransactionSummary> SERVICE_DEFINITION =
+ new StatefulServiceDefinition<>(SERVICE_NAME,
DeletedBlocksTransactionSummary.parser());
public DeletedBlockLogStateManagerImpl(Table<Long, DeletedBlocksTransaction>
deletedTable,
Table<String, ByteString> statefulServiceConfigTable,
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java
index 66c9d2070be..8b4932d7c09 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hdds.scm.block;
import static java.lang.Math.min;
-import static
org.apache.hadoop.hdds.scm.block.DeletedBlockLogStateManagerImpl.SERVICE_NAME;
+import static
org.apache.hadoop.hdds.scm.block.DeletedBlockLogStateManagerImpl.SERVICE_DEFINITION;
import static
org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus;
import static
org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus.SENT;
import static
org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus.TO_BE_SENT;
@@ -689,15 +689,15 @@ private void initDataDistributionData() throws
IOException {
private DeletedBlocksTransactionSummary loadDeletedBlocksSummary() throws
IOException {
String propertyName =
DeletedBlocksTransactionSummary.class.getSimpleName();
try {
- ByteString byteString = statefulConfigTable.get(SERVICE_NAME);
+ ByteString byteString =
statefulConfigTable.get(SERVICE_DEFINITION.getServiceName());
if (byteString == null) {
// for a new Ozone cluster, property not found is an expected state.
- LOG.info("Property {} for service {} not found. ", propertyName,
SERVICE_NAME);
+ LOG.info("Property {} for service {} not found. ", propertyName,
SERVICE_DEFINITION.getServiceName());
return null;
}
- return DeletedBlocksTransactionSummary.parseFrom(byteString);
+ return SERVICE_DEFINITION.deserialize(byteString);
} catch (IOException e) {
- LOG.error("Failed to get property {} for service {}.", propertyName,
SERVICE_NAME, e);
+ LOG.error("Failed to get property {} for service {}.", propertyName,
SERVICE_DEFINITION.getServiceName(), e);
throw new IOException("Failed to get property " + propertyName, e);
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
index 03df1ff2087..c516447f09f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancer.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
import org.apache.hadoop.hdds.scm.ha.StatefulService;
+import org.apache.hadoop.hdds.scm.ha.StatefulServiceDefinition;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +48,11 @@ public class ContainerBalancer extends
StatefulService<ContainerBalancerConfigur
private static final Logger LOG =
LoggerFactory.getLogger(ContainerBalancer.class);
+ private static final String SERVICE_NAME =
ContainerBalancer.class.getSimpleName();
+
+ public static final
StatefulServiceDefinition<ContainerBalancerConfigurationProto>
SERVICE_DEFINITION =
+ new StatefulServiceDefinition<>(SERVICE_NAME,
ContainerBalancerConfigurationProto.parser());
+
private StorageContainerManager scm;
private final SCMContext scmContext;
private OzoneConfiguration ozoneConfiguration;
@@ -65,8 +71,7 @@ public class ContainerBalancer extends
StatefulService<ContainerBalancerConfigur
* @param scm the storage container manager
*/
public ContainerBalancer(StorageContainerManager scm) {
- super(scm.getStatefulServiceStateManager(),
-
ContainerBalancerConfigurationProto.getDefaultInstance().getParserForType());
+ super(scm.getStatefulServiceStateManager(), SERVICE_DEFINITION);
this.scm = scm;
this.ozoneConfiguration = scm.getConfiguration();
this.config = ozoneConfiguration.getObject(
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
index 3343afe175e..5b35da371ce 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulService.java
@@ -19,7 +19,6 @@
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
-import com.google.protobuf.Parser;
import java.io.IOException;
/**
@@ -28,24 +27,22 @@
* @param <CONF> The configuration type, which is a protobuf {@link Message}.
*/
public abstract class StatefulService<CONF extends Message> implements
SCMService {
- private final String name;
private final StatefulServiceStateManager stateManager;
- private final Parser<CONF> parser;
+ private final StatefulServiceDefinition<CONF> definition;
/**
* Initialize a StatefulService from an extending class.
* @param stateManager a reference to the
* {@link StatefulServiceStateManager} from SCM.
*/
- protected StatefulService(StatefulServiceStateManager stateManager,
Parser<CONF> parser) {
- this.name = getClass().getSimpleName();
+ protected StatefulService(StatefulServiceStateManager stateManager,
StatefulServiceDefinition<CONF> definition) {
+ this.definition = definition;
this.stateManager = stateManager;
- this.parser = parser;
}
@Override
public final String getServiceName() {
- return name;
+ return definition.getServiceName();
}
/**
@@ -70,7 +67,7 @@ protected final CONF readConfiguration() throws IOException {
if (byteString == null) {
return null;
}
- return parser.parseFrom(byteString);
+ return definition.deserialize(byteString);
}
/**
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceDefinition.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceDefinition.java
new file mode 100644
index 00000000000..71f30d7a023
--- /dev/null
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/StatefulServiceDefinition.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.ha;
+
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.Message;
+import com.google.protobuf.Parser;
+
+/** Define static properties of stateful services. */
+public final class StatefulServiceDefinition<CONF extends Message> {
+ private final String name;
+ private final Parser<CONF> parser;
+
+ public StatefulServiceDefinition(String name, Parser<CONF> parser) {
+ this.name = name;
+ this.parser = parser;
+ }
+
+ public String getServiceName() {
+ return name;
+ }
+
+ public CONF deserialize(ByteString serialized) throws
InvalidProtocolBufferException {
+ if (serialized == null) {
+ return null;
+ }
+ return parser.parseFrom(serialized);
+ }
+}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
index 169bd39c59d..0826e4bff75 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationManager.java
@@ -56,6 +56,7 @@
import org.apache.hadoop.hdds.scm.ha.SequenceIdGenerator;
import org.apache.hadoop.hdds.scm.ha.SequenceIdType;
import org.apache.hadoop.hdds.scm.ha.StatefulService;
+import org.apache.hadoop.hdds.scm.ha.StatefulServiceDefinition;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.SecurityConfig;
@@ -81,6 +82,9 @@ public class RootCARotationManager extends
StatefulService<CertInfoProto> {
private static final String SERVICE_NAME =
RootCARotationManager.class.getSimpleName();
+ public static final StatefulServiceDefinition<CertInfoProto>
SERVICE_DEFINITION =
+ new StatefulServiceDefinition<>(SERVICE_NAME, CertInfoProto.parser());
+
private final StorageContainerManager scm;
private final OzoneConfiguration ozoneConf;
private final SecurityConfig secConf;
@@ -137,7 +141,7 @@ public class RootCARotationManager extends
StatefulService<CertInfoProto> {
* (4) Rotation Committed
*/
public RootCARotationManager(StorageContainerManager scm) {
- super(scm.getStatefulServiceStateManager(),
CertInfoProto.getDefaultInstance().getParserForType());
+ super(scm.getStatefulServiceStateManager(), SERVICE_DEFINITION);
this.scm = scm;
this.ozoneConf = scm.getConfiguration();
this.secConf = new SecurityConfig(ozoneConf);
diff --git
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java
index c5a34b19958..d0e0510716d 100644
---
a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java
+++
b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/ldb/DBScanner.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.debug.ldb;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.hadoop.hdds.scm.block.DeletedBlockLogStateManagerImpl.SERVICE_NAME;
import static
org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition.STATEFUL_SERVICE_CONFIG;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@@ -43,6 +42,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -57,9 +57,12 @@
import java.util.regex.Pattern;
import org.apache.hadoop.hdds.cli.AbstractSubcommand;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.block.DeletedBlockLogStateManagerImpl;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancer;
+import org.apache.hadoop.hdds.scm.ha.StatefulServiceDefinition;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.security.RootCARotationManager;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.hdds.utils.db.DBDefinition;
@@ -95,6 +98,12 @@ public class DBScanner extends AbstractSubcommand implements
Callable<Void> {
private static final Logger LOG = LoggerFactory.getLogger(DBScanner.class);
private static final String SCHEMA_V3 = "V3";
+ private static final List<StatefulServiceDefinition<?>>
STATEFUL_SERVICE_DEFINITIONS = Arrays.asList(
+ ContainerBalancer.SERVICE_DEFINITION,
+ DeletedBlockLogStateManagerImpl.SERVICE_DEFINITION,
+ RootCARotationManager.SERVICE_DEFINITION
+ );
+
@CommandLine.ParentCommand
private RDBParser parent;
@@ -727,6 +736,9 @@ public Void call() {
}
}
+ final boolean statefulServiceConfig =
+
dbColumnFamilyDefinition.getName().equals(STATEFUL_SERVICE_CONFIG.getName());
+
for (ByteArrayKeyValue byteArrayKeyValue : batch) {
StringBuilder sb = new StringBuilder();
if (!(sequenceId == FIRST_SEQUENCE_ID && results.isEmpty())) {
@@ -734,8 +746,9 @@ public Void call() {
// one, to ensure valid JSON format.
sb.append(", ");
}
- Object key = dbColumnFamilyDefinition.getKeyCodec()
- .fromPersistedFormat(byteArrayKeyValue.getKey());
+ Object key = withKey || statefulServiceConfig
+ ?
dbColumnFamilyDefinition.getKeyCodec().fromPersistedFormat(byteArrayKeyValue.getKey())
+ : null;
if (withKey) {
if (schemaV3) {
int index =
@@ -747,8 +760,8 @@ public Void call() {
exception = true;
break;
}
- String cid = key.toString().substring(0, index);
- String blockId = key.toString().substring(index);
+ String cid = keyStr.substring(0, index);
+ String blockId = keyStr.substring(index);
sb.append(writer.writeValueAsString(LongCodec.get()
.fromPersistedFormat(
FixedLengthStringCodec.string2Bytes(cid)) +
@@ -762,7 +775,9 @@ public Void call() {
Object o = dbColumnFamilyDefinition.getValueCodec()
.fromPersistedFormat(byteArrayKeyValue.getValue());
- o = parseStatefulServiceConfig(key, o, dbColumnFamilyDefinition);
+ if (statefulServiceConfig) {
+ o = parseStatefulServiceConfig(key, o);
+ }
if (valueFields != null) {
Map<String, Object> filteredValue = new HashMap<>();
@@ -835,16 +850,22 @@ List<Object>
getFieldsFilteredObjectCollection(Collection<?> valueObject, Map<St
}
}
- private Object parseStatefulServiceConfig(Object key, Object value,
DBColumnFamilyDefinition dbColumnFamily) {
- if (dbColumnFamily.getName().equals(STATEFUL_SERVICE_CONFIG.getName()) &&
key.equals(SERVICE_NAME) &&
- value instanceof ByteString) {
- try {
- return
HddsProtos.DeletedBlocksTransactionSummary.parseFrom((ByteString) value);
- } catch (IOException e) {
- LOG.error("Failed to parse {} for key {}",
STATEFUL_SERVICE_CONFIG.getName(), SERVICE_NAME, e);
+ private static Object parseStatefulServiceConfig(Object key, Object value) {
+ if (!(value instanceof ByteString)) {
+ return value;
+ }
+ try {
+ for (StatefulServiceDefinition<?> def : STATEFUL_SERVICE_DEFINITIONS) {
+ if (Objects.equals(key, def.getServiceName())) {
+ return def.deserialize((ByteString) value);
+ }
}
+ LOG.info("Unknown {} key {}", STATEFUL_SERVICE_CONFIG.getName(), key);
+ return value;
+ } catch (IOException e) {
+ LOG.error("Failed to parse {} for key {}",
STATEFUL_SERVICE_CONFIG.getName(), key, e);
+ return value;
}
- return value;
}
private static class ByteArrayKeyValue {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
