This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 58434d2176 HDDS-8310. Recon goes down with RepeatedOmKeyInfo cannot be
cast to OmKeyInfo (#5043)
58434d2176 is described below
commit 58434d2176ce23759526c1d9201c966cfdad205e
Author: Arafat2198 <[email protected]>
AuthorDate: Thu Jul 13 21:26:49 2023 +0530
HDDS-8310. Recon goes down with RepeatedOmKeyInfo cannot be cast to
OmKeyInfo (#5043)
---
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 20 +++
.../ozone/recon/tasks/OmUpdateEventValidator.java | 92 +++++++++++++
.../recon/tasks/TestContainerKeyMapperTask.java | 2 +
.../recon/tasks/TestOmUpdateEventValidator.java | 145 +++++++++++++++++++++
4 files changed, 259 insertions(+)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
index ff17939010..944ae32244 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
@@ -51,11 +51,13 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
private Map<Object, OMDBUpdateEvent> omdbLatestUpdateEvents
= new HashMap<>();
private OMDBDefinition omdbDefinition;
+ private OmUpdateEventValidator omUpdateEventValidator;
public OMDBUpdatesHandler(OMMetadataManager metadataManager) {
omMetadataManager = metadataManager;
tablesNames = metadataManager.getStore().getTableNames();
omdbDefinition = new OMDBDefinition();
+ omUpdateEventValidator = new OmUpdateEventValidator(omdbDefinition);
}
@Override
@@ -127,16 +129,34 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
if (action == PUT) {
final Object value =
cf.getValueCodec().fromPersistedFormat(valueBytes);
+
+ // If the updated value is not valid for this event, we skip it.
+ if (!omUpdateEventValidator.isValidEvent(tableName, value, key,
+ action)) {
+ return;
+ }
+
builder.setValue(value);
// If a PUT operation happens on an existing Key, it is tagged
// as an "UPDATE" event.
if (oldValue != null) {
+
+ // If the oldValue is not valid for this event, we skip it.
+ if (!omUpdateEventValidator.isValidEvent(tableName, oldValue, key,
+ action)) {
+ return;
+ }
+
builder.setOldValue(oldValue);
if (latestEvent == null || latestEvent.getAction() != DELETE) {
builder.setAction(UPDATE);
}
}
} else if (action.equals(DELETE)) {
+ if (oldValue != null && !omUpdateEventValidator.isValidEvent(tableName,
+ oldValue, key, action)) {
+ return;
+ }
// When you delete a Key, we add the old value to the event so that
// a downstream task can use it.
builder.setValue(oldValue);
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmUpdateEventValidator.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmUpdateEventValidator.java
new file mode 100644
index 0000000000..3c7ce844e9
--- /dev/null
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OmUpdateEventValidator.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.tasks;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * OmUpdateEventValidator is a utility class for validating OMDBUpdateEvents
+ * It can be further extended to different types of validations.
+ */
+public class OmUpdateEventValidator {
+
+ private static Logger log =
+ LoggerFactory.getLogger(OmUpdateEventValidator.class);
+ private OMDBDefinition omdbDefinition;
+
+ public OmUpdateEventValidator(OMDBDefinition omdbDefinition) {
+ this.omdbDefinition = omdbDefinition;
+ }
+
+ /**
+ * Validates the OMDBUpdateEvent based on the expected value type for a
+ * given table.
+ *
+ * @param tableName the name of the table associated with the event.
+ * @param actualValueType the actual value type of the event.
+ * @param keyType the key type of the event.
+ * @param action the action performed on the event.
+ * @return true if the event is valid, false otherwise.
+ * @throws IOException if an I/O error occurs during the validation.
+ */
+ public boolean isValidEvent(String tableName,
+ Object actualValueType,
+ Object keyType,
+ OMDBUpdateEvent.OMDBUpdateAction action) {
+
+ String expectedValueTypeString =
+ omdbDefinition.getColumnFamily(tableName).getValueType().getName();
+ String actualValueTypeString = actualValueType.getClass().getName();
+
+ // Check if both objects are of the same type
+ if (expectedValueTypeString.equals(actualValueTypeString)) {
+ // Both objects are of the same type
+ return true;
+ }
+ // Objects are not of the same type
+ logWarn(keyType.toString(), tableName, action.toString(),
+ expectedValueTypeString,
+ actualValueTypeString);
+ return false;
+ }
+
+ /**
+ * Logs an warning message indicating a validation failure.
+ */
+ private void logWarn(String keyType, String tableName, String action,
+ String expectedValueType, String actualValueType) {
+ String warnMessage = String.format(
+ "Validation failed for keyType: %s, tableName: %s, action: %s, " +
+ "Expected value type: %s, Actual value type: %s",
+ keyType, tableName, action, expectedValueType, actualValueType);
+ // Log the warning message as an WARN level log
+ log.warn(warnMessage);
+ }
+
+ @VisibleForTesting
+ public static void setLogger(Logger logger) {
+ OmUpdateEventValidator.log = logger;
+ }
+
+}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
index eea213a544..32ca5cab48 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestContainerKeyMapperTask.java
@@ -306,6 +306,7 @@ public class TestContainerKeyMapperTask {
OMUpdateEventBuilder<String, OmKeyInfo>()
.setKey(omKey)
.setAction(OMDBUpdateEvent.OMDBUpdateAction.DELETE)
+ .setValue(omKeyInfo)
.setTable(omMetadataManager.getKeyTable(getBucketLayout()).getName())
.build();
@@ -448,6 +449,7 @@ public class TestContainerKeyMapperTask {
OMUpdateEventBuilder<String, OmKeyInfo>()
.setKey(omKey)
.setAction(OMDBUpdateEvent.OMDBUpdateAction.DELETE)
+ .setValue(omKeyInfo)
.setTable(
omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED)
.getName())
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
new file mode 100644
index 0000000000..de866de152
--- /dev/null
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.tasks;
+
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.List;
+
+import static
org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.initializeNewOmMetadataManager;
+import static
org.apache.hadoop.ozone.recon.tasks.OMDBUpdateEvent.OMDBUpdateAction.PUT;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.times;
+
+/**
+ * Test class for OmUpdateEventValidator.
+ */
+public class TestOmUpdateEventValidator {
+
+ private OmUpdateEventValidator eventValidator;
+ private OMDBDefinition omdbDefinition;
+ private OMMetadataManager omMetadataManager;
+ private Logger logger;
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void setUp() throws IOException {
+ omMetadataManager = initializeNewOmMetadataManager(
+ temporaryFolder.newFolder());
+ omdbDefinition = new OMDBDefinition();
+ eventValidator = new OmUpdateEventValidator(omdbDefinition);
+ // Create a mock logger
+ logger = mock(Logger.class);
+ eventValidator.setLogger(logger);
+ }
+
+ @Test
+ public void testValidEvents() throws IOException {
+ // Validate a valid event for KeyTable
+ assertTrue(eventValidator.isValidEvent(
+ omMetadataManager.getKeyTable(BucketLayout.LEGACY).getName(),
+ mock(OmKeyInfo.class), "key1", PUT));
+
+ // Validate a valid event for BucketTable
+ assertTrue(eventValidator.isValidEvent(
+ omMetadataManager.getBucketTable().getName(),
+ mock(OmBucketInfo.class), "key1", PUT));
+
+ // Validate a valid event for DeletedTable
+ assertTrue(eventValidator.isValidEvent(
+ omMetadataManager.getDeletedTable().getName(),
+ mock(RepeatedOmKeyInfo.class), "key1", PUT));
+
+ // Validate a valid event for Prefix table
+ assertTrue(eventValidator.isValidEvent(
+ omMetadataManager.getPrefixTable().getName(),
+ mock(OmPrefixInfo.class), "key1", PUT));
+
+ // Validate a valid event for SnapshotInfo table
+ assertTrue(eventValidator.isValidEvent(
+ omMetadataManager.getSnapshotInfoTable().getName(),
+ mock(SnapshotInfo.class), "key1", PUT));
+
+ // Verify that no log message is printed
+ verify(logger, Mockito.never()).warn(Mockito.anyString());
+ }
+
+ @Test
+ public void testInvalidEvents() throws IOException {
+
+ // Validate an invalid event for VolumeInfo table
+ assertFalse(eventValidator.isValidEvent(
+ omMetadataManager.getVolumeTable().getName(),
+ "Invalid Object", "key1", PUT));
+
+ // Validate an invalid event for BucketTable
+ assertFalse(eventValidator.isValidEvent(
+ omMetadataManager.getBucketTable().getName(),
+ "Invalid Object", "key1", PUT));
+
+ // Validate an invalid event for DeletedTable
+ assertFalse(eventValidator.isValidEvent(
+ omMetadataManager.getDeletedTable().getName(),
+ "Invalid Object", "key1", PUT));
+
+ // Validate an invalid event for Prefix table
+ assertFalse(eventValidator.isValidEvent(
+ omMetadataManager.getPrefixTable().getName(),
+ "Invalid Object", "key1", PUT));
+
+ // Validate an invalid event for SnapshotInfo table
+ assertFalse(eventValidator.isValidEvent(
+ omMetadataManager.getSnapshotInfoTable().getName(),
+ "Invalid Object", "key1", PUT));
+ // Verify that the logger is called 5 times
+ verifyLogMessage(logger);
+ }
+
+ private void verifyLogMessage(Logger localLogger) {
+ // Use ArgumentCaptor to capture the log message
+ ArgumentCaptor<String> captor = ArgumentCaptor.forClass(String.class);
+ verify(localLogger, times(5)).warn(captor.capture());
+
+ // Assert that the captured log messages are not empty
+ List<String> logMessages = captor.getAllValues();
+ for (String logMessage : logMessages) {
+ assertFalse("Warning message is empty", logMessage.isEmpty());
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
