This is an automated email from the ASF dual-hosted git repository.
krathbun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7ebaf2f3bb Improved metadata columns test coverage (#5740)
7ebaf2f3bb is described below
commit 7ebaf2f3bb349cf6fe11bc8565c7484a7bd40589
Author: Imirie Billey <[email protected]>
AuthorDate: Tue Jul 22 14:50:39 2025 -0400
Improved metadata columns test coverage (#5740)
* Improved metadata columns test coverage
Improved the test coverage of metadata columns by adding a test within the
testAllColumns method of TabletMetadataTest that fails if new columns are added
to the metadata stores but are untested. In this method added tests for all
current ColumnTypes that had not been tested.
Closes #5521
---
.../accumulo/core/metadata/SuspendingTServer.java | 2 +-
.../core/metadata/schema/TabletMetadataTest.java | 78 +++++++++++++++++++++-
2 files changed, 78 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
b/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
index e45635fc80..335392d30d 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/SuspendingTServer.java
@@ -33,7 +33,7 @@ public class SuspendingTServer {
public final HostAndPort server;
public final SteadyTime suspensionTime;
- SuspendingTServer(HostAndPort server, SteadyTime suspensionTime) {
+ public SuspendingTServer(HostAndPort server, SteadyTime suspensionTime) {
this.server = Objects.requireNonNull(server);
this.suspensionTime = Objects.requireNonNull(suspensionTime);
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index a347275a49..b56315d8d7 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@ -26,17 +26,34 @@ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.FLUSH_NONCE_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.MIGRATION_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.SELECTED_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.MERGEABILITY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.AVAILABILITY;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.CLONED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FLUSH_ID;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FLUSH_NONCE;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGEABILITY;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MERGED;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.MIGRATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SCANS;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.UNSPLITTABLE;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
@@ -64,6 +81,7 @@ import java.util.stream.Stream;
import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.clientImpl.TabletAvailabilityUtil;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
@@ -106,6 +124,8 @@ public class TabletMetadataTest {
@Test
public void testAllColumns() {
+ Set<ColumnType> allColumns = EnumSet.allOf(ColumnType.class);
+
KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
@@ -114,9 +134,24 @@ public class TabletMetadataTest {
FateId fateId1 = FateId.from(type, UUID.randomUUID());
FateId fateId2 = FateId.from(type, UUID.randomUUID());
+ mutation.put(MetadataSchema.TabletsSection.CompactedColumnFamily.STR_NAME,
fateId1.canonical(),
+ "");
+
DIRECTORY_COLUMN.put(mutation, new Value("t-0001757"));
FLUSH_COLUMN.put(mutation, new Value("6"));
TIME_COLUMN.put(mutation, new Value("M123456789"));
+ var opid = TabletOperationId.from(TabletOperationType.SPLITTING,
+ FateId.from(FateInstanceType.META, UUID.randomUUID()));
+ OPID_COLUMN.put(mutation, new Value(opid.canonical()));
+ Path selectedPath =
+ new
Path("hdfs://nn.somewhere.com:86753/accumulo/tables/42/t-0000/F00001.rf");
+ SELECTED_COLUMN.put(mutation,
+ new Value(new SelectedFiles(Set.of(new
ReferencedTabletFile(selectedPath).insert()), true,
+ fateId1, SteadyTime.from(100,
TimeUnit.NANOSECONDS)).getMetadataValue()));
+ AVAILABILITY_COLUMN.put(mutation,
TabletAvailabilityUtil.toValue(TabletAvailability.ONDEMAND));
+ TabletMergeabilityMetadata tmm =
TabletMergeabilityMetadata.after(Duration.ofMinutes(3),
+ SteadyTime.from(Duration.ofMinutes(1)));
+ MERGEABILITY_COLUMN.put(mutation, TabletMergeabilityMetadata.toValue(tmm));
String bf1 = serialize("hdfs://nn1/acc/tables/1/t-0001/bf1");
String bf2 = serialize("hdfs://nn1/acc/tables/1/t-0001/bf2");
@@ -153,10 +188,13 @@ public class TabletMetadataTest {
UnSplittableMetadata.toUnSplittable(extent, 100, 110, 120, Set.of(sf1,
sf2));
SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new
Value(unsplittableMeta.toBase64()));
- SteadyTime suspensionTime = SteadyTime.from(System.currentTimeMillis(),
TimeUnit.MILLISECONDS);
+ SteadyTime suspensionTime = SteadyTime.from(1000L, TimeUnit.MILLISECONDS);
TServerInstance ser1 = new
TServerInstance(HostAndPort.fromParts("server1", 8555), "s001");
+ SuspendingTServer suspendingTServer =
+ new SuspendingTServer(HostAndPort.fromParts("server1", 8555),
suspensionTime);
Value suspend = SuspendingTServer.toValue(ser1, suspensionTime);
SUSPEND_COLUMN.put(mutation, suspend);
+
FLUSH_NONCE_COLUMN.put(mutation, new Value(Long.toHexString(10L)));
ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
@@ -170,6 +208,7 @@ public class TabletMetadataTest {
mutation.put(ExternalCompactionColumnFamily.STR_NAME, ecid.canonical(),
ecMeta.toJson());
TServerInstance tsi = new TServerInstance("localhost:9997", 5000L);
+
MIGRATION_COLUMN.put(mutation, new Value(tsi.getHostPortSession()));
SortedMap<Key,Value> rowMap = toRowMap(mutation);
@@ -177,39 +216,76 @@ public class TabletMetadataTest {
TabletMetadata tm = TabletMetadata.convertRow(rowMap.entrySet().iterator(),
EnumSet.allOf(ColumnType.class), true, false);
+ assertFalse(tm.getCompacted().isEmpty());
+ assertEquals(Set.of(fateId1), tm.getCompacted());
+ allColumns.remove(COMPACTED);
+ assertEquals(tmm, tm.getTabletMergeability());
+ allColumns.remove(MERGEABILITY);
+ assertEquals(TabletAvailability.ONDEMAND, tm.getTabletAvailability());
+ allColumns.remove(AVAILABILITY);
+ assertEquals(1, tm.getSelectedFiles().getFiles().size());
+ assertEquals(selectedPath.toString(),
+ tm.getSelectedFiles().getFiles().iterator().next().getMetadataPath());
+ allColumns.remove(SELECTED);
+ assertEquals(opid, tm.getOperationId());
+ allColumns.remove(OPID);
+ assertFalse(tm.getHostingRequested());
+ allColumns.remove(HOSTING_REQUESTED);
+ assertEquals(suspendingTServer, tm.getSuspend());
+ allColumns.remove(SUSPEND);
assertEquals("OK", tm.getCloned());
+ allColumns.remove(CLONED);
assertEquals("t-0001757", tm.getDirName());
+ allColumns.remove(DIR);
assertEquals(extent.endRow(), tm.getEndRow());
assertEquals(extent, tm.getExtent());
assertEquals(Set.of(tf1, tf2), Set.copyOf(tm.getFiles()));
+ allColumns.remove(FILES);
assertEquals(Map.of(tf1, dfv1, tf2, dfv2), tm.getFilesMap());
assertEquals(tm.getFilesMap().values().stream().mapToLong(DataFileValue::getSize).sum(),
tm.getFileSize());
assertEquals(6L, tm.getFlushId().getAsLong());
+ allColumns.remove(FLUSH_ID);
SortedMap<Key,Value> actualRowMap = tm.getKeyValues().stream().collect(
Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b,
TreeMap::new));
assertEquals(rowMap, actualRowMap);
assertEquals(Map.of(new StoredTabletFile(bf1), fateId1, new
StoredTabletFile(bf2), fateId2),
tm.getLoaded());
+ allColumns.remove(LOADED);
assertEquals(HostAndPort.fromParts("server1", 8555),
tm.getLocation().getHostAndPort());
+ allColumns.remove(LOCATION);
assertEquals("s001", tm.getLocation().getSession());
assertEquals(LocationType.CURRENT, tm.getLocation().getType());
assertTrue(tm.hasCurrent());
assertEquals(HostAndPort.fromParts("server2", 8555),
tm.getLast().getHostAndPort());
assertEquals("s000", tm.getLast().getSession());
+ allColumns.remove(LAST);
assertEquals(LocationType.LAST, tm.getLast().getType());
assertEquals(Set.of(le1, le2), tm.getLogs().stream().collect(toSet()));
+ allColumns.remove(LOGS);
assertEquals(extent.prevEndRow(), tm.getPrevEndRow());
+ allColumns.remove(PREV_ROW);
assertEquals(extent.tableId(), tm.getTableId());
assertTrue(tm.sawPrevEndRow());
assertEquals("M123456789", tm.getTime().encode());
+ allColumns.remove(TIME);
assertEquals(Set.of(sf1, sf2), Set.copyOf(tm.getScans()));
+ allColumns.remove(SCANS);
assertTrue(tm.hasMerged());
+ allColumns.remove(MERGED);
assertTrue(tm.getUserCompactionsRequested().contains(userCompactFateId));
+ allColumns.remove(USER_COMPACTION_REQUESTED);
assertEquals(unsplittableMeta, tm.getUnSplittable());
+ allColumns.remove(UNSPLITTABLE);
assertEquals(ecMeta.toJson(),
tm.getExternalCompactions().get(ecid).toJson());
+ allColumns.remove(ECOMP);
assertEquals(10, tm.getFlushNonce().getAsLong());
+ allColumns.remove(FLUSH_NONCE);
assertEquals(tsi, tm.getMigration());
+ allColumns.remove(MIGRATION);
+
+ assertTrue(allColumns.isEmpty(),
+ "Not all columns are tested. Add testing to remaining columns: " +
allColumns);
}
@Test
