kevinrr888 commented on code in PR #5474: URL: https://github.com/apache/accumulo/pull/5474#discussion_r2060498700
########## test/src/main/java/org/apache/accumulo/test/ComprehensiveTableOperationsIT.java: ########## @@ -0,0 +1,937 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TabletMergeability; +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.client.summary.SummarizerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.clientImpl.TabletMergeabilityUtil; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iterators.IteratorUtil; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iterators.user.VersioningIterator; +import org.apache.accumulo.core.metadata.AccumuloTable; +import org.apache.accumulo.core.metadata.ScanServerRefTabletFile; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.harness.SharedMiniClusterBase; +import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.test.functional.BasicSummarizer; +import org.apache.accumulo.test.functional.BulkNewIT; +import org.apache.accumulo.test.functional.CloneTestIT; +import org.apache.accumulo.test.functional.CompactionIT; +import org.apache.accumulo.test.functional.ConstraintIT; +import org.apache.accumulo.test.functional.DeleteRowsIT; +import org.apache.accumulo.test.functional.LocalityGroupIT; +import org.apache.accumulo.test.functional.ManagerAssignmentIT; +import org.apache.accumulo.test.functional.MergeTabletsIT; +import org.apache.accumulo.test.functional.ReadWriteIT; +import org.apache.accumulo.test.functional.RenameIT; +import org.apache.accumulo.test.functional.SlowIterator; +import org.apache.accumulo.test.functional.SummaryIT; +import org.apache.accumulo.test.functional.TabletAvailabilityIT; +import org.apache.accumulo.test.util.Wait; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Sets; +import com.google.common.net.HostAndPort; + +/** + * A comprehensive IT of all table operations against user tables and all system tables while + * avoiding duplicating existing testing. This does not test for edge cases, but rather tests for + * basic expected functionality of all table operations against user tables and all system tables. + */ +public class ComprehensiveTableOperationsIT extends SharedMiniClusterBase { + private static final Logger log = LoggerFactory.getLogger(ComprehensiveTableOperationsIT.class); + private static final String SLOW_ITER_NAME = "CustomSlowIter"; + private AccumuloClient client; + private TableOperations ops; + private String userTable; + + @Override + protected Duration defaultTimeout() { + return Duration.ofMinutes(1); + } + + @BeforeAll + public static void setup() throws Exception { + SharedMiniClusterBase.startMiniCluster(); + } + + @AfterAll + public static void teardown() { + SharedMiniClusterBase.stopMiniCluster(); + } + + @BeforeEach + public void beforeEach() { + client = Accumulo.newClient().from(getClientProps()).build(); + ops = client.tableOperations(); + } + + @AfterEach + public void afterEach() throws Exception { + // ensure none of the FATE or SCAN_REF data we created persists between tests. Also ensure the + // user table does not persist between tests + if (userTable != null) { + cleanupFateTable(); + ops.delete(userTable); + userTable = null; + } + cleanupScanRefTable(); + client.close(); + } + + @Test + public void testAllTested() { + var allTableOps = + Arrays.stream(TableOperations.class.getDeclaredMethods()).map(Method::getName); + var testMethodNames = Arrays.stream(ComprehensiveTableOperationsIT.class.getDeclaredMethods()) + .map(Method::getName).collect(Collectors.toSet()); + var untestedOps = + allTableOps.filter(op -> testMethodNames.stream().noneMatch(test -> test.contains(op))) + .collect(Collectors.toSet()); + assertTrue(untestedOps.isEmpty(), "The table operations " + untestedOps + " are untested"); + } + + @Test + public void testExpectedSystemTables() { + var testedSystemTableIds = + Set.of(AccumuloTable.ROOT.tableId(), AccumuloTable.METADATA.tableId(), + AccumuloTable.FATE.tableId(), AccumuloTable.SCAN_REF.tableId()); + assertEquals(testedSystemTableIds, AccumuloTable.allTableIds(), + "There are newly added system tables that are untested in this IT. Ensure each " + + "test has testing for the new system table"); + } + + @Test + public void test_list_tableIdMap() throws Exception { + // Thoroughly tested elsewhere, but simple enough to test here. Test that all system tables + // and user tables are returned + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + var expected = Arrays.stream(AccumuloTable.values()).map(AccumuloTable::tableName) + .collect(Collectors.toSet()); + expected.add(userTable); + assertEquals(expected, ops.list()); + assertEquals(expected, ops.tableIdMap().keySet()); + for (Map.Entry<String,String> entry : ops.tableIdMap().entrySet()) { + var tableName = entry.getKey(); + var tableId = TableId.of(entry.getValue()); + if (tableName.equals(AccumuloTable.ROOT.tableName())) { + assertEquals(AccumuloTable.ROOT.tableId(), tableId); + } else if (tableName.equals(AccumuloTable.METADATA.tableName())) { + assertEquals(AccumuloTable.METADATA.tableId(), tableId); + } else if (tableName.equals(AccumuloTable.FATE.tableName())) { + assertEquals(AccumuloTable.FATE.tableId(), tableId); + } else if (tableName.equals(AccumuloTable.SCAN_REF.tableName())) { + assertEquals(AccumuloTable.SCAN_REF.tableId(), tableId); + } else if (tableName.equals(userTable)) { + assertFalse(AccumuloTable.allTableIds().contains(tableId)); + } else { + throw new IllegalStateException("Unrecognized table: " + tableName); + } + } + } + + @Test + public void test_exists() throws Exception { + // Thoroughly tested elsewhere, but simple enough to test here. Test that all system tables + // and user tables exist + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + var expected = Arrays.stream(AccumuloTable.values()).map(AccumuloTable::tableName) + .collect(Collectors.toSet()); + expected.add(userTable); + for (String table : expected) { + assertTrue(ops.exists(table)); + } + } + + @Test + public void test_create() { + // Creating user tables is thoroughly tested. Make sure we can't create any of the already + // existing system tables, though. + for (var systemTable : AccumuloTable.values()) { + assertThrows(AccumuloException.class, () -> ops.create(systemTable.tableName())); + } + } + + @Test + public void test_exportTable_importTable() throws Exception { + // exportTable, importTable for user tables is tested in ImportExportIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(ImportExportIT.class.getName())); + // exportTable, importTable untested for system tables. Test basic functionality here + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + Path baseDir = ImportExportIT.createBaseDir(getCluster(), getClass()); + var fs = getCluster().getFileSystem(); + + // export the user table, so we can test importing into the system tables + ops.offline(userTable, true); + Path exportUserDir; + exportUserDir = new Path(baseDir, "export_userdir"); + fs.deleteOnExit(exportUserDir); + ops.exportTable(userTable, exportUserDir.toString()); + + var sysTables = AccumuloTable.values(); + for (int i = 0; i < sysTables.length; i++) { + var sysTable = sysTables[i]; + Path exportDir = new Path(baseDir, "export" + i); + fs.deleteOnExit(exportDir); + + // not offline, can't export + assertThrows(IllegalStateException.class, + () -> ops.exportTable(sysTable.tableName(), exportDir.toString())); + // can't offline, so will never be able to export + assertThrows(AccumuloException.class, () -> ops.offline(sysTable.tableName(), true)); + assertThrows(AccumuloException.class, + () -> ops.importTable(sysTable.tableName(), exportUserDir.toString())); + } + } + + @Test + public void test_addSplits_putSplits_listSplits_splitRangeByTablets() throws Exception { + // addSplits,listSplits,putSplits tested elsewhere for METADATA, ROOT, and user tables, but + // testing here as well since this setup is needed to test for splitRangeByTablets anyway, + // which is untested elsewhere + + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + // system and user tables + var allTables = ops.list(); + + for (String table : allTables) { + SortedSet<Text> splits1 = new TreeSet<>(); + splits1.add(new Text("split1")); + SortedSet<Text> splits2 = new TreeSet<>(); + splits2.add(new Text("split2")); + SortedMap<Text,TabletMergeability> splits2Map = + TabletMergeabilityUtil.userDefaultSplits(splits2); + + if (table.equals(AccumuloTable.ROOT.tableName())) { + // cannot add splits to ROOT + assertThrows(AccumuloException.class, () -> ops.addSplits(table, splits1)); + assertThrows(AccumuloException.class, () -> ops.putSplits(table, splits2Map)); + assertEquals(0, ops.listSplits(table).size()); + } else { + ops.addSplits(table, splits1); + ops.putSplits(table, splits2Map); + var listSplits = ops.listSplits(table); + assertTrue(listSplits.containsAll(splits1)); + assertTrue(listSplits.containsAll(splits2)); + } + + assertEquals(ops.splitRangeByTablets(table, new Range(), 99).size(), + ops.listSplits(table).size() + 1); + } + } + + @Test + public void test_locate() throws Exception { + // locate for user tables is tested in LocatorIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(LocatorIT.class.getName())); + // locate for METADATA and ROOT tables is tested in ManagerAssignmentIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(ManagerAssignmentIT.class.getName())); + + // basic functionality check for locate on FATE and SCAN_REF tables + var fateLocations = + ops.locate(AccumuloTable.FATE.tableName(), Collections.singletonList(new Range())); + var fateGroupByTablet = fateLocations.groupByTablet().keySet(); + assertFalse(fateGroupByTablet.isEmpty()); + fateGroupByTablet.forEach(tid -> { + var tabletLoc = fateLocations.getTabletLocation(tid); + assertNotNull(tabletLoc); + assertTrue(tabletLoc.contains(":")); + }); + var scanRefLocations = + ops.locate(AccumuloTable.SCAN_REF.tableName(), Collections.singletonList(new Range())); + var scanRefGroupByTablet = scanRefLocations.groupByTablet().keySet(); + assertFalse(scanRefGroupByTablet.isEmpty()); + scanRefGroupByTablet.forEach(tid -> { + var tabletLoc = scanRefLocations.getTabletLocation(tid); + assertNotNull(tabletLoc); + assertTrue(tabletLoc.contains(":")); + }); + } + + @Test + public void test_getMaxRow_deleteRows() throws Exception { + // getMaxRow for user tables is tested in FindMaxIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(FindMaxIT.class.getName())); + // getMaxRow not tested for system tables. Test basic functionality here + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + createFateTableRow(userTable); + createScanRefTableRow(); + for (var sysTable : AccumuloTable.values()) { + var maxRow = + ops.getMaxRow(sysTable.tableName(), Authorizations.EMPTY, null, true, null, true); + log.info("Max row of {} : {}", sysTable, maxRow); + assertNotNull(maxRow); + } + + // deleteRows for user tables is tested in DeleteRowsIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(DeleteRowsIT.class.getName())); + // deleteRows not tested for system tables. Test basic functionality here + for (var sysTable : AccumuloTable.values()) { + switch (sysTable) { + case METADATA: + case ROOT: + // should not be able to delete rows of METADATA or ROOT tables + assertThrows(AccumuloException.class, + () -> ops.deleteRows(sysTable.tableName(), null, null)); + break; + case FATE: + break; + case SCAN_REF: + cleanupScanRefTable(); + // check max row again + assertNull( + ops.getMaxRow(sysTable.tableName(), Authorizations.EMPTY, null, true, null, true)); + break; + default: + throw new IllegalStateException("Unrecognized table: " + sysTable); + } + } + } + + @Test + public void test_merge() throws Exception { + // merge for user tables is tested in various ITs. One example is MergeTabletsIT. Ensure + // test exists + assertDoesNotThrow(() -> Class.forName(MergeTabletsIT.class.getName())); + // merge for METADATA and ROOT system tables tested in MetaSplitIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(MetaSplitIT.class.getName())); + // merge for FATE and SCAN_REF tables not tested. Test basic functionality here + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + var fateRow1 = createFateTableRow(userTable); + createFateTableRow(userTable); // fate row 2 + var scanRefRow1 = createScanRefTableRow(); + createScanRefTableRow(); // scan ref row 2 + + SortedSet<Text> fateSplits = new TreeSet<>(); + fateSplits.add(new Text(fateRow1)); + ops.addSplits(AccumuloTable.FATE.tableName(), fateSplits); + // there may be preexisting splits on the FATE table from a previous test + assertTrue(ops.listSplits(AccumuloTable.FATE.tableName()).contains(fateRow1)); + ops.merge(AccumuloTable.FATE.tableName(), null, null); + assertTrue(ops.listSplits(AccumuloTable.FATE.tableName()).isEmpty()); + + SortedSet<Text> scanRefSplits = new TreeSet<>(); + scanRefSplits.add(new Text(scanRefRow1)); + ops.addSplits(AccumuloTable.SCAN_REF.tableName(), scanRefSplits); + // there may be preexisting splits on the SCAN_REF table from a previous test + assertTrue(ops.listSplits(AccumuloTable.SCAN_REF.tableName()).contains(scanRefRow1)); + ops.merge(AccumuloTable.SCAN_REF.tableName(), null, null); + assertTrue(ops.listSplits(AccumuloTable.SCAN_REF.tableName()).isEmpty()); + } + + @Test + public void test_compact() throws Exception { + // compact for user tables is tested in various ITs. One example is CompactionIT. Ensure + // test exists + assertDoesNotThrow(() -> Class.forName(CompactionIT.class.getName())); + // disable the GC to prevent automatic compactions on METADATA and ROOT tables + getCluster().getClusterControl().stopAllServers(ServerType.GARBAGE_COLLECTOR); + try { + // test basic functionality for system tables + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + + // create some RFiles for the METADATA and ROOT tables by creating some data in the user + // table, flushing that table, then the METADATA table, then the ROOT table + for (int i = 0; i < 3; i++) { + try (var bw = client.createBatchWriter(userTable)) { + var mut = new Mutation("r" + i); + mut.put("cf", "cq", "v"); + bw.addMutation(mut); + } + ops.flush(userTable, null, null, true); + ops.flush(AccumuloTable.METADATA.tableName(), null, null, true); + ops.flush(AccumuloTable.ROOT.tableName(), null, null, true); + } + + for (var sysTable : List.of(AccumuloTable.ROOT, AccumuloTable.METADATA, + AccumuloTable.SCAN_REF, AccumuloTable.FATE)) { + // create some RFiles for FATE and SCAN_REF tables + if (sysTable == AccumuloTable.SCAN_REF) { + createScanRefTableRow(); + ops.flush(AccumuloTable.SCAN_REF.tableName(), null, null, true); + } else if (sysTable == AccumuloTable.FATE) { + createFateTableRow(userTable); + ops.flush(AccumuloTable.FATE.tableName(), null, null, true); + } + + Set<StoredTabletFile> stfsBeforeCompact = getStoredTabFiles(sysTable); + + log.info("Compacting " + sysTable); + ops.compact(sysTable.tableName(), null, null, true, true); + log.info("Finished compacting " + sysTable); + + // RFiles resulting from a compaction begin with 'A'. Wait until we see an RFile beginning + // with 'A' that was not present before the compaction. + Wait.waitFor(() -> { + var stfsAfterCompact = getStoredTabFiles(sysTable); + String regex = "^A.*\\.rf$"; + var A_stfsBeforeCompaction = stfsBeforeCompact.stream() + .filter(stf -> stf.getFileName().matches(regex)).collect(Collectors.toSet()); + var A_stfsAfterCompaction = stfsAfterCompact.stream() + .filter(stf -> stf.getFileName().matches(regex)).collect(Collectors.toSet()); + return !Sets.difference(A_stfsAfterCompaction, A_stfsBeforeCompaction).isEmpty(); + }); + } + } finally { + getCluster().getClusterControl().startAllServers(ServerType.GARBAGE_COLLECTOR); + } + } + + @Test + public void test_cancelCompaction() throws Exception { + // cancelCompaction for user tables is tested in various ITs. One example is TableOperationsIT. + // Ensure test exists + assertDoesNotThrow(() -> Class.forName(TableOperationsIT.class.getName())); + // test basic functionality for system tables + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + + // Need some data in all the system tables. This allows the slow iterator we attach to the + // system table to work when we compact. ROOT and METADATA will already have data, so just need + // to create some data for the other system tables + createFateTableRow(userTable); + createScanRefTableRow(); + + for (var sysTable : AccumuloTable.values()) { + try { + var zrw = getCluster().getServerContext().getZooSession().asReaderWriter(); + attachSlowMajcIterator(sysTable.tableName()); + + var metaFatesBeforeCompact = new HashSet<>(zrw.getChildren(Constants.ZFATE)); + + log.info("Compacting " + sysTable); + ops.compact(sysTable.tableName(), null, null, true, false); + log.info("Initiated compaction for " + sysTable); + + // Wait for the compaction to be started + Wait.waitFor(() -> { + var metaFatesAfterCompact = new HashSet<>(zrw.getChildren(Constants.ZFATE)); + return !Sets.difference(metaFatesAfterCompact, metaFatesBeforeCompact).isEmpty(); + }); + + log.info("Cancelling compaction " + sysTable); + ops.cancelCompaction(sysTable.tableName()); + + // We can be sure that the compaction has been cancelled once we see no FATE operations + Wait.waitFor(() -> { + Set<String> metaFatesAfterCancelCompact = new HashSet<>(zrw.getChildren(Constants.ZFATE)); + return metaFatesAfterCancelCompact.isEmpty(); + }); + } finally { + removeSlowMajcIterator(sysTable.tableName()); + } + } + } + + @Test + public void test_delete() { + // delete for user tables is tested in TableOperationsIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(TableOperationsIT.class.getName())); + // delete not tested for system tables. Test basic functionality here + for (var sysTable : AccumuloTable.values()) { + assertThrows(AccumuloException.class, () -> ops.delete(sysTable.tableName())); + } + } + + @Test + public void test_clone() { + // cloning user and system tables is tested in CloneTestIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(CloneTestIT.class.getName())); + } + + @Test + public void test_rename() { + // rename for user tables is tested in RenameIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(RenameIT.class.getName())); + // rename not tested for system tables. Test basic functionality here + var sysTables = AccumuloTable.values(); + var tableNames = getUniqueNames(sysTables.length); + + for (int i = 0; i < sysTables.length; i++) { + var sysTable = sysTables[i]; + var tableName = tableNames[i]; + assertThrows(AccumuloException.class, () -> ops.rename(sysTable.tableName(), tableName)); + } + } + + @Test + public void test_flush() throws Exception { + // flush for user tables and the METADATA and ROOT tables is tested in CompactionIT. Ensure + // test exists + assertDoesNotThrow(() -> Class.forName(CompactionIT.class.getName())); + // flush for FATE and SCAN_REF not tested for. Test basic functionality here + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + createFateTableRow(userTable); + createScanRefTableRow(); + + for (var sysTable : Set.of(AccumuloTable.FATE, AccumuloTable.SCAN_REF)) { + int numFilesBeforeFlush = 0; + + try (TabletsMetadata tabletsMetadata = getCluster().getServerContext().getAmple() + .readTablets().forTable(sysTable.tableId()).build()) { + for (var tm : tabletsMetadata) { + numFilesBeforeFlush += tm.getFiles().size(); + } + } + ops.flush(sysTable.tableName(), null, null, true); + // Wait until we see the new file + final int finalNumFilesBeforeFlush = numFilesBeforeFlush; + Wait.waitFor(() -> { + try (TabletsMetadata tabletsMetadata = getCluster().getServerContext().getAmple() + .readTablets().forTable(sysTable.tableId()).build()) { + int numFilesAfterFlush = 0; + for (var tm : tabletsMetadata) { + numFilesAfterFlush += tm.getFiles().size(); + } + return numFilesAfterFlush > finalNumFilesBeforeFlush; Review Comment: Good point. Addressed in c88263153459d77dad6ce76bb3c7ac4449bdbde1 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@accumulo.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
