kevinrr888 commented on code in PR #5474: URL: https://github.com/apache/accumulo/pull/5474#discussion_r2050989517
########## test/src/main/java/org/apache/accumulo/test/ComprehensiveTableOperationsIT.java: ########## @@ -0,0 +1,958 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.accumulo.core.Constants; +import org.apache.accumulo.core.client.Accumulo; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.admin.TabletAvailability; +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.client.summary.SummarizerConfiguration; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.Filter; +import org.apache.accumulo.core.iterators.IteratorUtil; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.iterators.user.VersioningIterator; +import org.apache.accumulo.core.metadata.AccumuloTable; +import org.apache.accumulo.core.metadata.ScanServerRefTabletFile; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.harness.SharedMiniClusterBase; +import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.test.functional.BasicSummarizer; +import org.apache.accumulo.test.functional.BulkNewIT; +import org.apache.accumulo.test.functional.CloneTestIT; +import org.apache.accumulo.test.functional.CompactionIT; +import org.apache.accumulo.test.functional.ConstraintIT; +import org.apache.accumulo.test.functional.DeleteRowsIT; +import org.apache.accumulo.test.functional.LocalityGroupIT; +import org.apache.accumulo.test.functional.ManagerAssignmentIT; +import org.apache.accumulo.test.functional.MergeTabletsIT; +import org.apache.accumulo.test.functional.ReadWriteIT; +import org.apache.accumulo.test.functional.RenameIT; +import org.apache.accumulo.test.functional.SlowIterator; +import org.apache.accumulo.test.functional.SummaryIT; +import org.apache.accumulo.test.functional.TabletAvailabilityIT; +import org.apache.accumulo.test.util.Wait; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Sets; +import com.google.common.net.HostAndPort; + +/** + * A comprehensive IT of all table operations against user tables and all system tables while + * avoiding duplicating existing testing. This does not test for edge cases, but rather tests for + * basic expected functionality of all table operations against user tables and all system tables. + */ +public class ComprehensiveTableOperationsIT extends SharedMiniClusterBase { + private static final Logger log = LoggerFactory.getLogger(ComprehensiveTableOperationsIT.class); + private static final String SLOW_ITER_NAME = "CustomSlowIter"; + private AccumuloClient client; + private TableOperations ops; + private String userTable; + + @Override + protected Duration defaultTimeout() { + return Duration.ofMinutes(1); + } + + @BeforeAll + public static void setup() throws Exception { + SharedMiniClusterBase.startMiniCluster(); + } + + @AfterAll + public static void teardown() { + SharedMiniClusterBase.stopMiniCluster(); + } + + @BeforeEach + public void beforeEach() { + client = Accumulo.newClient().from(getClientProps()).build(); + ops = client.tableOperations(); + } + + @AfterEach + public void afterEach() throws Exception { + // ensure none of the FATE or SCAN_REF data we created persists between tests. Also ensure the + // user table does not persist between tests + if (userTable != null && ops.exists(userTable)) { + cleanupFateTable(); + ops.delete(userTable); + } + cleanupScanRefTable(); + client.close(); + } + + @Test + public void testAllTested() { + var allTableOps = Arrays.stream(TableOperations.class.getDeclaredMethods()).map(Method::getName) + .collect(Collectors.toSet()); + var testMethodNames = Arrays.stream(ComprehensiveTableOperationsIT.class.getDeclaredMethods()) + .map(Method::getName).collect(Collectors.toSet()); + var allTableOpsIter = allTableOps.iterator(); + while (allTableOpsIter.hasNext()) { + var tableOp = allTableOpsIter.next(); + for (var testMethodName : testMethodNames) { + if (testMethodName.contains(tableOp)) { + allTableOpsIter.remove(); + break; + } + } + } + assertTrue(allTableOps.isEmpty(), "The table operations " + allTableOps + " are untested"); + } + + @Test + public void testExpectedSystemTables() { + var testedSystemTableIds = + Set.of(AccumuloTable.ROOT.tableId(), AccumuloTable.METADATA.tableId(), + AccumuloTable.FATE.tableId(), AccumuloTable.SCAN_REF.tableId()); + assertEquals(testedSystemTableIds, AccumuloTable.allTableIds(), + "There are newly added system tables that are untested in this IT. Ensure each " + + "test has testing for the new system table"); + } + + @Test + public void test_list_tableIdMap() throws Exception { + // Thoroughly tested elsewhere, but simple enough to test here. Test that all system tables + // and user tables are returned + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + var expected = Arrays.stream(AccumuloTable.values()).map(AccumuloTable::tableName) + .collect(Collectors.toSet()); + expected.add(userTable); + assertEquals(expected, ops.list()); + assertEquals(expected, ops.tableIdMap().keySet()); + } + + @Test + public void test_exists() throws Exception { + // Thoroughly tested elsewhere, but simple enough to test here. Test that all system tables + // and user tables exist + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + var expected = Arrays.stream(AccumuloTable.values()).map(AccumuloTable::tableName) + .collect(Collectors.toSet()); + expected.add(userTable); + for (String table : expected) { + assertTrue(ops.exists(table)); + } + } + + @Test + public void test_create() throws Exception { + // Creating user tables is thoroughly tested. Make sure we can't create any of the already + // existing system tables, though. + for (var systemTable : AccumuloTable.values()) { + assertThrows(AccumuloException.class, () -> ops.create(systemTable.tableName())); + } + } + + @Test + public void test_exportTable_importTable() throws Exception { + // exportTable, importTable for user tables is tested in ImportExportIT. Ensure test exists + assertDoesNotThrow(() -> Class.forName(ImportExportIT.class.getName())); + // exportTable, importTable untested for system tables. Test basic functionality here + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + Path baseDir = ImportExportIT.createBaseDir(getCluster(), getClass()); + var fs = getCluster().getFileSystem(); + + // export the user table, so we can test importing into the system tables + ops.offline(userTable, true); + Path exportUserDir; + exportUserDir = new Path(baseDir, "export_userdir"); + fs.deleteOnExit(exportUserDir); + ops.exportTable(userTable, exportUserDir.toString()); + + var sysTables = AccumuloTable.values(); + for (int i = 0; i < sysTables.length; i++) { + var sysTable = sysTables[i]; + Path exportDir = new Path(baseDir, "export" + i); + fs.deleteOnExit(exportDir); + + // not offline, can't export + assertThrows(IllegalStateException.class, + () -> ops.exportTable(sysTable.tableName(), exportDir.toString())); + + switch (sysTable) { + case ROOT: + case METADATA: + // can't offline, so will never be able to export + assertThrows(AccumuloException.class, () -> ops.offline(sysTable.tableName(), true)); + assertThrows(AccumuloException.class, + () -> ops.importTable(sysTable.tableName(), exportUserDir.toString())); + break; + case FATE: + case SCAN_REF: + ops.offline(sysTable.tableName(), true); + try { + assertThrows(AccumuloException.class, + () -> ops.exportTable(sysTable.tableName(), exportDir.toString())); + assertThrows(AccumuloException.class, + () -> ops.importTable(sysTable.tableName(), exportUserDir.toString())); + } finally { + ops.online(sysTable.tableName(), true); + } + break; + default: + throw new IllegalStateException("Unrecognized table: " + sysTable); + } + } + } + + @Test + public void test_addSplits_putSplits_listSplits_splitRangeByTablets() throws Exception { + // note that addSplits and putSplits are implemented the same, just take different args. No + // need to test both + + // addSplits, listSplits tested elsewhere for METADATA, ROOT, and user tables, but testing here + // as well since this setup is needed to test for splitRangeByTablets anyway, which is untested + // elsewhere + + userTable = getUniqueNames(1)[0]; + ops.create(userTable); + // system and user tables + var allTables = new ArrayList<>(ops.list()); Review Comment: Since this is already tested in `test_list_tableIdMap`, chose to avoid same checks elsewhere -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@accumulo.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
