This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 32f5b4cc69e54c0fe6a1b4ff2d1990999ab779d3 Merge: 78dc351005 ffe0018b51 Author: Dave Marion <[email protected]> AuthorDate: Fri Jul 7 15:37:30 2023 +0000 Merge branch 'main' into elasticity .../compactions/InternalCompactionExecutor.java | 1 + .../accumulo/tserver/tablet/CompactableImpl.java | 14 +-- .../tablet/CompactableImplFileManagerTest.java | 2 +- .../org/apache/accumulo/test/LargeSplitRowIT.java | 2 +- .../java/org/apache/accumulo/test/LocatorIT.java | 12 +-- .../java/org/apache/accumulo/test/TestIngest.java | 10 ++ .../test/compaction/ExternalCompaction4_IT.java | 120 +++++++++++++++++++++ .../compaction/ExternalCompactionTestUtils.java | 5 +- .../accumulo/test/conf/PropStoreConfigIT.java | 48 ++++----- .../accumulo/test/conf/util/ZooPropEditorIT.java | 37 ++++--- .../accumulo/test/functional/CompactionIT.java | 76 +++++++++++++ .../test/functional/ErrorThrowingIterator.java | 106 ++++++++++++++++++ .../test/functional/KerberosRenewalIT.java | 3 +- .../test/functional/ManagerAssignmentIT.java | 18 ++-- .../test/functional/MemoryStarvedMajCIT.java | 2 +- .../test/functional/MemoryStarvedScanIT.java | 12 +-- .../accumulo/test/functional/ReadWriteIT.java | 6 ++ .../apache/accumulo/test/shell/ShellServerIT.java | 8 +- .../java/org/apache/accumulo/test/util/Wait.java | 69 ++++++++++-- 19 files changed, 455 insertions(+), 96 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/LocatorIT.java index bf090bcb33,289a6f8557..f1878cbfb2 --- a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java @@@ -117,21 -106,6 +117,21 @@@ public class LocatorIT extends Accumulo HashSet<String> tservers = new HashSet<>(client.instanceOperations().getTabletServers()); + // locate won't find any locations, tablets are not hosted + ranges.add(r1); + assertThrows(AccumuloException.class, () -> tableOps.locate(tableName, ranges)); + + ranges.add(r2); + assertThrows(AccumuloException.class, () -> tableOps.locate(tableName, ranges)); + + ranges.clear(); + + tableOps.setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); - assertTrue(Wait.waitFor( ++ Wait.waitFor( + () -> alwaysHostedAndCurrentNotNull.test( + ManagerAssignmentIT.getManagerTabletInfo(client, tableId, null).getTabletMetadata()), - 60000, 250)); ++ 60000, 250); + ranges.add(r1); Locations ret = tableOps.locate(tableName, ranges); assertContains(ret, tservers, Map.of(r1, Set.of(t1)), Map.of(t1, Set.of(r1))); @@@ -145,14 -119,6 +145,14 @@@ splits.add(new Text("r")); tableOps.addSplits(tableName, splits); + // ELASTICITY_TODO split does not set hosting goal, so this throws exception + assertThrows(AccumuloException.class, () -> tableOps.locate(tableName, ranges)); + tableOps.setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); - assertTrue(Wait.waitFor( ++ Wait.waitFor( + () -> alwaysHostedAndCurrentNotNull.test( + ManagerAssignmentIT.getManagerTabletInfo(client, tableId, null).getTabletMetadata()), - 60000, 250)); ++ 60000, 250); + ret = tableOps.locate(tableName, ranges); assertContains(ret, tservers, Map.of(r1, Set.of(t2), r2, Set.of(t2, t3)), Map.of(t2, Set.of(r1, r2), t3, Set.of(r2))); @@@ -161,24 -127,6 +161,24 @@@ assertThrows(TableOfflineException.class, () -> tableOps.locate(tableName, ranges)); + tableOps.online(tableName, true); + + // ELASTICITY_TODO Split does not set hosting goal + tableOps.setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); + + // TabletGroupWatcher interval set to 5s + Thread.sleep(7000); + - assertTrue(Wait.waitFor( ++ Wait.waitFor( + () -> alwaysHostedAndCurrentNotNull.test(ManagerAssignmentIT + .getManagerTabletInfo(client, tableId, new Text("r")).getTabletMetadata()), - 60000, 250)); ++ 60000, 250); + + ArrayList<Range> ranges2 = new ArrayList<>(); + ranges2.add(r1); + ret = tableOps.locate(tableName, ranges2); + assertContains(ret, tservers, Map.of(r1, Set.of(t2)), Map.of(t2, Set.of(r1))); + tableOps.delete(tableName); assertThrows(TableNotFoundException.class, () -> tableOps.locate(tableName, ranges)); diff --cc test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java index 0000000000,12b0eb5286..3543f97810 mode 000000,100644..100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction4_IT.java @@@ -1,0 -1,135 +1,120 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.accumulo.test.compaction; + -import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1; + import static org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable; + import static org.junit.jupiter.api.Assertions.assertEquals; + import static org.junit.jupiter.api.Assertions.assertThrows; + + import java.util.EnumSet; + import java.util.NoSuchElementException; + -import org.apache.accumulo.compactor.Compactor; -import org.apache.accumulo.coordinator.CompactionCoordinator; + import org.apache.accumulo.core.client.Accumulo; + import org.apache.accumulo.core.client.AccumuloClient; + import org.apache.accumulo.core.client.IteratorSetting; + import org.apache.accumulo.core.client.admin.CompactionConfig; + import org.apache.accumulo.core.clientImpl.ClientContext; + import org.apache.accumulo.core.conf.Property; + import org.apache.accumulo.core.data.TableId; + import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; + import org.apache.accumulo.core.iterators.user.AgeOffFilter; + import org.apache.accumulo.core.metadata.schema.Ample; + import org.apache.accumulo.core.metadata.schema.TabletMetadata; + import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; + import org.apache.accumulo.core.metadata.schema.TabletsMetadata; + import org.apache.accumulo.harness.AccumuloClusterHarness; -import org.apache.accumulo.minicluster.ServerType; + import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; + import org.apache.accumulo.test.functional.ErrorThrowingIterator; + import org.apache.accumulo.test.functional.ReadWriteIT; + import org.apache.hadoop.conf.Configuration; + import org.junit.jupiter.api.Test; + + public class ExternalCompaction4_IT extends AccumuloClusterHarness { + + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) { + ExternalCompactionTestUtils.configureMiniCluster(cfg, coreSite); - cfg.setNumCompactors(2); + } + + @Test + public void testErrorDuringCompactionNoOutput() throws Exception { + final String table1 = this.getUniqueNames(1)[0]; + try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class); - getCluster().getClusterControl().startCompactors(Compactor.class, 1, QUEUE1); + createTable(client, table1, "cs1"); + client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.getKey(), "51"); + TableId tid = TableId.of(client.tableOperations().tableIdMap().get(table1)); + + ReadWriteIT.ingest(client, 50, 1, 1, 0, "colf", table1, 1); + ReadWriteIT.verify(client, 50, 1, 1, 0, table1); + + Ample ample = ((ClientContext) client).getAmple(); + TabletsMetadata tms = ample.readTablets().forTable(tid).fetch(ColumnType.FILES).build(); + TabletMetadata tm = tms.iterator().next(); + assertEquals(50, tm.getFiles().size()); + + IteratorSetting setting = new IteratorSetting(50, "ageoff", AgeOffFilter.class); + setting.addOption("ttl", "0"); + setting.addOption("currentTime", Long.toString(System.currentTimeMillis() + 86400)); + client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc)); + + // Since this iterator is on the top, it will throw an error 3 times, then allow the + // ageoff iterator to do its work. + IteratorSetting setting2 = new IteratorSetting(51, "error", ErrorThrowingIterator.class); + setting2.addOption(ErrorThrowingIterator.TIMES, "3"); + client.tableOperations().attachIterator(table1, setting2, EnumSet.of(IteratorScope.majc)); + client.tableOperations().compact(table1, new CompactionConfig().setWait(true)); + + assertThrows(NoSuchElementException.class, () -> ample.readTablets().forTable(tid) + .fetch(ColumnType.FILES).build().iterator().next()); + assertEquals(0, client.createScanner(table1).stream().count()); - } finally { - getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR); - getCluster().getClusterControl().stopAllServers(ServerType.COMPACTION_COORDINATOR); + } + } + + @Test + public void testErrorDuringUserCompaction() throws Exception { + final String table1 = this.getUniqueNames(1)[0]; + try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) { - getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class); - getCluster().getClusterControl().startCompactors(Compactor.class, 1, QUEUE1); + createTable(client, table1, "cs1"); + client.tableOperations().setProperty(table1, Property.TABLE_FILE_MAX.getKey(), "1001"); + client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.getKey(), "1001"); + TableId tid = TableId.of(client.tableOperations().tableIdMap().get(table1)); + + ReadWriteIT.ingest(client, 1000, 1, 1, 0, "colf", table1, 1); + + Ample ample = ((ClientContext) client).getAmple(); + TabletsMetadata tms = ample.readTablets().forTable(tid).fetch(ColumnType.FILES).build(); + TabletMetadata tm = tms.iterator().next(); + assertEquals(1000, tm.getFiles().size()); + + IteratorSetting setting = new IteratorSetting(50, "error", ErrorThrowingIterator.class); + setting.addOption(ErrorThrowingIterator.TIMES, "3"); + client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorScope.majc)); + client.tableOperations().compact(table1, new CompactionConfig().setWait(true)); + + tms = ample.readTablets().forTable(tid).fetch(ColumnType.FILES).build(); + tm = tms.iterator().next(); + assertEquals(1, tm.getFiles().size()); + + ReadWriteIT.verify(client, 1000, 1, 1, 0, table1); + - } finally { - getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR); - getCluster().getClusterControl().stopAllServers(ServerType.COMPACTION_COORDINATOR); + } + + } + + } diff --cc test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java index 26ad0c0f79,76c3f5c54c..0a3e35ae66 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompactionTestUtils.java @@@ -292,21 -298,6 +291,21 @@@ public class ExternalCompactionTestUtil return ecids; } + public static void waitForRunningCompactions(ServerContext ctx, TableId tid, + Set<ExternalCompactionId> idsToWaitFor) throws Exception { + - assertTrue(Wait.waitFor(() -> { ++ Wait.waitFor(() -> { + Set<ExternalCompactionId> seen; + try (TabletsMetadata tm = + ctx.getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) { + seen = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()) + .collect(Collectors.toSet()); + } + + return Collections.disjoint(seen, idsToWaitFor); - })); ++ }); + } + public static int confirmCompactionRunning(ServerContext ctx, Set<ExternalCompactionId> ecids) throws Exception { int matches = 0; diff --cc test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java index 12490a93d5,928810fbe1..298d321cce --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java @@@ -72,12 -69,15 +74,17 @@@ import org.apache.accumulo.core.iterato import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; + import org.apache.accumulo.core.iterators.user.AgeOffFilter; import org.apache.accumulo.core.iterators.user.GrepIterator; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.StoredTabletFile; + import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; + import org.apache.accumulo.core.metadata.schema.TabletMetadata; + import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; + import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.harness.AccumuloClusterHarness; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; diff --cc test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java index 8857f41659,c3fc72d242..4908655e9a --- a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java @@@ -154,171 -84,10 +154,171 @@@ public class ManagerAssignmentIT extend // put it back online c.tableOperations().online(tableName, true); - TabletLocationState online = getTabletLocationState(c, tableId); - assertNull(online.future); - assertNotNull(online.current); - assertEquals(online.getCurrentServer(), online.getLastServer()); + TabletMetadata online = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + assertTrue(online.hasCurrent()); + assertNotNull(online.getLocation()); + assertEquals(online.getLocation().getHostPort(), online.getLast().getHostPort()); + assertEquals(TabletHostingGoal.ONDEMAND, online.getHostingGoal()); + + // set the hosting goal to always + c.tableOperations().setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); + + Predicate<TabletMetadata> alwaysHostedOrCurrentNotNull = + t -> (t.getHostingGoal() == TabletHostingGoal.ALWAYS && t.hasCurrent()); + - assertTrue(Wait.waitFor(() -> alwaysHostedOrCurrentNotNull - .test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), 60000, 250)); ++ Wait.waitFor(() -> alwaysHostedOrCurrentNotNull ++ .test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), 60000, 250); + + final TabletMetadata always = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + assertTrue(alwaysHostedOrCurrentNotNull.test(always)); + assertTrue(always.hasCurrent()); + assertEquals(flushed.getLocation().getHostPort(), always.getLast().getHostPort()); + assertEquals(TabletHostingGoal.ALWAYS, always.getHostingGoal()); + + // set the hosting goal to never + c.tableOperations().setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.NEVER); + Predicate<TabletMetadata> neverHostedOrCurrentNull = + t -> (t.getHostingGoal() == TabletHostingGoal.NEVER && !t.hasCurrent()); - assertTrue(Wait.waitFor(() -> neverHostedOrCurrentNull - .test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), 60000, 250)); ++ Wait.waitFor(() -> neverHostedOrCurrentNull ++ .test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), 60000, 250); + + final TabletMetadata never = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + assertTrue(neverHostedOrCurrentNull.test(never)); + assertNull(never.getLocation()); + assertEquals(flushed.getLocation().getHostPort(), never.getLast().getHostPort()); + assertEquals(TabletHostingGoal.NEVER, never.getHostingGoal()); + + // set the hosting goal to ondemand + c.tableOperations().setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ONDEMAND); + Predicate<TabletMetadata> ondemandHosted = + t -> t.getHostingGoal() == TabletHostingGoal.ONDEMAND; - assertTrue(Wait.waitFor( ++ Wait.waitFor( + () -> ondemandHosted.test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), - 60000, 250)); ++ 60000, 250); + final TabletMetadata ondemand = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + assertTrue(ondemandHosted.test(ondemand)); + assertNull(ondemand.getLocation()); + assertEquals(flushed.getLocation().getHostPort(), ondemand.getLast().getHostPort()); + assertEquals(TabletHostingGoal.ONDEMAND, ondemand.getHostingGoal()); + } + } + + private String prepTableForScanTest(AccumuloClient c, String tableName) throws Exception { + + TreeSet<Text> splits = new TreeSet<>(); + splits.add(new Text("f")); + splits.add(new Text("m")); + splits.add(new Text("t")); + + NewTableConfiguration ntc = new NewTableConfiguration(); + ntc.withSplits(splits); + c.tableOperations().create(tableName, ntc); + String tableId = c.tableOperations().tableIdMap().get(tableName); + + // The initial set of tablets should be unassigned - assertTrue(Wait.waitFor(() -> getTabletStats(c, tableId).isEmpty(), 60000, 50)); ++ Wait.waitFor(() -> getTabletStats(c, tableId).isEmpty(), 60000, 50); + + assertEquals(0, ClientTabletCache.getInstance((ClientContext) c, TableId.of(tableId)) + .getTabletHostingRequestCount()); + + // loading data will force the tablets to be hosted + loadDataForScan(c, tableName); + + assertTrue(ClientTabletCache.getInstance((ClientContext) c, TableId.of(tableId)) + .getTabletHostingRequestCount() > 0); + - assertTrue(Wait.waitFor(() -> getTabletStats(c, tableId).size() == 4, 60000, 50)); ++ Wait.waitFor(() -> getTabletStats(c, tableId).size() == 4, 60000, 50); + + // offline table to force unassign tablets without having to wait for the tablet unloader + c.tableOperations().offline(tableName, true); + + c.tableOperations().clearLocatorCache(tableName); + + // online the table again, confirm still no tablets hosted + c.tableOperations().online(tableName, true); + - assertTrue(Wait.waitFor(() -> getTabletStats(c, tableId).isEmpty(), 60000, 50)); ++ Wait.waitFor(() -> getTabletStats(c, tableId).isEmpty(), 60000, 50); + assertEquals(0, ClientTabletCache.getInstance((ClientContext) c, TableId.of(tableId)) + .getTabletHostingRequestCount()); + + return tableId; + + } + + @Test + public void testScannerAssignsOneOnDemandTablets() throws Exception { + try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { + String tableName = super.getUniqueNames(1)[0]; + + String tableId = prepTableForScanTest(c, tableName); + + Range scanRange = new Range("a", "c"); + Scanner s = c.createScanner(tableName); + s.setRange(scanRange); + // Should return keys for a, b, c + assertEquals(3, Iterables.size(s)); + + List<TabletStats> stats = getTabletStats(c, tableId); + // There should be one tablet online + assertEquals(1, stats.size()); + assertTrue(ClientTabletCache.getInstance((ClientContext) c, TableId.of(tableId)) + .getTabletHostingRequestCount() > 0); + + } + } + + @Test + public void testScannerAssignsMultipleOnDemandTablets() throws Exception { + try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { + String tableName = super.getUniqueNames(1)[0]; + + String tableId = prepTableForScanTest(c, tableName); + + try (Scanner s = c.createScanner(tableName)) { + s.setRange(new Range("a", "s")); + assertEquals(19, Iterables.size(s)); + } + + List<TabletStats> stats = getTabletStats(c, tableId); + assertEquals(3, stats.size()); + long hostingRequestCount = ClientTabletCache + .getInstance((ClientContext) c, TableId.of(tableId)).getTabletHostingRequestCount(); + assertTrue(hostingRequestCount > 0); + + // Run another scan, all tablets should be loaded + try (Scanner s = c.createScanner(tableName)) { + s.setRange(new Range("a", "t")); + assertEquals(20, Iterables.size(s)); + } + + stats = getTabletStats(c, tableId); + assertEquals(3, stats.size()); + // No more tablets should have been brought online + assertEquals(hostingRequestCount, ClientTabletCache + .getInstance((ClientContext) c, TableId.of(tableId)).getTabletHostingRequestCount()); + + } + } + + @Test + public void testBatchScannerAssignsOneOnDemandTablets() throws Exception { + try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { + String tableName = super.getUniqueNames(1)[0]; + + String tableId = prepTableForScanTest(c, tableName); + + try (BatchScanner s = c.createBatchScanner(tableName)) { + s.setRanges(List.of(new Range("a", "c"))); + // Should return keys for a, b, c + assertEquals(3, Iterables.size(s)); + } + + List<TabletStats> stats = getTabletStats(c, tableId); + // There should be one tablet online + assertEquals(1, stats.size()); + assertTrue(ClientTabletCache.getInstance((ClientContext) c, TableId.of(tableId)) + .getTabletHostingRequestCount() > 0); + } }
