This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c49e8f6a5c fixes metadata compactions and adds tests (#3539)
c49e8f6a5c is described below
commit c49e8f6a5c50382fe261988a50977d0574ef09d9
Author: Keith Turner <[email protected]>
AuthorDate: Fri Jun 30 12:14:03 2023 -0400
fixes metadata compactions and adds tests (#3539)
There are three changes in this commit.
1. Compactions were not being initiated for the root tablet. This was
fixed
2. Configures an external compaction queue named 'accumulo_meta' for
the root and metadata table. This was set in default properties.
3. Added an IT that ensures the root and metadata table will compact
down to one file.
fixes #3463
---
.../org/apache/accumulo/core/conf/Property.java | 6 +-
.../server/manager/state/TabletStateStore.java | 5 +-
.../server/manager/state/ZooTabletStateStore.java | 23 +-
.../manager/state/RootTabletStateStoreTest.java | 311 ---------------------
.../manager/state/ZooTabletStateStoreTest.java | 63 +++++
.../accumulo/tserver/tablet/CompactableImpl.java | 3 +-
.../test/ManagerRepairsDualAssignmentIT.java | 4 +-
.../accumulo/test/functional/CompactionIT.java | 67 +++++
8 files changed, 157 insertions(+), 325 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 06bc8383c4..b9cd411847 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -570,8 +570,7 @@ public enum Property {
"The maximum number of files a compaction will open", "2.1.0"),
TSERV_COMPACTION_SERVICE_ROOT_EXECUTORS(
"tserver.compaction.major.service.root.planner.opts.executors",
-
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':1},{'name':'huge','type':'internal','numThreads':1}]"
- .replaceAll("'", "\""),
+
"[{'name':'all','type':'external','queue':'accumulo_meta'}]".replaceAll("'",
"\""),
PropertyType.STRING,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} ",
"2.1.0"),
@@ -588,8 +587,7 @@ public enum Property {
"The maximum number of files a compaction will open", "2.1.0"),
TSERV_COMPACTION_SERVICE_META_EXECUTORS(
"tserver.compaction.major.service.meta.planner.opts.executors",
-
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':2},{'name':'huge','type':'internal','numThreads':2}]"
- .replaceAll("'", "\""),
+
"[{'name':'all','type':'external','queue':'accumulo_meta'}]".replaceAll("'",
"\""),
PropertyType.STRING,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %} ",
"2.1.0"),
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
index 9b127b6220..6bba19ddac 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java
@@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.manager.state.TabletManagement;
import org.apache.accumulo.core.metadata.TServerInstance;
@@ -103,11 +102,11 @@ public interface TabletStateStore extends
Iterable<TabletManagement> {
return getStoreForLevel(DataLevel.of(extent.tableId()), context);
}
- public static TabletStateStore getStoreForLevel(DataLevel level,
ClientContext context) {
+ public static TabletStateStore getStoreForLevel(DataLevel level,
ServerContext context) {
return getStoreForLevel(level, context, null);
}
- public static TabletStateStore getStoreForLevel(DataLevel level,
ClientContext context,
+ public static TabletStateStore getStoreForLevel(DataLevel level,
ServerContext context,
CurrentState state) {
TabletStateStore tss;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
index 69eabaeb9f..7b9008249d 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java
@@ -19,11 +19,10 @@
package org.apache.accumulo.server.manager.state;
import java.util.Collection;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.manager.state.TabletManagement;
import
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction;
import org.apache.accumulo.core.metadata.RootTable;
@@ -31,6 +30,10 @@ import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.Ample.ReadConsistency;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.ServiceEnvironmentImpl;
+import org.apache.accumulo.server.compaction.CompactionJobGenerator;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,9 +42,11 @@ class ZooTabletStateStore extends AbstractTabletStateStore
implements TabletStat
private static final Logger log =
LoggerFactory.getLogger(ZooTabletStateStore.class);
private final Ample ample;
+ private final ServerContext ctx;
- ZooTabletStateStore(ClientContext context) {
+ ZooTabletStateStore(ServerContext context) {
super(context);
+ this.ctx = context;
this.ample = context.getAmple();
}
@@ -60,7 +65,17 @@ class ZooTabletStateStore extends AbstractTabletStateStore
implements TabletStat
public TabletManagement next() {
finished = true;
TabletMetadata tm = ample.readTablet(RootTable.EXTENT,
ReadConsistency.EVENTUAL);
- return new
TabletManagement(Set.of(ManagementAction.NEEDS_LOCATION_UPDATE), tm);
+
+ var actions = EnumSet.of(ManagementAction.NEEDS_LOCATION_UPDATE);
+
+ CompactionJobGenerator cjg = new CompactionJobGenerator(new
ServiceEnvironmentImpl(ctx));
+ var jobs = cjg.generateJobs(tm,
+ EnumSet.of(CompactionKind.SYSTEM, CompactionKind.USER,
CompactionKind.SELECTOR));
+ if (!jobs.isEmpty()) {
+ actions.add(ManagementAction.NEEDS_COMPACTING);
+ }
+
+ return new TabletManagement(actions, tm);
}
@Override
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
deleted file mode 100644
index 55f7cc2779..0000000000
---
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.manager.state;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static
org.apache.accumulo.server.init.ZooKeeperInitializer.getInitialRootTabletJson;
-import static org.easymock.EasyMock.expect;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.OptionalLong;
-import java.util.Set;
-import java.util.SortedMap;
-
-import org.apache.accumulo.core.client.ConditionalWriter;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.admin.TabletHostingGoal;
-import org.apache.accumulo.core.data.ConditionalMutation;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.manager.state.TabletManagement;
-import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.core.metadata.StoredTabletFile;
-import org.apache.accumulo.core.metadata.SuspendingTServer;
-import org.apache.accumulo.core.metadata.TServerInstance;
-import org.apache.accumulo.core.metadata.TabletState;
-import org.apache.accumulo.core.metadata.schema.Ample;
-import org.apache.accumulo.core.metadata.schema.DataFileValue;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
-import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
-import org.apache.accumulo.core.metadata.schema.MetadataTime;
-import org.apache.accumulo.core.metadata.schema.RootTabletMetadata;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
-import org.apache.accumulo.core.metadata.schema.TabletOperationId;
-import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
-import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.server.MockServerContext;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl;
-import org.apache.hadoop.io.Text;
-import org.easymock.EasyMock;
-import org.junit.jupiter.api.Test;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-import com.google.common.net.HostAndPort;
-
-public class RootTabletStateStoreTest {
-
- private static class BrokenTabletMetadata extends TabletMetadata {
- private final TabletMetadata tm;
-
- BrokenTabletMetadata(TabletMetadata tm) {
- this.tm = tm;
- }
-
- public boolean equals(Object obj) {
- return tm.equals(obj);
- }
-
- public TableId getTableId() {
- return TableId.of("0");
- }
-
- public KeyExtent getExtent() {
- return new KeyExtent(TableId.of("0"), null, null);
- }
-
- public Text getPrevEndRow() {
- return tm.getPrevEndRow();
- }
-
- public boolean sawPrevEndRow() {
- return tm.sawPrevEndRow();
- }
-
- public Text getOldPrevEndRow() {
- return tm.getOldPrevEndRow();
- }
-
- public boolean sawOldPrevEndRow() {
- return tm.sawOldPrevEndRow();
- }
-
- public Text getEndRow() {
- return tm.getEndRow();
- }
-
- public Location getLocation() {
- return tm.getLocation();
- }
-
- public boolean hasCurrent() {
- return tm.hasCurrent();
- }
-
- public Map<StoredTabletFile,Long> getLoaded() {
- return tm.getLoaded();
- }
-
- public Location getLast() {
- return tm.getLast();
- }
-
- public SuspendingTServer getSuspend() {
- return tm.getSuspend();
- }
-
- public Set<StoredTabletFile> getFiles() {
- return tm.getFiles();
- }
-
- public Map<StoredTabletFile,DataFileValue> getFilesMap() {
- return tm.getFilesMap();
- }
-
- public Collection<LogEntry> getLogs() {
- return tm.getLogs();
- }
-
- public List<StoredTabletFile> getScans() {
- return tm.getScans();
- }
-
- public String getDirName() {
- return tm.getDirName();
- }
-
- public MetadataTime getTime() {
- return tm.getTime();
- }
-
- public String getCloned() {
- return tm.getCloned();
- }
-
- public OptionalLong getFlushId() {
- return tm.getFlushId();
- }
-
- public OptionalLong getCompactId() {
- return tm.getCompactId();
- }
-
- public Double getSplitRatio() {
- return tm.getSplitRatio();
- }
-
- public boolean hasChopped() {
- return tm.hasChopped();
- }
-
- public TabletHostingGoal getHostingGoal() {
- return tm.getHostingGoal();
- }
-
- public boolean getHostingRequested() {
- return tm.getHostingRequested();
- }
-
- public SortedMap<Key,Value> getKeyValues() {
- return tm.getKeyValues();
- }
-
- public TabletState getTabletState(Set<TServerInstance> liveTServers) {
- return tm.getTabletState(liveTServers);
- }
-
- public Map<ExternalCompactionId,ExternalCompactionMetadata>
getExternalCompactions() {
- return tm.getExternalCompactions();
- }
-
- public TabletOperationId getOperationId() {
- return tm.getOperationId();
- }
-
- public int hashCode() {
- return tm.hashCode();
- }
-
- public String toString() {
- return tm.toString();
- }
- }
-
- private static class TestAmple implements Ample {
-
- private String json =
- new String(getInitialRootTabletJson("dir",
"hdfs://nn/acc/tables/some/dir/0000.rf"), UTF_8);
-
- @Override
- public TabletMetadata readTablet(KeyExtent extent, ReadConsistency rc,
- ColumnType... colsToFetch) {
- Preconditions.checkArgument(extent.equals(RootTable.EXTENT));
- return new RootTabletMetadata(json).toTabletMetadata();
- }
-
- @Override
- public TabletsMetadata.TableOptions readTablets() {
- throw new UnsupportedOperationException("This method should be
implemented in subclasses");
- }
-
- public ConditionalTabletsMutator conditionallyMutateTablets() {
- return new ConditionalTabletsMutatorImpl(null) {
- protected ConditionalWriter createConditionalWriter(Ample.DataLevel
dataLevel)
- throws TableNotFoundException {
- Preconditions.checkArgument(dataLevel == DataLevel.ROOT);
- return new ConditionalWriter() {
- @Override
- public Iterator<Result> write(Iterator<ConditionalMutation>
mutations) {
- return Iterators.transform(mutations, this::write);
- }
-
- @Override
- public Result write(ConditionalMutation mutation) {
- var rtm = new RootTabletMetadata(json);
- rtm.update(mutation);
- json = rtm.toJson();
- return new Result(Status.ACCEPTED, mutation, "server");
- }
-
- @Override
- public void close() {
-
- }
- };
- }
- };
- }
-
- }
-
- @Test
- public void testRootTabletStateStore() throws DistributedStoreException {
- ServerContext context = MockServerContext.get();
- expect(context.getAmple()).andReturn(new TestAmple()).anyTimes();
- EasyMock.replay(context);
- ZooTabletStateStore tstore = new ZooTabletStateStore(context);
- KeyExtent root = RootTable.EXTENT;
- String sessionId = "this is my unique session data";
- TServerInstance server =
- new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000),
sessionId);
- List<Assignment> assignments = Collections.singletonList(new
Assignment(root, server, null));
- tstore.setFutureLocations(assignments);
- int count = 0;
- for (TabletManagement mti : tstore) {
- assertEquals(mti.getTabletMetadata().getExtent(), root);
-
assertTrue(mti.getTabletMetadata().getLocation().getType().equals(LocationType.FUTURE));
- assertEquals(mti.getTabletMetadata().getLocation().getServerInstance(),
server);
- assertFalse(mti.getTabletMetadata().hasCurrent());
- count++;
- }
- assertEquals(count, 1);
- tstore.setLocations(assignments);
- count = 0;
- for (TabletManagement mti : tstore) {
- assertEquals(mti.getTabletMetadata().getExtent(), root);
-
assertFalse(mti.getTabletMetadata().getLocation().getType().equals(LocationType.FUTURE));
- assertTrue(mti.getTabletMetadata().hasCurrent());
- assertEquals(mti.getTabletMetadata().getLocation().getServerInstance(),
server);
- count++;
- }
- assertEquals(count, 1);
- TabletManagement rootTabletMetadataInfo = tstore.iterator().next();
-
tstore.unassign(Collections.singletonList(rootTabletMetadataInfo.getTabletMetadata()),
null);
- count = 0;
- for (TabletManagement mti : tstore) {
- assertEquals(mti.getTabletMetadata().getExtent(), root);
- assertFalse(mti.getTabletMetadata().hasCurrent());
- assertNull(mti.getTabletMetadata().getLocation());
- count++;
- }
- assertEquals(count, 1);
-
- KeyExtent notRoot = new KeyExtent(TableId.of("0"), null, null);
- final var assignmentList = List.of(new Assignment(notRoot, server, null));
- assertThrows(IllegalArgumentException.class, () ->
tstore.setLocations(assignmentList));
- assertThrows(IllegalArgumentException.class, () ->
tstore.setFutureLocations(assignmentList));
- final List<TabletMetadata> assignmentList1 =
- List.of(new
BrokenTabletMetadata(rootTabletMetadataInfo.getTabletMetadata()));
- assertThrows(IllegalArgumentException.class, () ->
tstore.unassign(assignmentList1, null));
- }
-}
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/manager/state/ZooTabletStateStoreTest.java
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/ZooTabletStateStoreTest.java
new file mode 100644
index 0000000000..c6023246c2
--- /dev/null
+++
b/server/base/src/test/java/org/apache/accumulo/server/manager/state/ZooTabletStateStoreTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.manager.state;
+
+import static org.easymock.EasyMock.expect;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.List;
+
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.server.MockServerContext;
+import org.apache.accumulo.server.ServerContext;
+import org.easymock.EasyMock;
+import org.junit.jupiter.api.Test;
+
+import com.google.common.net.HostAndPort;
+
+public class ZooTabletStateStoreTest {
+
+ @Test
+ public void testZooTabletStateStore() throws DistributedStoreException {
+ ServerContext context = MockServerContext.get();
+ Ample ample = EasyMock.createMock(Ample.class);
+ expect(context.getAmple()).andReturn(ample).anyTimes();
+ EasyMock.replay(context, ample);
+ ZooTabletStateStore tstore = new ZooTabletStateStore(context);
+
+ String sessionId = "this is my unique session data";
+ TServerInstance server =
+ new TServerInstance(HostAndPort.fromParts("127.0.0.1", 10000),
sessionId);
+
+ KeyExtent notRoot = new KeyExtent(TableId.of("0"), null, null);
+ final var assignmentList = List.of(new Assignment(notRoot, server, null));
+ assertThrows(IllegalArgumentException.class, () ->
tstore.setLocations(assignmentList));
+ assertThrows(IllegalArgumentException.class, () ->
tstore.setFutureLocations(assignmentList));
+
+ var nonRootMeta = TabletMetadata.builder(new
KeyExtent(TableId.of("notroot"), null, null))
+ .putPrevEndRow(null).build();
+
+ final List<TabletMetadata> assignmentList1 = List.of(nonRootMeta);
+ assertThrows(IllegalArgumentException.class, () ->
tstore.unassign(assignmentList1, null));
+ }
+}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 539bc26f11..ec3ed5d71c 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -1089,7 +1089,8 @@ public class CompactableImpl implements Compactable {
return Optional.empty();
}
- if (!getExtent().isRootTablet() && !getExtent().isMeta() && kind ==
CompactionKind.SYSTEM) {
+ if (kind == CompactionKind.SYSTEM || kind == CompactionKind.USER
+ || kind == CompactionKind.SELECTOR || kind == CompactionKind.CHOP) {
// ELASTICITY_TODO a hack added to disable system compactions for user
tablets
return Optional.empty();
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java
b/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java
index c0f7c168e1..249c7072f9 100644
---
a/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java
@@ -88,7 +88,7 @@ public class ManagerRepairsDualAssignmentIT extends
ConfigurableMacBase {
// scan the metadata table and get the two table location states
Set<TabletMetadata.Location> states = new HashSet<>();
Set<TabletMetadata> oldLocations = new HashSet<>();
- TabletStateStore store =
TabletStateStore.getStoreForLevel(DataLevel.USER, context);
+ TabletStateStore store =
TabletStateStore.getStoreForLevel(DataLevel.USER, serverContext);
while (states.size() < 2) {
UtilWaitThread.sleep(250);
oldLocations.clear();
@@ -140,7 +140,7 @@ public class ManagerRepairsDualAssignmentIT extends
ConfigurableMacBase {
serverContext.getAmple().mutateTablet(new
KeyExtent(MetadataTable.ID, null, null));
tabletMutator.putLocation(moved.getLocation());
tabletMutator.mutate();
- waitForCleanStore(TabletStateStore.getStoreForLevel(DataLevel.METADATA,
context));
+ waitForCleanStore(TabletStateStore.getStoreForLevel(DataLevel.METADATA,
serverContext));
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index 50bea6c05e..0cb24e691f 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@ -19,6 +19,8 @@
package org.apache.accumulo.test.functional;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -66,6 +68,8 @@ import
org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.GrepIterator;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
@@ -75,6 +79,7 @@ import org.apache.accumulo.test.VerifyIngest;
import org.apache.accumulo.test.VerifyIngest.VerifyParams;
import org.apache.accumulo.test.compaction.CompactionExecutorIT;
import org.apache.accumulo.test.compaction.ExternalCompaction_1_IT.FSelector;
+import org.apache.accumulo.test.util.Wait;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -622,6 +627,68 @@ public class CompactionIT extends AccumuloClusterHarness {
}
}
+ @Test
+ public void testMetadataCompactions() throws Exception {
+ // The metadata and root table have default config that causes them to
compact down to one
+ // tablet. This test verifies that both tables compact to one file after a
flush.
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String[] tableNames = getUniqueNames(2);
+
+ // creating a user table should cause a write to the metadata table
+ c.tableOperations().create(tableNames[0]);
+
+ var mfiles1 =
getServerContext().getAmple().readTablets().forTable(MetadataTable.ID).build()
+ .iterator().next().getFiles();
+ var rootFiles1 =
getServerContext().getAmple().readTablet(RootTable.EXTENT).getFiles();
+
+ log.debug("mfiles1 {}",
+
mfiles1.stream().map(StoredTabletFile::getFileName).collect(toList()));
+ log.debug("rootFiles1 {}",
+
rootFiles1.stream().map(StoredTabletFile::getFileName).collect(toList()));
+
+ c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ c.tableOperations().flush(RootTable.NAME, null, null, true);
+
+ // create another table to cause more metadata writes
+ c.tableOperations().create(tableNames[1]);
+ try (var writer = c.createBatchWriter(tableNames[1])) {
+ var m = new Mutation("r1");
+ m.put("f1", "q1", "v1");
+ writer.addMutation(m);
+ }
+ c.tableOperations().flush(tableNames[1], null, null, true);
+
+ // create another metadata file
+ c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ c.tableOperations().flush(RootTable.NAME, null, null, true);
+
+ // The multiple flushes should create multiple files. We expect the file
sets to changes and
+ // eventually equal one.
+
+ Wait.waitFor(() -> {
+ var mfiles2 =
getServerContext().getAmple().readTablets().forTable(MetadataTable.ID).build()
+ .iterator().next().getFiles();
+ log.debug("mfiles2 {}",
+
mfiles2.stream().map(StoredTabletFile::getFileName).collect(toList()));
+ return mfiles2.size() == 1 && !mfiles2.equals(mfiles1);
+ });
+
+ Wait.waitFor(() -> {
+ var rootFiles2 =
getServerContext().getAmple().readTablet(RootTable.EXTENT).getFiles();
+ log.debug("rootFiles2 {}",
+
rootFiles2.stream().map(StoredTabletFile::getFileName).collect(toList()));
+ return rootFiles2.size() == 1 && !rootFiles2.equals(rootFiles1);
+ });
+
+ var entries = c.createScanner(tableNames[1]).stream()
+ .map(e -> e.getKey().getRow() + ":" + e.getKey().getColumnFamily() +
":"
+ + e.getKey().getColumnQualifier() + ":" + e.getValue())
+ .collect(toSet());
+
+ assertEquals(Set.of("r1:f1:q1:v1"), entries);
+ }
+ }
+
private int countFiles(AccumuloClient c) throws Exception {
try (Scanner s = c.createScanner(MetadataTable.NAME,
Authorizations.EMPTY)) {
s.fetchColumnFamily(new Text(TabletColumnFamily.NAME));
