(accumulo) branch elasticity updated: Fixed check in TabletResourceGroupBalanceIT to use Wait.waitFor (#4479)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new ee04ad3aa5 Fixed check in TabletResourceGroupBalanceIT to use
Wait.waitFor (#4479)
ee04ad3aa5 is described below
commit ee04ad3aa522404cb37a1eceefd2bec34b2d00e7
Author: Dave Marion
AuthorDate: Tue Apr 23 07:28:56 2024 -0400
Fixed check in TabletResourceGroupBalanceIT to use Wait.waitFor (#4479)
The assertion in testResourceGroupBalanceWithNoTServers started returning
zero for the number of hosted tablets after the `waitForBalance`. Not sure
which modification caused this to change behavior, but this fix is likely
the correct one regardless.
---
.../functional/TabletResourceGroupBalanceIT.java | 30 --
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
index 6b289a72b4..0605a69c2a 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletResourceGroupBalanceIT.java
@@ -225,20 +225,22 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
.addTabletServerResourceGroup("GROUP2", 1);
getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
- client.instanceOperations().waitForBalance();
- assertEquals(26, getCountOfHostedTablets(client, tableName));
- ingest.join();
- assertNull(error.get());
-
- client.tableOperations().delete(tableName);
- // Stop all tablet servers because there is no way to just stop
- // the GROUP2 server yet.
-
getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
-
getCluster().getConfig().getClusterServerConfiguration().clearTServerResourceGroups();
- getCluster().getConfig().getClusterServerConfiguration()
- .addTabletServerResourceGroup("GROUP1", 1);
- getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
+ try {
+client.instanceOperations().waitForBalance();
+Wait.waitFor(() -> getCountOfHostedTablets(client, tableName) == 26);
+ingest.join();
+assertNull(error.get());
+ } finally {
+client.tableOperations().delete(tableName);
+// Stop all tablet servers because there is no way to just stop
+// the GROUP2 server yet.
+
getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+
getCluster().getConfig().getClusterServerConfiguration().clearTServerResourceGroups();
+getCluster().getConfig().getClusterServerConfiguration()
+.addTabletServerResourceGroup("GROUP1", 1);
+getCluster().getClusterControl().start(ServerType.TABLET_SERVER);
+ }
}
}
@@ -258,7 +260,7 @@ public class TabletResourceGroupBalanceIT extends
SharedMiniClusterBase {
client.tableOperations().create(tableName, ntc1);
// wait for all tablets to be hosted
- Wait.waitFor(() -> 26 != getCountOfHostedTablets(client, tableName));
+ Wait.waitFor(() -> 26 == getCountOfHostedTablets(client, tableName));
client.instanceOperations().waitForBalance();
(accumulo) branch elasticity updated: Allow tablet refresh while in the process of closing (#4483)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 45653be366 Allow tablet refresh while in the process of closing (#4483)
45653be366 is described below
commit 45653be36678c9f4fa970e7db819ad67c6b02113
Author: Keith Turner
AuthorDate: Tue Apr 23 07:27:47 2024 -0400
Allow tablet refresh while in the process of closing (#4483)
There was check in the tablet refresh code that was preventing tablet
refresh while a tablet was in the middle of closing. Modified the check
to only prevent refresh after a tablet is competely closed.
fixes #4477
---
.../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 348807f161..38cd3cff18 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1635,7 +1635,7 @@ public class Tablet extends TabletBase {
}
synchronized (this) {
-if (isClosed()) {
+if (isCloseComplete()) {
log.debug("Unable to refresh tablet {} for {} because the tablet is
closed", extent,
refreshPurpose);
return false;
(accumulo) branch elasticity updated: Log warning when no Compactors for system tables (#4464)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 0039e5fa75 Log warning when no Compactors for system tables (#4464)
0039e5fa75 is described below
commit 0039e5fa75fa78af4d6e17488f5ff82ac96062b5
Author: Dave Marion
AuthorDate: Wed Apr 17 08:37:48 2024 -0400
Log warning when no Compactors for system tables (#4464)
Added logic in the SimpleGarbageCollector that will
periodically log a warning when there are no compactors
running for the system tables' resource group.
Fixes #4318
---
.../apache/accumulo/gc/SimpleGarbageCollector.java | 31 ++
1 file changed, 31 insertions(+)
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 8f9d406488..ec1f623377 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -23,6 +23,11 @@ import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterrup
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
@@ -33,6 +38,7 @@ import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.clientImpl.thrift.TInfo;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.gc.thrift.GCMonitorService.Iface;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
@@ -45,12 +51,16 @@ import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
+import org.apache.accumulo.core.spi.balancer.TableLoadBalancer;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.Halt;
+import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
import org.apache.accumulo.core.util.threads.ThreadPools;
+import org.apache.accumulo.core.util.time.NanoTime;
import org.apache.accumulo.gc.metrics.GcCycleMetrics;
import org.apache.accumulo.gc.metrics.GcMetrics;
import org.apache.accumulo.server.AbstractServer;
+import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.manager.LiveTServerSet;
import org.apache.accumulo.server.rpc.ServerAddress;
@@ -79,6 +89,8 @@ public class SimpleGarbageCollector extends AbstractServer
implements Iface {
private final GcCycleMetrics gcCycleMetrics = new GcCycleMetrics();
+ private NanoTime lastCompactorCheck = NanoTime.now();
+
SimpleGarbageCollector(ConfigOpts opts, String[] args) {
super("gc", opts, args);
@@ -297,6 +309,25 @@ public class SimpleGarbageCollector extends AbstractServer
implements Iface {
gcCycleMetrics.incrementRunCycleCount();
long gcDelay =
getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
+
+if (NanoTime.now().subtract(lastCompactorCheck).toMillis() > gcDelay *
3) {
+ Map> resourceMapping = new HashMap<>();
+ for (TableId tid : AccumuloTable.allTableIds()) {
+TableConfiguration tconf = getContext().getTableConfiguration(tid);
+String resourceGroup =
tconf.get(TableLoadBalancer.TABLE_ASSIGNMENT_GROUP_PROPERTY);
+resourceGroup =
+resourceGroup == null ? Constants.DEFAULT_RESOURCE_GROUP_NAME
: resourceGroup;
+resourceMapping.getOrDefault(resourceGroup, new
HashSet<>()).add(tid);
+ }
+ for (Entry> e : resourceMapping.entrySet()) {
+if (ExternalCompactionUtil.countCompactors(e.getKey(),
getContext()) == 0) {
+ log.warn("No Compactors exist in resource group {} for system
table {}", e.getKey(),
+ e.getValue());
+}
+ }
+ lastCompactorCheck = NanoTime.now();
+}
+
log.debug("Sleeping for {} milliseconds", gcDelay);
Thread.sleep(gcDelay);
} catch (InterruptedException e) {
(accumulo) branch elasticity updated (93c3418cea -> 8c68e4732e)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 93c3418cea Logged msg instead of throwing exception in setFutureLocations (#4442) add 8c68e4732e Fixed NPE in CompactionDriver when null passed to Text constructor (#4443) No new revisions were added by this update. Summary of changes: .../org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
(accumulo) branch elasticity updated (9503ebe22f -> 93c3418cea)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9503ebe22f verifies tablets are seen by compaction driver (#4434) add 93c3418cea Logged msg instead of throwing exception in setFutureLocations (#4442) No new revisions were added by this update. Summary of changes: .../server/manager/state/AbstractTabletStateStore.java | 16 1 file changed, 12 insertions(+), 4 deletions(-)
(accumulo) branch elasticity updated: Moved removeUnusedWALEntries to Tablet (#4404)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 193c010d36 Moved removeUnusedWALEntries to Tablet (#4404)
193c010d36 is described below
commit 193c010d36732b3a9a1cd602afad225498b6ac54
Author: Dave Marion
AuthorDate: Mon Apr 8 10:58:42 2024 -0400
Moved removeUnusedWALEntries to Tablet (#4404)
Moved MetadataTableUtil.removeUnusedWALEntries code to
Tablet constructor. Changed logic to use conditional mutations.
---
.../accumulo/server/util/MetadataTableUtil.java| 9 --
.../org/apache/accumulo/tserver/tablet/Tablet.java | 35 +++---
2 files changed, 24 insertions(+), 20 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index f87805d092..e5db26344f 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -31,7 +31,6 @@ import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -244,14 +243,6 @@ public class MetadataTableUtil {
return new Pair<>(result, sizes);
}
- public static void removeUnusedWALEntries(ServerContext context, KeyExtent
extent,
- final Collection entries, ServiceLock zooLock) {
-TabletMutator tablet = context.getAmple().mutateTablet(extent);
-entries.forEach(tablet::deleteWal);
-tablet.putZooLock(context.getZooKeeperRoot(), zooLock);
-tablet.mutate();
- }
-
private static Mutation createCloneMutation(TableId srcTableId, TableId
tableId,
Map tablet) {
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index a0f8b95857..414fedd105 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -59,12 +59,17 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FilePrefix;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.Ample;
+import org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult;
+import org.apache.accumulo.core.metadata.schema.Ample.ConditionalResult.Status;
+import org.apache.accumulo.core.metadata.schema.Ample.ConditionalTabletMutator;
+import
org.apache.accumulo.core.metadata.schema.Ample.ConditionalTabletsMutator;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
@@ -85,7 +90,6 @@ import org.apache.accumulo.server.problems.ProblemType;
import
org.apache.accumulo.server.tablets.ConditionCheckerContext.ConditionChecker;
import org.apache.accumulo.server.tablets.TabletNameGenerator;
import org.apache.accumulo.server.tablets.TabletTime;
-import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.tserver.InMemoryMap;
import org.apache.accumulo.tserver.MinorCompactionReason;
import org.apache.accumulo.tserver.TabletServer;
@@ -126,7 +130,6 @@ public class Tablet extends TabletBase {
private final TabletTime tabletTime;
- private Location lastLocation = null;
private final Set checkedTabletDirs = new ConcurrentSkipListSet<>();
private final AtomicLong dataSourceDeletions = new AtomicLong(0);
@@ -231,10 +234,6 @@ public class Tablet extends TabletBase {
this.tabletServer = tabletServer;
this.tabletResources = trm;
this.latestMetadata = metadata;
-
-// TODO look into this.. also last could be null
-this.lastLocation = metadata.getLast();
-
this.tabletTime = TabletTime.getInstance(metadata.getTime());
this.logId = tabletServer.createLogId();
@@ -277,10 +276,25 @@ public class Tablet extends TabletBase {
commitSession.updateMaxCommittedTime(tabletTime.getTime());
if (entriesUsedOnTable
(accumulo) branch elasticity updated: Existing logging in ManagerClientServiceHandler is sufficient, removed todo (#4433)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 77e95f95b9 Existing logging in ManagerClientServiceHandler is
sufficient, removed todo (#4433)
77e95f95b9 is described below
commit 77e95f95b9b3b76c42f5b619937dbcb58553f029
Author: Dave Marion
AuthorDate: Fri Apr 5 16:28:31 2024 -0400
Existing logging in ManagerClientServiceHandler is sufficient, removed todo
(#4433)
---
.../java/org/apache/accumulo/manager/ManagerClientServiceHandler.java| 1 -
1 file changed, 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
index 9a1fdaf478..dd14024db6 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/ManagerClientServiceHandler.java
@@ -648,7 +648,6 @@ public class ManagerClientServiceHandler implements
ManagerClientService.Iface {
inProgress.forEach(hostingRequestInProgress::remove);
}
-// ELASTICITY_TODO pass ranges of individual tablets
manager.getEventCoordinator().event(success, "Tablet hosting requested for
%d tablets in %s",
success.size(), tableId);
}
(accumulo) branch elasticity updated: Removed FileUtil.cleanupIndexOp to resolve TODO, related changes (#4385)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new b6769788c0 Removed FileUtil.cleanupIndexOp to resolve TODO, related
changes (#4385)
b6769788c0 is described below
commit b6769788c054407bc1ee269758684a86b8ca6e10
Author: Dave Marion
AuthorDate: Fri Apr 5 16:12:51 2024 -0400
Removed FileUtil.cleanupIndexOp to resolve TODO, related changes (#4385)
The existing TODO in FileUtil was to determine if the split code
in elasticity was missing something. The cleanupIndexOp method was
called in earlier versions, but is no longer called in elasticity.
I determined that the SplitUtils.IndexIterable.close method was a
likely replacement for the cleanupIndexOp method. I removed this
method and FileUtilTest as it was only testing this method. The
remaining method in FileUtil is only called from Splitter, so I
moved the method, related code, and associated test. I also fixed
up references that were broken due to the code move.
---
.../org/apache/accumulo/server/util/FileUtil.java | 135
.../apache/accumulo/server/util/FileUtilTest.java | 176 -
.../apache/accumulo/manager/split/Splitter.java| 81 +-
.../manager/tableOps/split/UpdateTablets.java | 6 +-
.../manager/upgrade/SplitRecovery12to13.java | 6 +-
.../manager/tableOps/split}/FileInfoTest.java | 4 +-
.../manager/tableOps/split/UpdateTabletsTest.java | 5 +-
7 files changed, 88 insertions(+), 325 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
deleted file mode 100644
index 78a541ca6e..00
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.server.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.accumulo.core.file.FileOperations;
-import org.apache.accumulo.core.file.FileSKVIterator;
-import org.apache.accumulo.core.metadata.TabletFile;
-import org.apache.accumulo.server.ServerContext;
-import org.apache.accumulo.server.conf.TableConfiguration;
-import org.apache.accumulo.server.fs.VolumeManager;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FileUtil {
-
- public static class FileInfo {
-final Text firstRow;
-final Text lastRow;
-
-public FileInfo(Text firstRow, Text lastRow) {
- this.firstRow = firstRow;
- this.lastRow = lastRow;
-}
-
-public Text getFirstRow() {
- return firstRow;
-}
-
-public Text getLastRow() {
- return lastRow;
-}
- }
-
- private static final Logger log = LoggerFactory.getLogger(FileUtil.class);
-
- // ELASTICITY_TODO this is only used by test. Determine what the test are
doing and if some
- // functionality is missing in the new split code.
- protected static void cleanupIndexOp(Path tmpDir, VolumeManager fs,
- ArrayList readers) throws IOException {
-// close all of the index sequence files
-for (FileSKVIterator r : readers) {
- try {
-if (r != null) {
- r.close();
-}
- } catch (IOException e) {
-// okay, try to close the rest anyway
-log.error("{}", e.getMessage(), e);
- }
-}
-
-if (tmpDir != null) {
- FileSystem actualFs = fs.getFileSystemByPath(tmpDir);
- if (actualFs.exists(tmpDir)) {
-fs.deleteRecursively(tmpDir);
-return;
- }
-
- log.error("Did not delete tmp dir because it wasn't a tmp dir {}",
tmpDir);
-}
- }
-
- public static Map
tryToGetFirstAndLastRows(
- ServerContext context, TableConfiguration ta
(accumulo) branch elasticity updated: Removed TODO in TabletRefresher (#4417)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 995f318459 Removed TODO in TabletRefresher (#4417)
995f318459 is described below
commit 995f318459efe6e1c3b29cf8fc9c0a27302f04dc
Author: Dave Marion
AuthorDate: Fri Apr 5 16:13:40 2024 -0400
Removed TODO in TabletRefresher (#4417)
Other exceptions which could be more serious than TException would
be raised as an ExecutionException when get() is called on the Future
and would result in a RuntimeException being raised from
TabletRefresher.refreshTablets. If a TException is thrown, then
the refresh for all of the Tablets will be retried unless the
TabletServer is no longer online.
---
.../org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java | 3 ---
1 file changed, 3 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
index a3d341a12b..cb963e583a 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/TabletRefresher.java
@@ -171,9 +171,6 @@ public class TabletRefresher {
} catch (TException ex) {
log.debug("rpc failed server: " + location + ", " + logId + " " +
ex.getMessage(), ex);
- // ELASTICITY_TODO are there any other exceptions we should catch in
this method and check if
- // the tserver is till alive?
-
// something went wrong w/ RPC return all extents as unrefreshed
return refreshes;
} finally {
(accumulo) branch elasticity updated (637dd0fd3f -> eea8ce48e2)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 637dd0fd3f Removed elasticity comment to do in ExternalCompaction_2_IT (#4416) add 9d4d68b2a3 Changed return variable for ExternalCompactionUtil.getCompactorAddrs (#4419) new eea8ce48e2 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/core/clientImpl/InstanceOperationsImpl.java | 2 +- .../accumulo/core/util/compaction/ExternalCompactionUtil.java | 7 --- .../src/main/java/org/apache/accumulo/monitor/Monitor.java| 2 +- .../monitor/rest/compactions/external/CoordinatorInfo.java| 4 ++-- .../monitor/rest/compactions/external/ExternalCompactionInfo.java | 8 .../org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java | 6 -- 6 files changed, 16 insertions(+), 13 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit eea8ce48e2d5d7bd5c6b69e05c6824f697d8de0a
Merge: 637dd0fd3f 9d4d68b2a3
Author: Dave Marion
AuthorDate: Fri Apr 5 15:09:17 2024 +
Merge branch 'main' into elasticity
.../apache/accumulo/core/clientImpl/InstanceOperationsImpl.java | 2 +-
.../accumulo/core/util/compaction/ExternalCompactionUtil.java | 7 ---
.../src/main/java/org/apache/accumulo/monitor/Monitor.java| 2 +-
.../monitor/rest/compactions/external/CoordinatorInfo.java| 4 ++--
.../monitor/rest/compactions/external/ExternalCompactionInfo.java | 8
.../org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java | 6 --
6 files changed, 16 insertions(+), 13 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index 86181247a8,0046af7dc6..48895192bd
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@@ -105,26 -106,26 +106,26 @@@ public class ExternalCompactionUtil
}
/**
- * @return map of queue names to compactor addresses
+ * @return map of group names to compactor addresses
*/
- public static Map> getCompactorAddrs(ClientContext
context) {
+ public static Map> getCompactorAddrs(ClientContext
context) {
try {
- final Map> groupsAndAddresses = new
HashMap<>();
- final Map> queuesAndAddresses = new HashMap<>();
- final String compactorQueuesPath = context.getZooKeeperRoot() +
Constants.ZCOMPACTORS;
++ final Map> groupsAndAddresses = new HashMap<>();
+ final String compactorGroupsPath = context.getZooKeeperRoot() +
Constants.ZCOMPACTORS;
ZooReader zooReader = context.getZooReader();
- List queues = zooReader.getChildren(compactorQueuesPath);
- for (String queue : queues) {
-queuesAndAddresses.putIfAbsent(queue, new HashSet<>());
+ List groups = zooReader.getChildren(compactorGroupsPath);
+ for (String group : groups) {
try {
- List compactors = zooReader.getChildren(compactorQueuesPath
+ "/" + queue);
+ List compactors = zooReader.getChildren(compactorGroupsPath
+ "/" + group);
for (String compactor : compactors) {
// compactor is the address, we are checking to see if there is a
child node which
// represents the compactor's lock as a check that it's alive.
List children =
-zooReader.getChildren(compactorQueuesPath + "/" + queue + "/"
+ compactor);
+zooReader.getChildren(compactorGroupsPath + "/" + group + "/"
+ compactor);
if (!children.isEmpty()) {
LOG.trace("Found live compactor {} ", compactor);
- groupsAndAddresses.putIfAbsent(group, new ArrayList<>());
-
queuesAndAddresses.get(queue).add(HostAndPort.fromString(compactor));
++ groupsAndAddresses.putIfAbsent(group, new HashSet<>());
+
groupsAndAddresses.get(group).add(HostAndPort.fromString(compactor));
}
}
} catch (NoNodeException e) {
diff --cc
server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/external/CoordinatorInfo.java
index d17ce19f18,8724f758bb..eccda4569e
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/external/CoordinatorInfo.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/external/CoordinatorInfo.java
@@@ -33,9 -33,9 +33,9 @@@ public class CoordinatorInfo
public CoordinatorInfo(Optional serverOpt,
ExternalCompactionInfo ecInfo) {
server = serverOpt.map(HostAndPort::toString).orElse("none");
-var queueToCompactors = ecInfo.getCompactors();
-numQueues = queueToCompactors.size();
-numCompactors =
queueToCompactors.values().stream().mapToInt(Set::size).sum();
+var groupToCompactors = ecInfo.getCompactors();
+numQueues = groupToCompactors.size();
- numCompactors =
groupToCompactors.values().stream().mapToInt(List::size).sum();
++numCompactors =
groupToCompactors.values().stream().mapToInt(Set::size).sum();
lastContact = System.currentTimeMillis() - ecInfo.getFetchedTimeMillis();
}
}
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
index 03f7442c91,771d74d588..637a71eca8
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
@@@ -22,10 -22,
(accumulo) branch main updated: Changed return variable for ExternalCompactionUtil.getCompactorAddrs (#4419)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9d4d68b2a3 Changed return variable for
ExternalCompactionUtil.getCompactorAddrs (#4419)
9d4d68b2a3 is described below
commit 9d4d68b2a373df2c36763a5ed55675a5f8d127a3
Author: Dave Marion
AuthorDate: Fri Apr 5 10:55:12 2024 -0400
Changed return variable for ExternalCompactionUtil.getCompactorAddrs (#4419)
---
.../apache/accumulo/core/clientImpl/InstanceOperationsImpl.java | 2 +-
.../accumulo/core/util/compaction/ExternalCompactionUtil.java | 7 ---
.../org/apache/accumulo/coordinator/CompactionCoordinator.java| 8
.../src/main/java/org/apache/accumulo/monitor/Monitor.java| 2 +-
.../monitor/rest/compactions/external/CoordinatorInfo.java| 4 ++--
.../monitor/rest/compactions/external/ExternalCompactionInfo.java | 8
6 files changed, 16 insertions(+), 15 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/InstanceOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/InstanceOperationsImpl.java
index 74bd2ece59..215f7c6214 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/InstanceOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/InstanceOperationsImpl.java
@@ -303,7 +303,7 @@ public class InstanceOperationsImpl implements
InstanceOperations {
public List getActiveCompactions()
throws AccumuloException, AccumuloSecurityException {
-Map> compactors =
ExternalCompactionUtil.getCompactorAddrs(context);
+Map> compactors =
ExternalCompactionUtil.getCompactorAddrs(context);
List tservers = getTabletServers();
int numThreads = Math.max(4, Math.min((tservers.size() +
compactors.size()) / 10, 256));
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index 35c358b7ed..0046af7dc6 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -107,14 +108,14 @@ public class ExternalCompactionUtil {
/**
* @return map of queue names to compactor addresses
*/
- public static Map> getCompactorAddrs(ClientContext
context) {
+ public static Map> getCompactorAddrs(ClientContext
context) {
try {
- final Map> queuesAndAddresses = new HashMap<>();
+ final Map> queuesAndAddresses = new HashMap<>();
final String compactorQueuesPath = context.getZooKeeperRoot() +
Constants.ZCOMPACTORS;
ZooReader zooReader = context.getZooReader();
List queues = zooReader.getChildren(compactorQueuesPath);
for (String queue : queues) {
-queuesAndAddresses.putIfAbsent(queue, new ArrayList<>());
+queuesAndAddresses.putIfAbsent(queue, new HashSet<>());
try {
List compactors = zooReader.getChildren(compactorQueuesPath
+ "/" + queue);
for (String compactor : compactors) {
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 7358ce4416..c86a93350c 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -304,7 +304,7 @@ public class CompactionCoordinator extends AbstractServer
long now = System.currentTimeMillis();
- Map> idleCompactors = getIdleCompactors();
+ Map> idleCompactors = getIdleCompactors();
TIME_COMPACTOR_LAST_CHECKED.forEach((queue, lastCheckTime) -> {
if ((now - lastCheckTime) > getMissingCompactorWarningTime()
&& QUEUE_SUMMARIES.isCompactionsQueued(queue) &&
idleCompactors.containsKey(queue)) {
@@ -325,16 +325,16 @@ public class CompactionCoordinator extends AbstractServer
LOG.info("Shutting down");
}
- private Map> getIdleCompactors() {
+ private Map> getIdleCompactors() {
-Map> allCompactors =
+Map> allCompactors =
ExternalCompactionUtil.getCompactorAddrs(getContext());
Set emptyQueues = new HashSet<>();
(accumulo) branch elasticity updated: Removed elasticity comment to do in ExternalCompaction_2_IT (#4416)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 637dd0fd3f Removed elasticity comment to do in ExternalCompaction_2_IT
(#4416)
637dd0fd3f is described below
commit 637dd0fd3f4ce9e408b93db59816e1b2b07b6a9e
Author: Dave Marion
AuthorDate: Fri Apr 5 10:11:25 2024 -0400
Removed elasticity comment to do in ExternalCompaction_2_IT (#4416)
The comment said that the operation id needed to be set when
deleting the tablets. This is now done in ReserveTablets.isReady.
Co-authored-by: Keith Turner
---
.../org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
index 937cb03037..4ca3794ca9 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java
@@ -238,9 +238,7 @@ public class ExternalCompaction_2_IT extends
SharedMiniClusterBase {
confirmCompactionCompleted(getCluster().getServerContext(), ecids,
TCompactionState.CANCELLED);
- // ELASTICITY_TODO make delete table fate op get operation ids before
deleting
- // there should be no metadata for the table, check to see if the
compaction wrote anything
- // after table delete
+ // Ensure compaction did not write anything to metadata table after
delete table
try (var scanner =
client.createScanner(AccumuloTable.METADATA.tableName())) {
scanner.setRange(MetadataSchema.TabletsSection.getRange(tid));
assertEquals(0, scanner.stream().count());
(accumulo) branch elasticity updated: Added missing Fate.shutdown in FateOpsCommandsIT (#4428)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 996952ff55 Added missing Fate.shutdown in FateOpsCommandsIT (#4428)
996952ff55 is described below
commit 996952ff5515c1525788cf6cbdee4a752f440229
Author: Dave Marion
AuthorDate: Fri Mar 29 09:03:48 2024 -0400
Added missing Fate.shutdown in FateOpsCommandsIT (#4428)
---
test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsIT.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsIT.java
b/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsIT.java
index b716d12d5f..5bddd4f35c 100644
--- a/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/fate/FateOpsCommandsIT.java
@@ -267,6 +267,8 @@ public abstract class FateOpsCommandsIT extends
ConfigurableMacBase
|| result.contains(
"Fate ID Filters: [" + fateId2.canonical() + ", " +
fateId1.canonical() + "]"));
assertTrue(result.contains("Instance Types Filters: [" +
store.type().name() + "]"));
+
+fate.shutdown(10, TimeUnit.MINUTES);
}
@Test
(accumulo) branch elasticity updated: Uncomment code in Monitor that retrieved TServer active compactions (#4407)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 98b994ce0a Uncomment code in Monitor that retrieved TServer active
compactions (#4407)
98b994ce0a is described below
commit 98b994ce0ad7d67a0e9d028b227909f7b7882ac7
Author: Dave Marion
AuthorDate: Fri Mar 22 08:41:14 2024 -0400
Uncomment code in Monitor that retrieved TServer active compactions (#4407)
Code was commented out in the Monitor that called
TabletClientHandler.getActiveCompactions. This method was removed
when major compactions were removed from the tserver, then restored
in #3827 because we still need to report minor compaction stats to
the monitor.
---
.../monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 9f8eb68473..1743235fd5 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -762,10 +762,8 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
Client tserver = null;
try {
tserver = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER,
parsedServer, context);
-// ELASTICITY_TODO tservers no longer have any compaction information,
following code was
-// commented out as the thrift calls no longer exists
-// var compacts = tserver.getActiveCompactions(null,
context.rpcCreds());
-// allCompactions.put(parsedServer, new CompactionStats(compacts));
+var compacts = tserver.getActiveCompactions(null, context.rpcCreds());
+allCompactions.put(parsedServer, new CompactionStats(compacts));
compactsFetchedNanos = System.nanoTime();
} catch (Exception ex) {
log.debug("Failed to get active compactions from {}", server, ex);
(accumulo) branch elasticity updated: Modified TabletGoalState log msg to remove todo (#4406)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 6176c3d07c Modified TabletGoalState log msg to remove todo (#4406)
6176c3d07c is described below
commit 6176c3d07cd57d9e208a6bb8e2580a2e62ae8545
Author: Dave Marion
AuthorDate: Fri Mar 22 08:38:01 2024 -0400
Modified TabletGoalState log msg to remove todo (#4406)
---
.../accumulo/server/manager/state/TabletGoalState.java | 13 -
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
index 0b9b83f159..ca9b2ada8c 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
@@ -146,15 +146,10 @@ public enum TabletGoalState {
return UNASSIGNED;
}
} else {
- // ELASTICITY_TODO this log level was set to error so that this case
can be examined for
- // bugs. A tablet server should always have a resource group. If
there are unavoidable
- // race conditions for getting tablet servers and their RGs, that
that should be handled
- // in the TabletManagementParameters data acquisition phase so that
not all code has to
- // deal with it. Eventually this log level should possibly be
adjusted or converted to an
- // exception.
- log.error(
- "Could not find resource group for tserver {}, so did not
consult balancer. Need to determine the cause of this.",
- tm.getLocation().getServerInstance());
+ log.warn("Could not find resource group for tserver {}, did not
consult balancer to"
+ + " check if tablet {} needs to be re-assigned. This tablet will
be rechecked"
+ + " soon. If this condition is not transient, then it could
indicate a bug so"
+ + " please report it.", tm.getLocation().getServerInstance(),
tm.getExtent());
}
}
(accumulo) branch elasticity updated: Made tablet refresh thread pool size configurable (#4405)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 1b7861b063 Made tablet refresh thread pool size configurable (#4405)
1b7861b063 is described below
commit 1b7861b0633d93e3f4dc0a7c8a177dfce296ed9b
Author: Dave Marion
AuthorDate: Fri Mar 22 08:34:49 2024 -0400
Made tablet refresh thread pool size configurable (#4405)
---
.../java/org/apache/accumulo/core/conf/Property.java | 12
.../java/org/apache/accumulo/manager/Manager.java | 12
.../manager/tableOps/bulkVer2/RefreshTablets.java | 9 ++---
.../manager/tableOps/bulkVer2/TabletRefresher.java| 19 ++-
.../manager/tableOps/compact/CompactionDriver.java| 3 +--
.../manager/tableOps/compact/RefreshTablets.java | 3 +--
6 files changed, 34 insertions(+), 24 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index a3057f9d36..d5a9504f28 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -367,6 +367,18 @@ public enum Property {
"Maximum number of threads the TabletGroupWatcher will use in its
BatchScanner to"
+ " look for tablets that need maintenance.",
"4.0.0"),
+ MANAGER_TABLET_REFRESH_MINTHREADS("manager.tablet.refresh.threads.mininum",
"10",
+ PropertyType.COUNT,
+ "The Manager will notify TabletServers that a Tablet needs to be
refreshed after certain operations"
+ + " are performed (e.g. Bulk Import). This property specifies the
number of core threads in a"
+ + " ThreadPool in the Manager that will be used to request these
refresh operations.",
+ "4.0.0"),
+ MANAGER_TABLET_REFRESH_MAXTHREADS("manager.tablet.refresh.threads.maximum",
"10",
+ PropertyType.COUNT,
+ "The Manager will notify TabletServers that a Tablet needs to be
refreshed after certain operations"
+ + " are performed (e.g. Bulk Import). This property specifies the
maximum number of threads in a"
+ + " ThreadPool in the Manager that will be used to request these
refresh operations.",
+ "4.0.0"),
MANAGER_BULK_TIMEOUT("manager.bulk.timeout", "5m", PropertyType.TIMEDURATION,
"The time to wait for a tablet server to process a bulk import
request.", "1.4.3"),
MANAGER_RENAME_THREADS("manager.rename.threadpool.size", "20",
PropertyType.COUNT,
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 174edd3dcf..d17f5f570c 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -50,6 +50,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -236,6 +237,7 @@ public class Manager extends AbstractServer
private final long timeToCacheRecoveryWalExistence;
private ExecutorService tableInformationStatusPool = null;
+ private ThreadPoolExecutor tabletRefreshThreadPool;
private final TabletStateStore rootTabletStore;
private final TabletStateStore metadataTabletStore;
@@ -436,6 +438,10 @@ public class Manager extends AbstractServer
return getContext().getTableManager();
}
+ public ThreadPoolExecutor getTabletRefreshThreadPool() {
+return tabletRefreshThreadPool;
+ }
+
public static void main(String[] args) throws Exception {
try (Manager manager = new Manager(new ConfigOpts(), args)) {
manager.runServer();
@@ -991,6 +997,11 @@ public class Manager extends AbstractServer
tableInformationStatusPool = ThreadPools.getServerThreadPools()
.createExecutorService(getConfiguration(),
Property.MANAGER_STATUS_THREAD_POOL_SIZE, false);
+tabletRefreshThreadPool =
ThreadPools.getServerThreadPools().getPoolBuilder("Tablet refresh ")
+
.numCoreThreads(getConfiguration().getCount(Property.MANAGER_TABLET_REFRESH_MINTHREADS))
+
.numMaxThreads(getConfiguration().getCount(Property.MANAGER_TABLET_REFRESH_MAXTHREADS))
+.build();
+
Thread st
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0ea9f6a1dd9386ee1f7a8789094cc89e812923d7 Merge: 1d6fd65ded a3a6fc2634 Author: Dave Marion AuthorDate: Fri Mar 22 12:28:27 2024 + Merge branch 'main' into elasticity .../main/java/org/apache/accumulo/server/rpc/TServerUtils.java | 9 + 1 file changed, 9 insertions(+)
(accumulo) branch main updated (eae367264b -> a3a6fc2634)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from eae367264b Merge branch '2.1' add 33894e6997 Fixed race condition in TServerUtils exposed by TServerUtilsTest (#4413) new a3a6fc2634 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/server/rpc/TServerUtils.java | 9 + 1 file changed, 9 insertions(+)
(accumulo) branch elasticity updated (1d6fd65ded -> 0ea9f6a1dd)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1d6fd65ded Merge branch 'main' into elasticity add 33894e6997 Fixed race condition in TServerUtils exposed by TServerUtilsTest (#4413) add a3a6fc2634 Merge branch '2.1' new 0ea9f6a1dd Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/server/rpc/TServerUtils.java | 9 + 1 file changed, 9 insertions(+)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a3a6fc2634d2d067d4c951f441bcebb2a06ed7b0 Merge: eae367264b 33894e6997 Author: Dave Marion AuthorDate: Fri Mar 22 12:28:06 2024 + Merge branch '2.1' .../main/java/org/apache/accumulo/server/rpc/TServerUtils.java | 9 + 1 file changed, 9 insertions(+) diff --cc server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java index 5f0cebd05a,9f37990926..0f28ecab8b --- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java +++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java @@@ -46,7 -46,9 +46,8 @@@ import org.apache.accumulo.core.rpc.Ssl import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.rpc.UGIAssumingTransportFactory; import org.apache.accumulo.core.util.Halt; -import org.apache.accumulo.core.util.HostAndPort; import org.apache.accumulo.core.util.Pair; + import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.server.ServerContext; @@@ -70,7 -72,7 +71,8 @@@ import org.apache.thrift.transport.TTra import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; import com.google.common.primitives.Ints; /**
(accumulo) branch 2.1 updated: Fixed race condition in TServerUtils exposed by TServerUtilsTest (#4413)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 33894e6997 Fixed race condition in TServerUtils exposed by
TServerUtilsTest (#4413)
33894e6997 is described below
commit 33894e69979afc70efca448ea31fb29ac73288f3
Author: Dave Marion
AuthorDate: Fri Mar 22 08:14:21 2024 -0400
Fixed race condition in TServerUtils exposed by TServerUtilsTest (#4413)
There are several tests in TServerUtilsTest that have the form:
```
TServer server = null;
try {
ServerAddress address = startServer();
server = address.getServer();
} finally {
if (server != null) {
server.stop();
}
}
```
The TServerUtilsTest.startServer method calls TServerUtils.startServer
which ends up creating a Thread that calls TServer.serve(). When TServer
is a TThreadPoolServer, the serve method calls preServe first, which sets
the internal boolean variable `stopped_` to false, and then calls execute
which will loop while `stopped_` is false.
In the case where the Thread created by TServerUtils.startServer is not
started right away, then it's possible that the test method will call
stop (setting `stopped_` to true) before the TServer.serve method calls
preServe (setting `stopped_` back to false) resulting in the Thread
being in an endless loop.
This can be seen by running TServerTestUtils, where the output contains
many lines like:
```
[server.TThreadPoolServer] WARN : Transport error occurred during
acceptance of message
org.apache.thrift.transport.TTransportException: No underlying server
socket.
at
org.apache.thrift.transport.TServerSocket.accept(TServerSocket.java:113)
~[libthrift-0.17.0.jar:0.17.0]
at
org.apache.thrift.transport.TServerSocket.accept(TServerSocket.java:31)
~[libthrift-0.17.0.jar:0.17.0]
at
org.apache.thrift.server.TThreadPoolServer.execute(TThreadPoolServer.java:162)
~[libthrift-0.17.0.jar:0.17.0]
at
org.apache.thrift.server.TThreadPoolServer.serve(TThreadPoolServer.java:148)
~[libthrift-0.17.0.jar:0.17.0]
at
org.apache.accumulo.server.rpc.TServerUtils.lambda$startTServer$9(TServerUtils.java:654)
~[classes/:?]
at
org.apache.accumulo.core.trace.TraceWrappedRunnable.run(TraceWrappedRunnable.java:52)
[accumulo-core-2.1.3-SNAPSHOT.jar:2.1.3-SNAPSHOT]
at java.base/java.lang.Thread.run(Thread.java:840) [?:?]
```
Because the surefire configuration reuses JVM forks these Threads persist
for the duration of the unit tests in server base and pollute every test
output file after TServerUtilsTest is executed. The surefire forkCount is
set to `1C`, so the volume of output in the logs is dependent on the number
of forks.
Co-authored-by: Keith Turner
---
.../main/java/org/apache/accumulo/server/rpc/TServerUtils.java | 9 +
1 file changed, 9 insertions(+)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
index 8ae472cf8b..9f37990926 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java
@@ -48,6 +48,7 @@ import
org.apache.accumulo.core.rpc.UGIAssumingTransportFactory;
import org.apache.accumulo.core.util.Halt;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.server.ServerContext;
@@ -71,6 +72,7 @@ import org.apache.thrift.transport.TTransportFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
import com.google.common.primitives.Ints;
/**
@@ -657,6 +659,13 @@ public class TServerUtils {
}
}).start();
+while (!finalServer.isServing()) {
+ // Wait for the thread to start and for the TServer to start
+ // serving events
+ UtilWaitThread.sleep(10);
+ Preconditions.checkState(!finalServer.getShouldStop());
+}
+
// check for the special "bind to everything address"
if (serverAddress.address.getHost().equals("0.0.0.0")) {
// can't get the address from the bind, so we'll do our best to invent
our hostname
(accumulo) branch elasticity updated: Reduced number of compactions in CompactionConfigChangeIT (#4402)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new ce83488245 Reduced number of compactions in CompactionConfigChangeIT
(#4402)
ce83488245 is described below
commit ce834882456546df79e957f2f242adc37d059bdc
Author: Dave Marion
AuthorDate: Tue Mar 19 12:20:55 2024 -0400
Reduced number of compactions in CompactionConfigChangeIT (#4402)
The test waits for 60s for the number of F files to reach zero.
However, there are 100 files and there is a comment that says
each compaction should take about 1s. I reduced the number of
files from 100 to 50 to allow the test to complete in the allotted
time.
---
.../apache/accumulo/test/compaction/CompactionConfigChangeIT.java | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
index c734493287..fe1783b841 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionConfigChangeIT.java
@@ -75,26 +75,26 @@ public class CompactionConfigChangeIT extends
AccumuloClusterHarness {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
final String table = getUniqueNames(1)[0];
- createTable(client, table, "cs1", 100);
+ createTable(client, table, "cs1", 50);
ExternalCompactionTestUtils.writeData(client, table, MAX_DATA);
client.tableOperations().flush(table, null, null, true);
- assertEquals(100, countFiles(client, table, "F"));
+ assertEquals(50, countFiles(client, table, "F"));
// Start 100 slow compactions, each compaction should take ~1 second.
There are 2 tservers
// each with 2 threads and then 8 threads.
CompactionConfig compactionConfig = new CompactionConfig();
IteratorSetting iteratorSetting = new IteratorSetting(100,
SlowIterator.class);
- SlowIterator.setSleepTime(iteratorSetting, 100);
+ SlowIterator.setSleepTime(iteratorSetting, 50);
compactionConfig.setIterators(List.of(iteratorSetting));
compactionConfig.setWait(false);
client.tableOperations().compact(table, compactionConfig);
// give some time for compactions to start running
- Wait.waitFor(() -> countFiles(client, table, "F") < 95);
+ Wait.waitFor(() -> countFiles(client, table, "F") < 45);
// Change config deleting groups named small, medium, and large. There
was bug where
// deleting groups running compactions would leave the tablet in a bad
state for future
(accumulo) branch elasticity updated: Resurrected CompactionDriverTest resolving TODO in PreDeleteTable (#4377)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new ca56493db7 Resurrected CompactionDriverTest resolving TODO in
PreDeleteTable (#4377)
ca56493db7 is described below
commit ca56493db7eb122345a7c954c820fe9b51e4687e
Author: Dave Marion
AuthorDate: Tue Mar 19 12:20:17 2024 -0400
Resurrected CompactionDriverTest resolving TODO in PreDeleteTable (#4377)
The TODO asked if the the delete marker in ZooKeeper was still
being used / tested. It is still being used by the CompactionDriver.
However, the CompactionDriverTest class was deleted at some point
which tests it. I resurrected and fixed the test.
---
.../manager/tableOps/compact/CompactionDriver.java | 3 +-
.../manager/tableOps/delete/PreDeleteTable.java| 1 -
.../tableOps/compact/CompactionDriverTest.java | 148 +
3 files changed, 150 insertions(+), 2 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
index f8ad23172b..6167ca05cf 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
@@ -129,7 +129,8 @@ class CompactionDriver extends ManagerRepo {
return sleepTime;
}
- private boolean isCancelled(FateId fateId, ServerContext context)
+ // visible for testing
+ protected boolean isCancelled(FateId fateId, ServerContext context)
throws InterruptedException, KeeperException {
return CompactionConfigStorage.getConfig(context, fateId) == null;
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java
index 196e898c09..6094960b38 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/PreDeleteTable.java
@@ -60,7 +60,6 @@ public class PreDeleteTable extends ManagerRepo {
private void preventFutureCompactions(Manager environment)
throws KeeperException, InterruptedException {
-// ELASTICITY_TODO investigate this. Is still needed? Is it still working
as expected?
String deleteMarkerPath =
createDeleteMarkerPath(environment.getInstanceID(), tableId);
ZooReaderWriter zoo = environment.getContext().getZooReaderWriter();
zoo.putPersistentData(deleteMarkerPath, new byte[] {},
NodeExistsPolicy.SKIP);
diff --git
a/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java
b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java
new file mode 100644
index 00..caf8be89d5
--- /dev/null
+++
b/server/manager/src/test/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriverTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.manager.tableOps.compact;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import java.util.UUID;
+
+import
org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException;
+import org.apache.accumulo.core.clientImpl.TableOperationsImpl;
+import org.apache.accumulo.core.clientImpl.thrift.TableOperation;
+import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
+import org.apache.accumulo.core.data.InstanceId;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.fate.FateId;
+import org.apache.accumulo.core.fate.FateInstanceType;
+import org.apache.accumulo.core.fate.zookeeper.ZooR
(accumulo) branch elasticity updated: Resolved todo in tabletserver.thrift, removed majors from TabletStats (#4366)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 2a2b44670e Resolved todo in tabletserver.thrift, removed majors from
TabletStats (#4366)
2a2b44670e is described below
commit 2a2b44670e732ad1e50c9c10ccad52d258190e28
Author: Dave Marion
AuthorDate: Tue Mar 19 10:26:35 2024 -0400
Resolved todo in tabletserver.thrift, removed majors from TabletStats
(#4366)
---
.../core/tabletserver/thrift/TabletStats.java | 132 ++---
core/src/main/thrift/tabletserver.thrift | 4 +-
.../core/spi/balancer/SimpleLoadBalancerTest.java | 2 +-
.../monitor/rest/tservers/CurrentOperations.java | 12 +-
.../tservers/TabletServerDetailInformation.java| 6 +-
.../rest/tservers/TabletServerResource.java| 29 +
.../apache/accumulo/tserver/TabletStatsKeeper.java | 2 +-
.../accumulo/test/ChaoticLoadBalancerTest.java | 2 +-
8 files changed, 22 insertions(+), 167 deletions(-)
diff --git
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletStats.java
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletStats.java
index b975e7b22e..a1b5b2c2ac 100644
---
a/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletStats.java
+++
b/core/src/main/thrift-gen-java/org/apache/accumulo/core/tabletserver/thrift/TabletStats.java
@@ -29,7 +29,6 @@ public class TabletStats implements
org.apache.thrift.TBase tmpMap =
new java.util.EnumMap<_Fields,
org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
tmpMap.put(_Fields.EXTENT, new
org.apache.thrift.meta_data.FieldMetaData("extent",
org.apache.thrift.TFieldRequirementType.DEFAULT,
new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
org.apache.accumulo.core.dataImpl.thrift.TKeyExtent.class)));
-tmpMap.put(_Fields.MAJORS, new
org.apache.thrift.meta_data.FieldMetaData("majors",
org.apache.thrift.TFieldRequirementType.DEFAULT,
-new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
ActionStats.class)));
tmpMap.put(_Fields.MINORS, new
org.apache.thrift.meta_data.FieldMetaData("minors",
org.apache.thrift.TFieldRequirementType.DEFAULT,
new
org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT,
ActionStats.class)));
tmpMap.put(_Fields.SPLITS, new
org.apache.thrift.meta_data.FieldMetaData("splits",
org.apache.thrift.TFieldRequirementType.DEFAULT,
@@ -158,7 +151,6 @@ public class TabletStats implements
org.apache.thrift.TBase tsStats = new ArrayList<>();
try {
@@ -205,21 +204,11 @@ public class TabletServerResource {
if (total.minors.elapsed != 0 && total.minors.num != 0) {
currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num,
total.minors.sumDev);
}
-if (total.majors.num != 0) {
- currentMajorAvg = total.majors.elapsed / total.majors.num;
-}
-if (total.majors.elapsed != 0 && total.majors.num != 0
-&& total.majors.elapsed > total.majors.num) {
- currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num,
total.majors.sumDev);
-}
ActionStatsUpdator.update(total.minors, historical.minors);
-ActionStatsUpdator.update(total.majors, historical.majors);
minorStdDev = stddev(total.minors.elapsed, total.minors.num,
total.minors.sumDev);
minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num,
total.minors.queueSumDev);
-majorStdDev = stddev(total.majors.elapsed, total.majors.num,
total.majors.sumDev);
-majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num,
total.majors.queueSumDev);
splitStdDev =
stddev(historical.splits.elapsed, historical.splits.num,
historical.splits.sumDev);
@@ -267,7 +256,7 @@ public class TabletServerResource {
private TabletServerDetailInformation doDetails(int numTablets) {
return new TabletServerDetailInformation(numTablets, total.numEntries,
total.minors.status,
-total.majors.status, historical.splits.status);
+historical.splits.status);
}
private List doAllTimeResults(double majorQueueStdDev,
@@ -282,12 +271,6 @@ public class TabletServerResource {
minorQueueStdDev, total.minors.num != 0 ? (total.minors.elapsed /
total.minors.num) : null,
minorStdDev, total.minors.elapsed));
-// Major Compaction Operation
-allTime.add(new AllTimeTabletResults("MajorCompaction",
total.majors.num,
-total.majors.fail,
-total.majors.num != 0 ? (total.majors.queueTime / total.majors.num) :
null,
-majorQueueStdDev, total.majors.num != 0 ? (tot
(accumulo) branch elasticity updated: Call requireSame(SUSPEND) in MetaDataStateStore to resolve todo (#4381)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5cb7f7b87b Call requireSame(SUSPEND) in MetaDataStateStore to resolve
todo (#4381)
5cb7f7b87b is described below
commit 5cb7f7b87b6f6b7b69dbdd6e170e537274d1d717
Author: Dave Marion
AuthorDate: Tue Mar 19 10:25:04 2024 -0400
Call requireSame(SUSPEND) in MetaDataStateStore to resolve todo (#4381)
---
.../MiniAccumuloClusterControl.java| 20 +
.../server/manager/state/MetaDataStateStore.java | 7 +-
.../metadata/ConditionalTabletMutatorImpl.java | 11 +++
.../test/functional/AmpleConditionalWriterIT.java | 85 ++
4 files changed, 120 insertions(+), 3 deletions(-)
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index 9bf50cff2e..1e10be5faa 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@ -274,6 +274,26 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
}
}
+ public void stopTabletServerGroup(String tserverResourceGroup) {
+synchronized (tabletServerProcesses) {
+ var group = tabletServerProcesses.get(tserverResourceGroup);
+ if (group == null) {
+return;
+ }
+ group.forEach(process -> {
+try {
+ cluster.stopProcessWithTimeout(process, 30, TimeUnit.SECONDS);
+} catch (ExecutionException | TimeoutException e) {
+ log.warn("TabletServer did not fully stop after 30 seconds", e);
+ throw new RuntimeException(e);
+} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+}
+ });
+ tabletServerProcesses.remove(tserverResourceGroup);
+}
+ }
+
@Override
public synchronized void stop(ServerType server, String hostname) throws
IOException {
switch (server) {
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
index 689a667e4b..f69f280ed4 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.server.manager.state;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
+
import java.util.Collection;
import java.util.List;
@@ -68,9 +70,8 @@ class MetaDataStateStore extends AbstractTabletStateStore
implements TabletState
try (var tabletsMutator = ample.conditionallyMutateTablets()) {
for (TabletMetadata tm : tablets) {
if (tm.getSuspend() != null) {
- // ELASTICITY_TODO add conditional mutation check that tls.suspend
is what currently
- // exists in the tablet
-
tabletsMutator.mutateTablet(tm.getExtent()).requireAbsentOperation().deleteSuspension()
+ tabletsMutator.mutateTablet(tm.getExtent()).requireAbsentOperation()
+ .requireSame(tm, SUSPEND).deleteSuspension()
.submit(tabletMetadata -> tabletMetadata.getSuspend() == null);
}
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
index 6003fd73e6..6995941089 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
@@ -24,6 +24,7 @@ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.OPID_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.SELECTED_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN
(accumulo) branch elasticity updated: Removed MAC methods deprecated in 3.1 (#4395)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 59b873b49c Removed MAC methods deprecated in 3.1 (#4395)
59b873b49c is described below
commit 59b873b49c71649e633b95382088be6a4c94f11e
Author: Dave Marion
AuthorDate: Mon Mar 18 13:18:19 2024 -0400
Removed MAC methods deprecated in 3.1 (#4395)
---
.../accumulo/minicluster/MiniAccumuloConfig.java | 35 --
.../accumulo/minicluster/MiniAccumuloRunner.java | 4 ---
2 files changed, 39 deletions(-)
diff --git
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index d43008be97..6bf6739d5d 100644
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -51,31 +51,6 @@ public class MiniAccumuloConfig {
this.impl = new MiniAccumuloConfigImpl(dir, rootPassword);
}
- /**
- * Calling this method is optional. If not set, it defaults to two.
- *
- * @param numTservers the number of tablet servers that mini accumulo
cluster should start
- */
- // ELASTICITY_TODO: Deprecate in 3.0.0 and remove in elasticity on the merge
- @Deprecated(since = "3.1.0")
- public MiniAccumuloConfig setNumTservers(int numTservers) {
-// impl.setNumTservers(numTservers);
-return this;
- }
-
- /**
- * Calling this method is optional. If not set, it defaults to zero.
- *
- * @param numScanServers the number of scan servers that mini accumulo
cluster should start
- * @since 2.1.0
- */
- // ELASTICITY_TODO: Deprecate in 3.0.0 and remove in elasticity on the merge
- @Deprecated(since = "3.1.0")
- public MiniAccumuloConfig setNumScanServers(int numScanServers) {
-// impl.setNumScanServers(numScanServers);
-return this;
- }
-
/**
* Calling this method is optional. If not set, defaults to 'miniInstance'
*
@@ -225,16 +200,6 @@ public class MiniAccumuloConfig {
return impl.getRootPassword();
}
- /**
- * @return the number of tservers configured for this cluster
- */
- // ELASTICITY_TODO: Deprecate in 3.0.0 and remove in elasticity on the merge
- @Deprecated(since = "3.1.0")
- public int getNumTservers() {
-return
impl.getClusterServerConfiguration().getTabletServerConfiguration().values().stream()
-.reduce(0, Integer::sum);
- }
-
/**
* @return is the current configuration in jdwpEnabled mode?
*
diff --git
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
index 48dcf7a337..70e8c388a9 100644
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
@@ -150,7 +150,6 @@ public class MiniAccumuloRunner {
*
* @param args An optional -p argument can be specified with the path to a
valid properties file.
*/
- @SuppressWarnings("deprecation")
@SuppressFBWarnings(value = {"PATH_TRAVERSAL_IN",
"UNENCRYPTED_SERVER_SOCKET"},
justification = "code runs in same security context as user who provided
input file name; "
+ "socket need not be encrypted, since this class is provided for
testing only")
@@ -181,9 +180,6 @@ public class MiniAccumuloRunner {
if (opts.prop.containsKey(INSTANCE_NAME_PROP)) {
config.setInstanceName(opts.prop.getProperty(INSTANCE_NAME_PROP));
}
-if (opts.prop.containsKey(NUM_T_SERVERS_PROP)) {
-
config.setNumTservers(Integer.parseInt(opts.prop.getProperty(NUM_T_SERVERS_PROP)));
-}
if (opts.prop.containsKey(ZOO_KEEPER_PORT_PROP)) {
config.setZooKeeperPort(Integer.parseInt(opts.prop.getProperty(ZOO_KEEPER_PORT_PROP)));
}
(accumulo) branch elasticity updated: Updated coordinator log warning to account for busy compactors (#4372)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 4099860261 Updated coordinator log warning to account for busy
compactors (#4372)
4099860261 is described below
commit 4099860261a6cdb68700176ced44eb0519420e88
Author: Dave Marion
AuthorDate: Mon Mar 18 11:00:08 2024 -0400
Updated coordinator log warning to account for busy compactors (#4372)
Modified the logic in CompactionCoordinator to only warn about
compactors not checking in when there are idle compactors for
that group. Refactored code to remove a TODO.
Fixes #4219
---
.../coordinator/CompactionCoordinator.java | 84 ++
.../compaction/CompactionCoordinatorTest.java | 12 ++--
2 files changed, 64 insertions(+), 32 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 178b4f1e95..48419a47a0 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@ -18,11 +18,9 @@
*/
package org.apache.accumulo.manager.compaction.coordinator;
-import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
-import static java.util.stream.Collectors.toMap;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
@@ -37,12 +35,15 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -163,7 +164,7 @@ public class CompactionCoordinator
private final CompactionJobQueues jobQueues;
private final AtomicReference>>
fateInstances;
// Exposed for tests
- protected volatile Boolean shutdown = false;
+ protected CountDownLatch shutdown = new CountDownLatch(1);
private final ScheduledThreadPoolExecutor schedExecutor;
@@ -220,7 +221,7 @@ public class CompactionCoordinator
}
public void shutdown() {
-shutdown = true;
+shutdown.countDown();
var localThread = serviceThread;
if (localThread != null) {
try {
@@ -243,6 +244,28 @@ public class CompactionCoordinator
ThreadPools.watchNonCriticalScheduledTask(future);
}
+ protected void startIdleCompactionWatcher() {
+
+ScheduledFuture future =
schedExecutor.scheduleWithFixedDelay(this::idleCompactionWarning,
+getTServerCheckInterval(), getTServerCheckInterval(),
TimeUnit.MILLISECONDS);
+ThreadPools.watchNonCriticalScheduledTask(future);
+ }
+
+ private void idleCompactionWarning() {
+
+long now = System.currentTimeMillis();
+Map> idleCompactors = getIdleCompactors();
+TIME_COMPACTOR_LAST_CHECKED.forEach((groupName, lastCheckTime) -> {
+ if ((now - lastCheckTime) > getMissingCompactorWarningTime()
+ && jobQueues.getQueuedJobs(groupName) > 0
+ && idleCompactors.containsKey(groupName.canonical())) {
+LOG.warn("No compactors have checked in with coordinator for group {}
in {}ms", groupName,
+getMissingCompactorWarningTime());
+ }
+});
+
+ }
+
@Override
public void run() {
@@ -270,35 +293,40 @@ public class CompactionCoordinator
startDeadCompactionDetector();
-// ELASTICITY_TODO the main function of the following loop was getting
group summaries from
-// tservers. Its no longer doing that. May be best to remove the loop and
make the remaining
-// task a scheduled one.
-
-LOG.info("Starting loop to check for compactors not checking in");
-while (!shutdown) {
- long start = System.currentTimeMillis();
-
- long now = System.currentTimeMillis();
- TIME_COMPACTOR_LAST_CHECKED.forEach((k, v) -> {
-if ((now - v) > getMissingCompactorWarningTime()) {
- // ELASTICITY_TODO may want t
(accumulo) branch elasticity updated (158bbaddf7 -> bf9a6a59fe)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 158bbaddf7 Resolve TODO in SplitCancelsMajCIT (#4382) add 8b0262d5b3 Deprecated MiniAccumuloConfig setNumServer methods (#4374) new bf9a6a59fe Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java | 3 +++ .../main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java | 1 + 2 files changed, 4 insertions(+)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit bf9a6a59fe34b135ba163c9e9ee4411d0f6c3c12
Merge: 158bbaddf7 8b0262d5b3
Author: Dave Marion
AuthorDate: Mon Mar 18 12:03:26 2024 +
Merge branch 'main' into elasticity
.../main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java | 3 +++
.../main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java | 1 +
2 files changed, 4 insertions(+)
diff --cc
minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 6446dfd44c,4ee34fcf30..d43008be97
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@@ -56,9 -56,9 +56,10 @@@ public class MiniAccumuloConfig
*
* @param numTservers the number of tablet servers that mini accumulo
cluster should start
*/
+ // ELASTICITY_TODO: Deprecate in 3.0.0 and remove in elasticity on the merge
+ @Deprecated(since = "3.1.0")
public MiniAccumuloConfig setNumTservers(int numTservers) {
-impl.setNumTservers(numTservers);
+// impl.setNumTservers(numTservers);
return this;
}
@@@ -68,9 -68,9 +69,10 @@@
* @param numScanServers the number of scan servers that mini accumulo
cluster should start
* @since 2.1.0
*/
+ // ELASTICITY_TODO: Deprecate in 3.0.0 and remove in elasticity on the merge
+ @Deprecated(since = "3.1.0")
public MiniAccumuloConfig setNumScanServers(int numScanServers) {
-impl.setNumScanServers(numScanServers);
+// impl.setNumScanServers(numScanServers);
return this;
}
@@@ -226,10 -226,9 +228,11 @@@
/**
* @return the number of tservers configured for this cluster
*/
+ // ELASTICITY_TODO: Deprecate in 3.0.0 and remove in elasticity on the merge
+ @Deprecated(since = "3.1.0")
public int getNumTservers() {
-return impl.getNumTservers();
+return
impl.getClusterServerConfiguration().getTabletServerConfiguration().values().stream()
+.reduce(0, Integer::sum);
}
/**
(accumulo) branch main updated: Deprecated MiniAccumuloConfig setNumServer methods (#4374)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8b0262d5b3 Deprecated MiniAccumuloConfig setNumServer methods (#4374)
8b0262d5b3 is described below
commit 8b0262d5b31142b5728f463adffe0bdfd301ec79
Author: Dave Marion
AuthorDate: Mon Mar 18 07:54:19 2024 -0400
Deprecated MiniAccumuloConfig setNumServer methods (#4374)
These methods are removed in the elasticity branch in favor
of a cluster server configuration object
---
.../main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java | 3 +++
.../main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java | 1 +
2 files changed, 4 insertions(+)
diff --git
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 2945eb7650..4ee34fcf30 100644
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -56,6 +56,7 @@ public class MiniAccumuloConfig {
*
* @param numTservers the number of tablet servers that mini accumulo
cluster should start
*/
+ @Deprecated(since = "3.1.0")
public MiniAccumuloConfig setNumTservers(int numTservers) {
impl.setNumTservers(numTservers);
return this;
@@ -67,6 +68,7 @@ public class MiniAccumuloConfig {
* @param numScanServers the number of scan servers that mini accumulo
cluster should start
* @since 2.1.0
*/
+ @Deprecated(since = "3.1.0")
public MiniAccumuloConfig setNumScanServers(int numScanServers) {
impl.setNumScanServers(numScanServers);
return this;
@@ -224,6 +226,7 @@ public class MiniAccumuloConfig {
/**
* @return the number of tservers configured for this cluster
*/
+ @Deprecated(since = "3.1.0")
public int getNumTservers() {
return impl.getNumTservers();
}
diff --git
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
index 31cd0898f9..cb4e9da20a 100644
---
a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
+++
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloRunner.java
@@ -150,6 +150,7 @@ public class MiniAccumuloRunner {
*
* @param args An optional -p argument can be specified with the path to a
valid properties file.
*/
+ @SuppressWarnings("deprecation")
@SuppressFBWarnings(value = {"PATH_TRAVERSAL_IN",
"UNENCRYPTED_SERVER_SOCKET"},
justification = "code runs in same security context as user who provided
input file name; "
+ "socket need not be encrypted, since this class is provided for
testing only")
(accumulo) branch elasticity updated: Resolve TODO in SplitCancelsMajCIT (#4382)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 158bbaddf7 Resolve TODO in SplitCancelsMajCIT (#4382)
158bbaddf7 is described below
commit 158bbaddf73717566f452d4bf3be20ff9a59151c
Author: Dave Marion
AuthorDate: Mon Mar 18 07:38:52 2024 -0400
Resolve TODO in SplitCancelsMajCIT (#4382)
---
.../test/compaction/SplitCancelsMajCIT.java| 32 --
1 file changed, 29 insertions(+), 3 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java
index 640b52499e..e7ee6a19bb 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/SplitCancelsMajCIT.java
@@ -19,10 +19,13 @@
package org.apache.accumulo.test.compaction;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import java.util.EnumSet;
+import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicReference;
@@ -31,11 +34,18 @@ import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.compaction.thrift.TCompactionState;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.functional.SlowIterator;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
@@ -44,8 +54,14 @@ import org.junit.jupiter.api.Test;
// ACCUMULO-2862
public class SplitCancelsMajCIT extends SharedMiniClusterBase {
- // ELASTICITY_TODO: Need to check new split code to ensure that it
- // still cancels running MAJC.
+ public static class ClusterConfigForTest implements
MiniClusterConfigurationCallback {
+
+@Override
+public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
coreSite) {
+ cfg.setProperty(Property.COMPACTOR_CANCEL_CHECK_INTERVAL, "10s");
+}
+
+ }
@Override
protected Duration defaultTimeout() {
@@ -54,7 +70,7 @@ public class SplitCancelsMajCIT extends SharedMiniClusterBase
{
@BeforeAll
public static void setup() throws Exception {
-SharedMiniClusterBase.startMiniCluster();
+SharedMiniClusterBase.startMiniClusterWithConfig(new
ClusterConfigForTest());
}
@AfterAll
@@ -67,6 +83,7 @@ public class SplitCancelsMajCIT extends SharedMiniClusterBase
{
final String tableName = getUniqueNames(1)[0];
try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
c.tableOperations().create(tableName);
+ TableId tid =
TableId.of(c.tableOperations().tableIdMap().get(tableName));
// majc should take 100 * .5 secs
IteratorSetting it = new IteratorSetting(100, SlowIterator.class);
SlowIterator.setSleepTime(it, 500);
@@ -90,12 +107,21 @@ public class SplitCancelsMajCIT extends
SharedMiniClusterBase {
});
thread.start();
+ Set compactionIds = ExternalCompactionTestUtils
+
.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
+ assertNotNull(compactionIds);
+ assertEquals(1, compactionIds.size());
+
long now = System.currentTimeMillis();
Thread.sleep(SECONDS.toMillis(10));
// split the table, interrupts the compaction
SortedSet partitionKeys = new TreeSet<>();
partitionKeys.add(new Text("10"));
c.tableOperations().addSplits(tableName, partitionKeys);
+
+
ExternalCompactionTestUtils.confirmCompactionCompleted(getCluster().getServerContext(),
+ compactionIds, TCompactionState.CANCELLED);
+
thread.join();
// wait for the restarted compaction
assertTrue(System.currentTimeMillis() - now > 59_000);
(accumulo) branch elasticity updated: Resolved TODOs in TabletGroupWatcher (#4373)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 99cee92af5 Resolved TODOs in TabletGroupWatcher (#4373)
99cee92af5 is described below
commit 99cee92af5325870a01881ea442fd1a8f342ba97
Author: Dave Marion
AuthorDate: Mon Mar 18 07:36:23 2024 -0400
Resolved TODOs in TabletGroupWatcher (#4373)
---
.../apache/accumulo/manager/TabletGroupWatcher.java | 21 ++---
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index bd177c2deb..8779568916 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -22,6 +22,7 @@ import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterrup
import static java.lang.Math.min;
import static java.util.Objects.requireNonNull;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
import java.io.IOException;
import java.util.ArrayList;
@@ -285,6 +286,10 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
needsFullScan = false;
}
+public synchronized boolean isNeedsFullScan() {
+ return needsFullScan;
+}
+
@Override
public void process(EventCoordinator.Event event) {
@@ -640,10 +645,12 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
boolean lookForTabletsNeedingVolReplacement = true;
while (manager.stillManager()) {
- // slow things down a little, otherwise we spam the logs when there are
many wake-up events
- sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
- // ELASTICITY_TODO above sleep in the case when not doing a full scan to
make manager more
- // responsive
+ if (!eventHandler.isNeedsFullScan()) {
+// If an event handled by the EventHandler.RangeProcessor indicated
+// that we need to do a full scan, then do it. Otherwise wait a bit
+// before re-checking the tablets.
+sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ }
final long waitTimeBetweenScans = manager.getConfiguration()
.getTimeInMillis(Property.MANAGER_TABLET_GROUP_WATCHER_INTERVAL);
@@ -978,9 +985,9 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
private void replaceVolumes(List
volumeReplacementsList) {
try (var tabletsMutator =
manager.getContext().getAmple().conditionallyMutateTablets()) {
for (VolumeUtil.VolumeReplacements vr : volumeReplacementsList) {
-// ELASTICITY_TODO can require same on WALS once that is implemented,
see #3948
-var tabletMutator =
tabletsMutator.mutateTablet(vr.tabletMeta.getExtent())
-
.requireAbsentOperation().requireAbsentLocation().requireSame(vr.tabletMeta,
FILES);
+var tabletMutator =
+
tabletsMutator.mutateTablet(vr.tabletMeta.getExtent()).requireAbsentOperation()
+.requireAbsentLocation().requireSame(vr.tabletMeta, FILES,
LOGS);
vr.logsToRemove.forEach(tabletMutator::deleteWal);
vr.logsToAdd.forEach(tabletMutator::putWal);
(accumulo) branch elasticity updated: Modified UnloadTabletHandler to use Tablets metadata (#4368)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 80452089b9 Modified UnloadTabletHandler to use Tablets metadata (#4368)
80452089b9 is described below
commit 80452089b9f9775d31b334d9acc06b6f006e58cd
Author: Dave Marion
AuthorDate: Mon Mar 18 07:34:53 2024 -0400
Modified UnloadTabletHandler to use Tablets metadata (#4368)
Resolved TODO in UnloadTabletHandler by using the Tablets
metadata reference, removing the code that was looking up
the tablet metadata.
---
.../org/apache/accumulo/tserver/UnloadTabletHandler.java | 12 +---
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
index f4ab2ec7b4..a1d46257ec 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/UnloadTabletHandler.java
@@ -20,14 +20,10 @@ package org.apache.accumulo.tserver;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
-import org.apache.accumulo.core.client.admin.TabletAvailability;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.manager.thrift.TabletLoadState;
-import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
-import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.tablet.thrift.TUnloadTabletGoal;
import org.apache.accumulo.server.manager.state.DistributedStoreException;
import org.apache.accumulo.server.manager.state.TabletStateStore;
@@ -109,13 +105,7 @@ class UnloadTabletHandler implements Runnable {
server.onlineTablets.remove(extent);
try {
- TServerInstance instance = server.getTabletSession();
- // ELASTICITY_TODO: Modify Tablet to keep a reference to TableMetadata
so that we
- // can avoid building a tablet metadata that may not have needed
information, for example may
- // need the last location
- TabletMetadata tm =
TabletMetadata.builder(extent).putLocation(Location.current(instance))
- .putTabletAvailability(TabletAvailability.ONDEMAND)
- .build(ColumnType.LAST, ColumnType.SUSPEND);
+ TabletMetadata tm = t.getMetadata();
if (!goalState.equals(TUnloadTabletGoal.SUSPENDED) ||
extent.isRootTablet()
|| (extent.isMeta()
&&
!server.getConfiguration().getBoolean(Property.MANAGER_METADATA_SUSPENDABLE))) {
(accumulo) branch elasticity updated (9550da7c83 -> f1914da45a)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9550da7c83 Merge branch 'main' into elasticity add 7c9204274c Fixes ConcurrentModificationException in RunningCompaction (#4383) add fe552af0dd Merge branch '2.1' new f1914da45a Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/core/util/compaction/RunningCompaction.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f1914da45a73f6010b5eba5827a22e90b94557be Merge: 9550da7c83 fe552af0dd Author: Dave Marion AuthorDate: Fri Mar 15 17:23:43 2024 + Merge branch 'main' into elasticity .../apache/accumulo/core/util/compaction/RunningCompaction.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit fe552af0dd23d263aca96dd6d356df7070641b29 Merge: 9839e4d42f 7c9204274c Author: Dave Marion AuthorDate: Fri Mar 15 17:22:56 2024 + Merge branch '2.1' .../apache/accumulo/core/util/compaction/RunningCompaction.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) branch main updated (9839e4d42f -> fe552af0dd)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9839e4d42f Merge branch '2.1' add 7c9204274c Fixes ConcurrentModificationException in RunningCompaction (#4383) new fe552af0dd Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/core/util/compaction/RunningCompaction.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) branch 2.1 updated: Fixes ConcurrentModificationException in RunningCompaction (#4383)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 7c9204274c Fixes ConcurrentModificationException in RunningCompaction
(#4383)
7c9204274c is described below
commit 7c9204274c195e258c6f45dbfe93e9d720533929
Author: Dave Marion
AuthorDate: Fri Mar 15 13:21:33 2024 -0400
Fixes ConcurrentModificationException in RunningCompaction (#4383)
While working on #4382 I ran into an issue where a CME was being
raised in a Thrift thread that was trying to serialize the updates
from the RunningCompaction object.
---
.../apache/accumulo/core/util/compaction/RunningCompaction.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/RunningCompaction.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/RunningCompaction.java
index b2e4fd1581..4d666e4962 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/RunningCompaction.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/RunningCompaction.java
@@ -43,11 +43,15 @@ public class RunningCompaction {
}
public Map getUpdates() {
-return updates;
+synchronized (updates) {
+ return new TreeMap<>(updates);
+}
}
public void addUpdate(Long timestamp, TCompactionStatusUpdate update) {
-this.updates.put(timestamp, update);
+synchronized (updates) {
+ this.updates.put(timestamp, update);
+}
}
public TExternalCompactionJob getJob() {
(accumulo) branch main updated (23e17129de -> 9839e4d42f)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 23e17129de Revert #4358 - Replace long + TimeUnit with Duration in ReadOnlyTStore.unreserve() add 05c2f45042 Reduced warning logs under normal conditions in compaction coordinator (#4362) new 9839e4d42f Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../coordinator/CompactionCoordinator.java | 32 -- .../accumulo/coordinator/QueueSummaries.java | 8 ++ 2 files changed, 37 insertions(+), 3 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9839e4d42fae9bd20b1529ca0e5e2fed4122
Merge: 23e17129de 05c2f45042
Author: Dave Marion
AuthorDate: Wed Mar 13 19:52:45 2024 +
Merge branch '2.1'
.../coordinator/CompactionCoordinator.java | 32 --
.../accumulo/coordinator/QueueSummaries.java | 8 ++
2 files changed, 37 insertions(+), 3 deletions(-)
diff --cc
server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index d5ecbf9ddd,b0ec498a9e..685080c5b1
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@@ -22,8 -23,7 +22,9 @@@ import static com.google.common.util.co
import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+ import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@@ -321,35 -325,58 +325,57 @@@ public class CompactionCoordinator exte
LOG.info("Shutting down");
}
+ private Map> getIdleCompactors() {
+
+ Map> allCompactors =
+ ExternalCompactionUtil.getCompactorAddrs(getContext());
+
+ Set emptyQueues = new HashSet<>();
+
+ // Remove all of the compactors that are running a compaction
+ RUNNING_CACHE.values().forEach(rc -> {
+ List busyCompactors = allCompactors.get(rc.getQueueName());
+ if (busyCompactors != null
+ &&
busyCompactors.remove(HostAndPort.fromString(rc.getCompactorAddress( {
+ if (busyCompactors.isEmpty()) {
+ emptyQueues.add(rc.getQueueName());
+ }
+ }
+ });
+ // Remove entries with empty queues
+ emptyQueues.forEach(e -> allCompactors.remove(e));
+ return allCompactors;
+ }
+
private void updateSummaries() {
-ExecutorService executor =
ThreadPools.getServerThreadPools().createFixedThreadPool(10,
-"Compaction Summary Gatherer", false);
-try {
- Set queuesSeen = new ConcurrentSkipListSet<>();
- tserverSet.getCurrentServers().forEach(tsi -> {
-executor.execute(() -> updateSummaries(tsi, queuesSeen));
- });
+final ArrayList> tasks = new ArrayList<>();
+Set queuesSeen = new ConcurrentSkipListSet<>();
- executor.shutdown();
+tserverSet.getCurrentServers().forEach(tsi -> {
+ tasks.add(summariesExecutor.submit(() -> updateSummaries(tsi,
queuesSeen)));
+});
- try {
-while (!executor.awaitTermination(1, TimeUnit.MINUTES)) {}
- } catch (InterruptedException e) {
-Thread.currentThread().interrupt();
-throw new RuntimeException(e);
+// Wait for all tasks to complete
+while (!tasks.isEmpty()) {
+ Iterator> iter = tasks.iterator();
+ while (iter.hasNext()) {
+Future f = iter.next();
+if (f.isDone()) {
+ iter.remove();
+}
}
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+}
- // remove any queues that were seen in the past, but were not seen in
the latest gathering of
- // summaries
- TIME_COMPACTOR_LAST_CHECKED.keySet().retainAll(queuesSeen);
+// remove any queues that were seen in the past, but were not seen in the
latest gathering of
+// summaries
+TIME_COMPACTOR_LAST_CHECKED.keySet().retainAll(queuesSeen);
- // add any queues that were never seen before
- queuesSeen.forEach(q -> {
-TIME_COMPACTOR_LAST_CHECKED.computeIfAbsent(q, k ->
System.currentTimeMillis());
- });
-} finally {
- executor.shutdownNow();
-}
+// add any queues that were never seen before
+queuesSeen.forEach(q -> {
+ TIME_COMPACTOR_LAST_CHECKED.computeIfAbsent(q, k ->
System.currentTimeMillis());
+});
}
private void updateSummaries(TServerInstance tsi, Set queuesSeen) {
(accumulo) branch 2.1 updated: Reduced warning logs under normal conditions in compaction coordinator (#4362)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 05c2f45042 Reduced warning logs under normal conditions in compaction
coordinator (#4362)
05c2f45042 is described below
commit 05c2f45042ee91a5fe04702caa77ab19f78c0f9a
Author: Dave Marion
AuthorDate: Wed Mar 13 11:52:47 2024 -0400
Reduced warning logs under normal conditions in compaction coordinator
(#4362)
Fixes #4219
---
.../coordinator/CompactionCoordinator.java | 32 --
.../accumulo/coordinator/QueueSummaries.java | 8 ++
.../coordinator/CompactionCoordinatorTest.java | 6
3 files changed, 43 insertions(+), 3 deletions(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index f4819ebefb..b0ec498a9e 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -23,6 +23,7 @@ import static
org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -303,9 +304,12 @@ public class CompactionCoordinator extends AbstractServer
updateSummaries();
long now = System.currentTimeMillis();
- TIME_COMPACTOR_LAST_CHECKED.forEach((k, v) -> {
-if ((now - v) > getMissingCompactorWarningTime()) {
- LOG.warn("No compactors have checked in with coordinator for queue
{} in {}ms", k,
+
+ Map> idleCompactors = getIdleCompactors();
+ TIME_COMPACTOR_LAST_CHECKED.forEach((queue, lastCheckTime) -> {
+if ((now - lastCheckTime) > getMissingCompactorWarningTime()
+&& QUEUE_SUMMARIES.isCompactionsQueued(queue) &&
idleCompactors.containsKey(queue)) {
+ LOG.warn("No compactors have checked in with coordinator for queue
{} in {}ms", queue,
getMissingCompactorWarningTime());
}
});
@@ -321,6 +325,28 @@ public class CompactionCoordinator extends AbstractServer
LOG.info("Shutting down");
}
+ private Map> getIdleCompactors() {
+
+Map> allCompactors =
+ExternalCompactionUtil.getCompactorAddrs(getContext());
+
+Set emptyQueues = new HashSet<>();
+
+// Remove all of the compactors that are running a compaction
+RUNNING_CACHE.values().forEach(rc -> {
+ List busyCompactors = allCompactors.get(rc.getQueueName());
+ if (busyCompactors != null
+ &&
busyCompactors.remove(HostAndPort.fromString(rc.getCompactorAddress( {
+if (busyCompactors.isEmpty()) {
+ emptyQueues.add(rc.getQueueName());
+}
+ }
+});
+// Remove entries with empty queues
+emptyQueues.forEach(e -> allCompactors.remove(e));
+return allCompactors;
+ }
+
private void updateSummaries() {
ExecutorService executor =
ThreadPools.getServerThreadPools().createFixedThreadPool(10,
"Compaction Summary Gatherer", false);
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
index 6edb2c0f36..1d89cd0321 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
@@ -100,6 +100,14 @@ public class QueueSummaries {
}
}
+ synchronized boolean isCompactionsQueued(String queue) {
+var q = QUEUES.get(queue);
+if (q == null) {
+ return false;
+}
+return !q.isEmpty();
+ }
+
synchronized PrioTserver getNextTserver(String queue) {
Entry> entry = getNextTserverEntry(queue);
diff --git
a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
index 117d50108a..87e7471bef 100644
---
a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
+++
b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
@@ -214,6 +214,7 @@ public class CompactionCoordinatorTest {
var coordinator = new TestCoordinator(null, null, null
(accumulo) branch elasticity updated: Resolved elasticity TODOs in Manager (#4365)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 75b717ed5f Resolved elasticity TODOs in Manager (#4365)
75b717ed5f is described below
commit 75b717ed5f0a7604d6cc25f0ce2a069d9335907b
Author: Dave Marion
AuthorDate: Wed Mar 13 08:28:15 2024 -0400
Resolved elasticity TODOs in Manager (#4365)
Removed the TODO for the bulkImports as it is still being set by
BulkImport V2 and referenced in the Monitor. Implemented the
suggestion in the other TODO.
Co-authored-by: Keith Turner
---
.../src/main/java/org/apache/accumulo/manager/Manager.java| 11 ---
.../java/org/apache/accumulo/manager/TabletGroupWatcher.java | 2 +-
2 files changed, 5 insertions(+), 8 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 6758acfc1f..45ab2b5e2b 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@ -53,6 +53,7 @@ import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.accumulo.core.Constants;
@@ -226,7 +227,6 @@ public class Manager extends AbstractServer
volatile SortedMap tserverStatusForBalancer =
emptySortedMap();
volatile Map> tServerGroupingForBalancer =
emptyMap();
- // ELASTICITY_TODO is this still needed?
final ServerBulkImportStatus bulkImportStatus = new ServerBulkImportStatus();
private final AtomicBoolean managerInitialized = new AtomicBoolean(false);
@@ -243,14 +243,11 @@ public class Manager extends AbstractServer
return state;
}
- // ELASTICITIY_TODO it would be nice if this method could take DataLevel as
an argument and only
- // retrieve information about compactions in that data level. Attempted this
and a lot of
- // refactoring was needed to get that small bit of information to this
method. Would be best to
- // address this after issue. May be best to attempt this after #3576.
- public Map> getCompactionHints() {
+ public Map> getCompactionHints(DataLevel level) {
+Predicate tablePredicate = (tableId) -> DataLevel.of(tableId) ==
level;
Map allConfig;
try {
- allConfig = CompactionConfigStorage.getAllConfig(getContext(), tableId
-> true);
+ allConfig = CompactionConfigStorage.getAllConfig(getContext(),
tablePredicate);
} catch (InterruptedException | KeeperException e) {
throw new RuntimeException(e);
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index e7ea20413a..bd177c2deb 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -340,7 +340,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
return new TabletManagementParameters(manager.getManagerState(),
parentLevelUpgrade,
manager.onlineTables(), tServersSnapshot, shutdownServers,
manager.migrationsSnapshot(),
-store.getLevel(), manager.getCompactionHints(), canSuspendTablets(),
+store.getLevel(), manager.getCompactionHints(store.getLevel()),
canSuspendTablets(),
lookForTabletsNeedingVolReplacement ?
manager.getContext().getVolumeReplacements()
: Map.of());
}
(accumulo) branch elasticity updated: Removed special handling logic in TabletManagementIterator (#4363)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new b4a2ac45b6 Removed special handling logic in TabletManagementIterator
(#4363)
b4a2ac45b6 is described below
commit b4a2ac45b69f2b7b6f052e029354cfebb2e66b9a
Author: Dave Marion
AuthorDate: Wed Mar 13 07:38:47 2024 -0400
Removed special handling logic in TabletManagementIterator (#4363)
Removed the logic that always returns the TabletMetadata
when the Manager state is not normal, or there are no
tablet servers, or no online tables. The code now just
calls computeTabletManagementActions in all cases.
Closes #4256
---
.../org/apache/accumulo/core/metadata/TabletState.java | 4
.../accumulo/server/manager/state/TabletGoalState.java | 3 +++
.../server/manager/state/TabletManagementIterator.java | 14 +-
3 files changed, 4 insertions(+), 17 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java
index 9fcf8add3f..ba182514d1 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java
@@ -21,14 +21,10 @@ package org.apache.accumulo.core.metadata;
import java.util.Set;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public enum TabletState {
UNASSIGNED, ASSIGNED, HOSTED, ASSIGNED_TO_DEAD_SERVER, SUSPENDED;
- private static Logger log = LoggerFactory.getLogger(TabletState.class);
-
public static TabletState compute(TabletMetadata tm, Set
liveTServers) {
TabletMetadata.Location current = null;
TabletMetadata.Location future = null;
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
index 81e796608c..0b9b83f159 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java
@@ -103,6 +103,9 @@ public enum TabletGoalState {
if (!tm.getHostingRequested()) {
return UNASSIGNED;
}
+break;
+ default:
+break;
}
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
index 2e6627c78e..39329b0e42 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java
@@ -43,7 +43,6 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.manager.state.TabletManagement;
import
org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction;
-import org.apache.accumulo.core.manager.thrift.ManagerState;
import org.apache.accumulo.core.metadata.TabletState;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletOperationType;
@@ -200,18 +199,7 @@ public class TabletManagementIterator extends
SkippingIterator {
Exception error = null;
try {
LOG.trace("Evaluating extent: {}", tm);
-if (tm.getExtent().isMeta()) {
- computeTabletManagementActions(tm, actions);
-} else {
- if (tabletMgmtParams.getManagerState() != ManagerState.NORMAL
- || tabletMgmtParams.getOnlineTsevers().isEmpty()
- || tabletMgmtParams.getOnlineTables().isEmpty()) {
-// when manager is in the process of starting up or shutting down
return everything.
-actions.add(ManagementAction.NEEDS_LOCATION_UPDATE);
- } else {
-computeTabletManagementActions(tm, actions);
- }
-}
+computeTabletManagementActions(tm, actions);
} catch (Exception e) {
LOG.error("Error computing tablet management actions for extent: {}",
tm.getExtent(), e);
error = e;
(accumulo-access) branch main updated: Modified Authorizations.of to only accept a Set (#68)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-access.git
The following commit(s) were added to refs/heads/main by this push:
new 33be659 Modified Authorizations.of to only accept a Set (#68)
33be659 is described below
commit 33be6595160d0f62041731a720cc2b80f898c243
Author: Dave Marion
AuthorDate: Tue Mar 12 13:47:23 2024 -0400
Modified Authorizations.of to only accept a Set (#68)
Closes #66
-
Co-authored-by: Keith Turner
---
.../antlr/AccessExpressionAntlrBenchmark.java | 3 +-
.../accumulo/access/grammar/antlr/Antlr4Tests.java | 7 ++--
.../apache/accumulo/access/AccessEvaluator.java| 7
.../accumulo/access/AccessEvaluatorImpl.java | 8
.../accumulo/access/AccessExpressionImpl.java | 2 +-
.../org/apache/accumulo/access/Authorizations.java | 21 ++
src/test/java/example/AccessExample.java | 4 +-
.../accumulo/access/AccessEvaluatorTest.java | 18 +
.../accumulo/access/AccessExpressionBenchmark.java | 7 ++--
.../apache/accumulo/access/AuthorizationTest.java | 45 ++
10 files changed, 84 insertions(+), 38 deletions(-)
diff --git
a/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
b/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
index 0c3422f..75ec0a7 100644
---
a/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
+++
b/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/AccessExpressionAntlrBenchmark.java
@@ -25,6 +25,7 @@ import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -86,7 +87,7 @@ public class AccessExpressionAntlrBenchmark {
et.expressions = new ArrayList<>();
et.evaluator = new AccessExpressionAntlrEvaluator(
-
Stream.of(testDataSet.auths).map(Authorizations::of).collect(Collectors.toList()));
+Stream.of(testDataSet.auths).map(a ->
Authorizations.of(Set.of(a))).collect(Collectors.toList()));
for (var tests : testDataSet.tests) {
if (tests.expectedResult != TestDataLoader.ExpectedResult.ERROR) {
diff --git
a/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/Antlr4Tests.java
b/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/Antlr4Tests.java
index be40f3b..09870ae 100644
---
a/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/Antlr4Tests.java
+++
b/src/it/antlr4-example/src/test/java/org/apache/accumulo/access/grammar/antlr/Antlr4Tests.java
@@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -120,7 +121,7 @@ public class Antlr4Tests {
@Test
public void testSimpleEvaluation() throws Exception {
String accessExpression = "(one)|(foo)";
-Authorizations auths = Authorizations.of("four", "three", "one", "two");
+Authorizations auths = Authorizations.of(Set.of("four", "three", "one",
"two"));
AccessExpressionAntlrEvaluator eval = new
AccessExpressionAntlrEvaluator(List.of(auths));
assertTrue(eval.canAccess(accessExpression));
}
@@ -128,7 +129,7 @@ public class Antlr4Tests {
@Test
public void testSimpleEvaluationFailure() throws Exception {
String accessExpression = "(A)";
-Authorizations auths = Authorizations.of("A", "C");
+Authorizations auths = Authorizations.of(Set.of("A", "C"));
AccessExpressionAntlrEvaluator eval = new
AccessExpressionAntlrEvaluator(List.of(auths));
assertFalse(eval.canAccess(accessExpression));
}
@@ -141,7 +142,7 @@ public class Antlr4Tests {
for (TestDataSet testSet : testData) {
List authSets =
-
Stream.of(testSet.auths).map(Authorizations::of).collect(Collectors.toList());
+ Stream.of(testSet.auths).map(a ->
Authorizations.of(Set.of(a))).collect(Collectors.toList());
AccessEvaluator evaluator = AccessEvaluator.of(authSets);
AccessExpressionAntlrEvaluator antlr = new
AccessExpressionAntlrEvaluator(authSets);
diff --git a/src/main/java/org/apache/accumulo/access/AccessEvaluator.java
b/src/main/java/org/apache/accumulo/access/AccessEvaluator.java
index 1ea4d23..3eff0d1 100644
--- a/src/main/java/org/apache/accumu
(accumulo) branch elasticity updated: Refactor classes to use the Caches object (#4359)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 7ea10de6c8 Refactor classes to use the Caches object (#4359)
7ea10de6c8 is described below
commit 7ea10de6c8af6a8c79e0ae6aadcc491566e75cee
Author: Dave Marion
AuthorDate: Tue Mar 12 08:10:10 2024 -0400
Refactor classes to use the Caches object (#4359)
---
.../java/org/apache/accumulo/core/util/cache/Caches.java | 7 ++-
.../accumulo/server/compaction/CompactionJobGenerator.java | 4 ++--
.../accumulo/server/conf/ServerConfigurationFactory.java | 14 --
.../org/apache/accumulo/server/fs/VolumeManagerImpl.java | 6 --
4 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
b/core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
index a96af36bc5..f5ef8e4c8f 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
@@ -42,12 +42,14 @@ public class Caches implements MetricsProducer {
COMPACTION_CONFIGS,
COMPACTION_DIR_CACHE,
COMPACTION_DISPATCHERS,
+COMPACTION_SERVICE_UNKNOWN,
COMPACTOR_GROUP_ID,
COMPRESSION_ALGORITHM,
CRYPT_PASSWORDS,
HOST_REGEX_BALANCER_TABLE_REGEX,
INSTANCE_ID,
NAMESPACE_ID,
+NAMESPACE_CONFIGS,
PROP_CACHE,
RECOVERY_MANAGER_PATH_CACHE,
SCAN_SERVER_TABLET_METADATA,
@@ -56,10 +58,13 @@ public class Caches implements MetricsProducer {
SPLITTER_FILES,
SPLITTER_STARTING,
SPLITTER_UNSPLITTABLE,
+TABLE_CONFIGS,
TABLE_ID,
+TABLE_PARENT_CONFIGS,
TABLE_ZOO_HELPER_CACHE,
TSRM_FILE_LENGTHS,
-TINYLFU_BLOCK_CACHE;
+TINYLFU_BLOCK_CACHE,
+VOLUME_HDFS_CONFIGS;
}
private static final Logger LOG = LoggerFactory.getLogger(Caches.class);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
index 02e3dc2fca..1d88de2eaa 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
@@ -55,7 +55,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
public class CompactionJobGenerator {
private static final Logger log =
LoggerFactory.getLogger(CompactionJobGenerator.class);
@@ -86,7 +85,8 @@ public class CompactionJobGenerator {
v.isEmpty() ? Map.of() : Collections.unmodifiableMap(v)));
}
unknownCompactionServiceErrorCache =
-Caffeine.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build();
+
Caches.getInstance().createNewBuilder(CacheName.COMPACTION_SERVICE_UNKNOWN,
false)
+.expireAfterWrite(5, TimeUnit.MINUTES).build();
}
public Collection generateJobs(TabletMetadata tablet,
Set kinds) {
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java
b/server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java
index 78b1e4f18b..c6f32946c3 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/conf/ServerConfigurationFactory.java
@@ -36,6 +36,8 @@ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.util.cache.Caches;
+import org.apache.accumulo.core.util.cache.Caches.CacheName;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.accumulo.server.ServerContext;
@@ -49,7 +51,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -80,11 +81,12 @@ public class ServerConfigurationFactory extends
ServerConfiguration {
this.systemConfig = memoize(() -> new SystemConfiguration(context,
SystemPropKey.of(context.getInstanceID()), siteConfig));
tableParentConfigs =
-Caffeine.newBuilder().expireAfterAccess(CACHE_EXPIRATION_HRS,
TimeUnit.HOURS).build();
-tableConfigs =
-Caffeine.newBuilder().expireAfterAccess(CACHE_EXPIRATION_HRS,
TimeUnit.HOURS).build();
-namespaceConf
(accumulo) 02/02: Merge remote-tracking branch 'upstream/elasticity' into elasticity
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1688cecd5a8889bcb41cec2bdb45d2f063d1d9a9 Merge: 3849fffedd 202198a588 Author: Dave Marion AuthorDate: Mon Mar 4 20:15:25 2024 + Merge remote-tracking branch 'upstream/elasticity' into elasticity .../apache/accumulo/core/logging/TabletLogger.java | 49 ++- .../coordinator/CompactionCoordinator.java | 27 +- .../coordinator/commit/CommitCompaction.java | 4 + .../manager/tableOps/compact/CompactionDriver.java | 22 +- .../manager/tableOps/split/UpdateTablets.java | 42 ++- .../compaction/CompactionCoordinatorTest.java | 148 ++ .../manager/tableOps/split/UpdateTabletsTest.java | 328 + .../apache/accumulo/test/functional/SplitIT.java | 54 test/src/main/resources/log4j2-test.properties | 3 + 9 files changed, 636 insertions(+), 41 deletions(-)
(accumulo) branch elasticity updated (202198a588 -> 1688cecd5a)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 202198a588 Updates compaction to use TabletLogger (#4333) add d91d016211 Optimized logic for getting a random TabletServer connection (#4309) add f3d5fb01d7 Merge branch '2.1' new 3849fffedd Merge branch 'main' into elasticity new 1688cecd5a Merge remote-tracking branch 'upstream/elasticity' into elasticity The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/clientImpl/ThriftTransportKey.java| 29 -- .../core/clientImpl/ThriftTransportPool.java | 110 + .../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +- .../accumulo/core/rpc/clients/TServerClient.java | 105 +++- .../core/rpc/clients/ThriftClientTypes.java| 6 +- .../core/clientImpl/ThriftTransportKeyTest.java| 25 ++--- .../apache/accumulo/test/TransportCachingIT.java | 42 .../test/functional/MemoryStarvedScanIT.java | 41 +--- 8 files changed, 182 insertions(+), 183 deletions(-)
(accumulo) 01/02: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3849fffeddeb29ec92b74926c15e756f182bb7c3
Merge: 422d48a432 f3d5fb01d7
Author: Dave Marion
AuthorDate: Mon Mar 4 20:14:59 2024 +
Merge branch 'main' into elasticity
.../core/clientImpl/ThriftTransportKey.java| 29 --
.../core/clientImpl/ThriftTransportPool.java | 110 +
.../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +-
.../accumulo/core/rpc/clients/TServerClient.java | 105 +++-
.../core/rpc/clients/ThriftClientTypes.java| 6 +-
.../core/clientImpl/ThriftTransportKeyTest.java| 25 ++---
.../apache/accumulo/test/TransportCachingIT.java | 42
.../test/functional/MemoryStarvedScanIT.java | 41 +---
8 files changed, 182 insertions(+), 183 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
index 8273f8e5b8,59b9a535b8..0becd57120
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedScanIT.java
@@@ -31,6 -30,6 +30,7 @@@ import java.util.Iterator
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
++import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.DoubleAdder;
@@@ -41,23 -39,12 +41,21 @@@ import org.apache.accumulo.core.client.
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.clientImpl.ClientContext;
- import org.apache.accumulo.core.clientImpl.ThriftTransportKey;
+import org.apache.accumulo.core.clientImpl.thrift.ClientService.Client;
+import org.apache.accumulo.core.clientImpl.thrift.TInfo;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.fate.zookeeper.ZooCache;
import org.apache.accumulo.core.iterators.WrappingIterator;
+import org.apache.accumulo.core.lock.ServiceLock;
- import org.apache.accumulo.core.lock.ServiceLock.ServiceLockPath;
++import org.apache.accumulo.core.lock.ServiceLockData;
+import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
import org.apache.accumulo.core.metrics.MetricsProducer;
+import org.apache.accumulo.core.rpc.ThriftUtil;
+import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
- import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
import org.apache.accumulo.harness.SharedMiniClusterBase;
import org.apache.accumulo.minicluster.MemoryUnit;
@@@ -67,7 -54,6 +65,8 @@@ import org.apache.accumulo.test.metrics
import org.apache.accumulo.test.metrics.TestStatsDSink;
import org.apache.accumulo.test.metrics.TestStatsDSink.Metric;
import org.apache.hadoop.conf.Configuration;
+import org.apache.thrift.transport.TTransport;
++import org.apache.thrift.transport.TTransportException;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@@ -75,6 -61,6 +74,8 @@@ import org.junit.jupiter.api.Test
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
++import com.google.common.net.HostAndPort;
++
public class MemoryStarvedScanIT extends SharedMiniClusterBase {
public static class MemoryStarvedITConfiguration implements
MiniClusterConfigurationCallback {
@@@ -187,25 -173,10 +188,35 @@@
}
static void freeServerMemory(AccumuloClient client) throws Exception {
-// Instantiating this class on the TabletServer will free the memory as it
-// frees the buffers created by the MemoryConsumingIterator in its
constructor.
-
client.instanceOperations().testClassLoad(MemoryFreeingIterator.class.getName(),
-WrappingIterator.class.getName());
+
++// This does not call ThriftClientTypes.CLIENT.execute because
++// we only want to communicate with the TabletServer for this test
+final ClientContext context = (ClientContext) client;
+final long rpcTimeout = context.getClientTimeoutInMillis();
- final ArrayList servers = new ArrayList<>();
+final String serverPath = context.getZooKeeperRoot() +
Constants.ZTSERVERS;
+final ZooCache zc = context.getZooCache();
+
+for (String server : zc.getChildren(serverPath)) {
- ServiceLockPath zLocPath = ServiceLock.path(serverPath + "/" + server);
- zc.getLockData(zLocPath).map(sld ->
sld.getAddress(ThriftService.CLIENT))
- .map(address -> new ThriftTransportKey(address, rpcTimeout,
context))
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f3d5fb01d701a6932d37d2f67f52cc0eefa64d50
Merge: ebf7054d1f d91d016211
Author: Dave Marion
AuthorDate: Mon Mar 4 18:14:16 2024 +
Merge branch '2.1'
.../core/clientImpl/ThriftTransportKey.java| 29 --
.../core/clientImpl/ThriftTransportPool.java | 110 +
.../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +-
.../accumulo/core/rpc/clients/TServerClient.java | 105 +++-
.../core/rpc/clients/ThriftClientTypes.java| 6 +-
.../core/clientImpl/ThriftTransportKeyTest.java| 25 ++---
.../coordinator/CompactionCoordinator.java | 4 +-
.../apache/accumulo/test/TransportCachingIT.java | 42
8 files changed, 158 insertions(+), 170 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
index f332a09492,f4c7047d6d..0f84154a15
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
@@@ -24,9 -24,10 +24,10 @@@ import java.util.Objects
import org.apache.accumulo.core.rpc.SaslConnectionParams;
import org.apache.accumulo.core.rpc.SslConnectionParams;
+ import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
-import org.apache.accumulo.core.util.HostAndPort;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
@VisibleForTesting
public class ThriftTransportKey {
@@@ -54,12 -58,18 +58,18 @@@
this.saslParams = saslParams;
if (saslParams != null && sslParams != null) {
// TSasl and TSSL transport factories don't play nicely together
- throw new RuntimeException("Cannot use both SSL and SASL thrift
transports");
+ throw new IllegalArgumentException("Cannot use both SSL and SASL thrift
transports");
}
- this.hash = Objects.hash(server, timeout, sslParams, saslParams);
+ this.hash = Objects.hash(type, server, timeout, sslParams, saslParams);
}
- HostAndPort getServer() {
+ @VisibleForTesting
+ public ThriftClientTypes getType() {
+ return type;
+ }
+
+ @VisibleForTesting
+ public HostAndPort getServer() {
return server;
}
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
index d1bc17e945,a3d38aa10a..b3f205fa2a
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
@@@ -41,6 -41,8 +41,7 @@@ import java.util.function.LongSupplier
import java.util.function.Supplier;
import org.apache.accumulo.core.rpc.ThriftUtil;
+ import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
-import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.thrift.TConfiguration;
@@@ -49,9 -51,7 +50,8 @@@ import org.apache.thrift.transport.TTra
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.net.HostAndPort;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@@ -109,71 -110,40 +109,40 @@@ public class ThriftTransportPool
return pool;
}
- public TTransport getTransport(HostAndPort location, long milliseconds,
ClientContext context)
- throws TTransportException {
- ThriftTransportKey cacheKey = new ThriftTransportKey(location,
milliseconds, context);
+ public TTransport getTransport(ThriftClientTypes type, HostAndPort
location, long milliseconds,
+ ClientContext context, boolean preferCached) throws TTransportException
{
- CachedConnection connection = connectionPool.reserveAny(cacheKey);
-
- if (connection != null) {
- log.trace("Using existing connection to {}", cacheKey.getServer());
- return connection.transport;
- } else {
- return createNewTransport(cacheKey);
+ ThriftTransportKey cacheKey = new ThriftTransportKey(type, location,
milliseconds, context);
+ if (preferCached) {
+ CachedConnection connection = connectionPool.reserveAny(cacheKey);
+ if (connection != null) {
+ log.trace("Using existing connection to {}", cacheKey.getServer());
+ return connection.transport;
+ }
}
+ return createNewTransport(cacheKey);
}
- @VisibleForTesting
- public Pair getAnyTransport(List
servers,
- boolean preferCachedConnection) throws TTransportException {
-
- servers = new ArrayList<>(servers);
-
- if (preferCachedConnection
(accumulo) branch main updated (ebf7054d1f -> f3d5fb01d7)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from ebf7054d1f Merge branch '2.1' add d91d016211 Optimized logic for getting a random TabletServer connection (#4309) new f3d5fb01d7 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/clientImpl/ThriftTransportKey.java| 29 -- .../core/clientImpl/ThriftTransportPool.java | 110 + .../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +- .../accumulo/core/rpc/clients/TServerClient.java | 105 +++- .../core/rpc/clients/ThriftClientTypes.java| 6 +- .../core/clientImpl/ThriftTransportKeyTest.java| 25 ++--- .../coordinator/CompactionCoordinator.java | 4 +- .../apache/accumulo/test/TransportCachingIT.java | 42 8 files changed, 158 insertions(+), 170 deletions(-)
(accumulo) branch 2.1 updated: Optimized logic for getting a random TabletServer connection (#4309)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d91d016211 Optimized logic for getting a random TabletServer
connection (#4309)
d91d016211 is described below
commit d91d0162115ae66112a104278bcd14e8085936d3
Author: Dave Marion
AuthorDate: Mon Mar 4 09:16:58 2024 -0500
Optimized logic for getting a random TabletServer connection (#4309)
The previous logic in this class would gather all of the Tserver
ZNodes in ZooKeeper, then get the data for each ZNode and validate
their ServiceLock. Then, after all of that it would randomly pick
one of the TabletServers to connect to. It did this through the
ZooCache object which on an initial connection would be empty and
causes a lot of back and forth to ZooKeeper. The side effect of
this is that the ZooCache would be populated with TabletServer
information.
This change modifies TServerClient such that it no longer populates
ZooCache information for each TabletServer and modifies the
default logic for getting a connection to a TabletServer. The new
logic will make 3 calls to ZooKeeper in the best case scenario, one to get
the list of TServer ZNodes in Zookeeper, one to get the ServiceLock for
a random TServer and another to get the ZNode data for one of it. This
is all done through ZooCache, so it is lazily populated over time instead
of incurring the penalty when getting the first TabletServer connection.
Fixes #4303
---
.../core/clientImpl/ThriftTransportKey.java| 29 --
.../core/clientImpl/ThriftTransportPool.java | 110 +
.../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +-
.../accumulo/core/rpc/clients/TServerClient.java | 73 --
.../core/rpc/clients/ThriftClientTypes.java| 6 +-
.../core/clientImpl/ThriftTransportKeyTest.java| 25 ++---
.../coordinator/CompactionCoordinator.java | 4 +-
.../apache/accumulo/test/TransportCachingIT.java | 42
8 files changed, 142 insertions(+), 154 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
index 8be320dcc5..f4c7047d6d 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
@@ -24,12 +24,14 @@ import java.util.Objects;
import org.apache.accumulo.core.rpc.SaslConnectionParams;
import org.apache.accumulo.core.rpc.SslConnectionParams;
+import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.util.HostAndPort;
import com.google.common.annotations.VisibleForTesting;
@VisibleForTesting
public class ThriftTransportKey {
+ private final ThriftClientTypes type;
private final HostAndPort server;
private final long timeout;
private final SslConnectionParams sslParams;
@@ -38,16 +40,18 @@ public class ThriftTransportKey {
private final int hash;
@VisibleForTesting
- public ThriftTransportKey(HostAndPort server, long timeout, ClientContext
context) {
-this(server, timeout, context.getClientSslParams(),
context.getSaslParams());
+ public ThriftTransportKey(ThriftClientTypes type, HostAndPort server,
long timeout,
+ ClientContext context) {
+this(type, server, timeout, context.getClientSslParams(),
context.getSaslParams());
}
/**
* Visible only for testing
*/
- ThriftTransportKey(HostAndPort server, long timeout, SslConnectionParams
sslParams,
- SaslConnectionParams saslParams) {
+ ThriftTransportKey(ThriftClientTypes type, HostAndPort server, long
timeout,
+ SslConnectionParams sslParams, SaslConnectionParams saslParams) {
requireNonNull(server, "location is null");
+this.type = type;
this.server = server;
this.timeout = timeout;
this.sslParams = sslParams;
@@ -56,14 +60,21 @@ public class ThriftTransportKey {
// TSasl and TSSL transport factories don't play nicely together
throw new RuntimeException("Cannot use both SSL and SASL thrift
transports");
}
-this.hash = Objects.hash(server, timeout, sslParams, saslParams);
+this.hash = Objects.hash(type, server, timeout, sslParams, saslParams);
}
- HostAndPort getServer() {
+ @VisibleForTesting
+ public ThriftClientTypes getType() {
+return type;
+ }
+
+ @VisibleForTesting
+ public HostAndPort getServer() {
return server;
}
- long getTimeout() {
+ @VisibleForTesting
+ public long getTimeout() {
return timeout;
}
@@ -81,7 +92,7 @@ public class ThriftTransportKey {
return false;
}
ThriftTransportKey ttk = (T
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 422d48a4325bb11b9aaae1ec8c61bdae4afca214
Merge: afe2857935 ebf7054d1f
Author: Dave Marion
AuthorDate: Mon Mar 4 12:51:34 2024 +
Merge branch 'main' into elasticity
assemble/bin/accumulo-cluster | 10 +-
.../accumulo/core/conf/cluster/ClusterConfigParser.java | 8 ++--
.../accumulo/core/conf/cluster/ClusterConfigParserTest.java | 11 ---
.../core/conf/cluster/cluster-with-optional-services.yaml | 3 ++-
4 files changed, 25 insertions(+), 7 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
index 0c465564d9,e00570154a..c4ebf8c7ec
---
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
@@@ -39,23 -39,13 +39,24 @@@ import edu.umd.cs.findbugs.annotations.
public class ClusterConfigParser {
private static final String PROPERTY_FORMAT = "%s=\"%s\"%n";
- private static final String[] SECTIONS = new String[] {"manager",
"monitor", "gc", "tserver"};
-
- private static final Set VALID_CONFIG_KEYS = Set.of("manager",
"monitor", "gc", "tserver",
- "tservers_per_host", "sservers_per_host", "compaction.coordinator",
"compactors_per_host");
+ private static final String COMPACTOR_PREFIX = "compactor.";
++ private static final String COMPACTORS_PER_HOST_KEY = "compactors_per_host";
+ private static final String GC_KEY = "gc";
+ private static final String MANAGER_KEY = "manager";
+ private static final String MONITOR_KEY = "monitor";
+ private static final String SSERVER_PREFIX = "sserver.";
+ private static final String SSERVERS_PER_HOST_KEY = "sservers_per_host";
+ private static final String TSERVER_PREFIX = "tserver.";
+ private static final String TSERVERS_PER_HOST_KEY = "tservers_per_host";
+
+ private static final String[] UNGROUPED_SECTIONS =
+ new String[] {MANAGER_KEY, MONITOR_KEY, GC_KEY};
+
- private static final Set VALID_CONFIG_KEYS =
- Set.of(MANAGER_KEY, MONITOR_KEY, GC_KEY, SSERVERS_PER_HOST_KEY,
TSERVERS_PER_HOST_KEY);
++ private static final Set VALID_CONFIG_KEYS = Set.of(MANAGER_KEY,
MONITOR_KEY, GC_KEY,
++ SSERVERS_PER_HOST_KEY, TSERVERS_PER_HOST_KEY, COMPACTORS_PER_HOST_KEY);
private static final Set VALID_CONFIG_PREFIXES =
- Set.of("compaction.compactor.", "sserver.");
+ Set.of(COMPACTOR_PREFIX, SSERVER_PREFIX, TSERVER_PREFIX);
private static final Predicate VALID_CONFIG_SECTIONS =
section -> VALID_CONFIG_KEYS.contains(section)
diff --cc
core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
index 148b5e4f24,1410dc569a..189e48afc3
---
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
@@@ -63,15 -63,17 +63,16 @@@ public class ClusterConfigParserTest
assertEquals("localhost1 localhost2", contents.get("monitor"));
assertTrue(contents.containsKey("gc"));
assertEquals("localhost", contents.get("gc"));
-assertTrue(contents.containsKey("tserver"));
-assertEquals("localhost1 localhost2 localhost3 localhost4",
contents.get("tserver"));
-assertFalse(contents.containsKey("compaction"));
-assertFalse(contents.containsKey("compaction.coordinator"));
-assertFalse(contents.containsKey("compaction.compactor"));
-assertFalse(contents.containsKey("compaction.compactor.queue"));
-assertFalse(contents.containsKey("compaction.compactor.q1"));
-assertFalse(contents.containsKey("compaction.compactor.q2"));
+assertFalse(contents.containsKey("tserver"));
+assertTrue(contents.containsKey("tserver.default"));
+assertEquals("localhost1 localhost2 localhost3 localhost4",
contents.get("tserver.default"));
+assertFalse(contents.containsKey("compactor"));
+assertFalse(contents.containsKey("compactor.queue"));
+assertFalse(contents.containsKey("compactor.q1"));
+assertFalse(contents.containsKey("compactor.q2"));
assertFalse(contents.containsKey("tservers_per_host"));
assertFalse(contents.containsKey("sservers_per_host"));
+ assertFalse(co
(accumulo) branch elasticity updated (afe2857935 -> 422d48a432)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from afe2857935 Merge branch 'main' into elasticity add 18b745466e Added compactors_per_host to accumulo-cluster script (#4329) add ebf7054d1f Merge branch '2.1' new 422d48a432 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/bin/accumulo-cluster | 10 +- .../accumulo/core/conf/cluster/ClusterConfigParser.java | 8 ++-- .../accumulo/core/conf/cluster/ClusterConfigParserTest.java | 11 --- .../core/conf/cluster/cluster-with-optional-services.yaml | 3 ++- 4 files changed, 25 insertions(+), 7 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ebf7054d1f6bf62d5170dcbbb07c7b0ee572cf2c Merge: c976af383f 18b745466e Author: Dave Marion AuthorDate: Mon Mar 4 12:34:51 2024 + Merge branch '2.1' assemble/bin/accumulo-cluster | 10 +- .../accumulo/core/conf/cluster/ClusterConfigParser.java | 5 - .../accumulo/core/conf/cluster/ClusterConfigParserTest.java | 11 --- .../core/conf/cluster/cluster-with-optional-services.yaml | 3 ++- 4 files changed, 23 insertions(+), 6 deletions(-)
(accumulo) branch main updated (c976af383f -> ebf7054d1f)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from c976af383f Merge branch '2.1' add 18b745466e Added compactors_per_host to accumulo-cluster script (#4329) new ebf7054d1f Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: assemble/bin/accumulo-cluster | 10 +- .../accumulo/core/conf/cluster/ClusterConfigParser.java | 5 - .../accumulo/core/conf/cluster/ClusterConfigParserTest.java | 11 --- .../core/conf/cluster/cluster-with-optional-services.yaml | 3 ++- 4 files changed, 23 insertions(+), 6 deletions(-)
(accumulo) branch 2.1 updated: Added compactors_per_host to accumulo-cluster script (#4329)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 18b745466e Added compactors_per_host to accumulo-cluster script (#4329)
18b745466e is described below
commit 18b745466eee10c41f72012908b867005ca89881
Author: Dave Marion
AuthorDate: Mon Mar 4 07:22:46 2024 -0500
Added compactors_per_host to accumulo-cluster script (#4329)
---
assemble/bin/accumulo-cluster | 10 +-
.../accumulo/core/conf/cluster/ClusterConfigParser.java | 5 -
.../accumulo/core/conf/cluster/ClusterConfigParserTest.java | 11 ---
.../core/conf/cluster/cluster-with-optional-services.yaml | 3 ++-
4 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster
index c9936cb78b..5dd9de7e4e 100755
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@ -119,6 +119,11 @@ function parse_config {
if [[ -z $NUM_SSERVERS ]]; then
echo "INFO: ${NUM_SSERVERS} sservers will be started per host"
fi
+
+ if [[ -z $NUM_COMPACTORS ]]; then
+echo "INFO: ${NUM_COMPACTORS} compactors will be started per host"
+ fi
+
}
function control_service() {
@@ -130,6 +135,7 @@ function control_service() {
last_instance_id=1
[[ $service == "tserver" ]] && last_instance_id=${NUM_TSERVERS:-1}
[[ $service == "sserver" ]] && last_instance_id=${NUM_SSERVERS:-1}
+ [[ $service == "compactor" ]] && last_instance_id=${NUM_COMPACTORS:-1}
for ((inst_id = 1; inst_id <= last_instance_id; inst_id++)); do
ACCUMULO_SERVICE_INSTANCE=""
@@ -510,10 +516,12 @@ tserver:
# to start on each host. If the following variables are not set, then they
default to 1.
# If the environment variable NUM_TSERVERS is set when running accumulo_cluster
# then its value will override what is set in this file for tservers_per_host.
Likewise if
-# NUM_SSERVERS is set then it will override sservers_per_host.
+# NUM_SSERVERS or NUM_COMPACTORS are set then it will override
sservers_per_host and
+# compactors_per_host.
#
tservers_per_host: 1
sservers_per_host: 1
+compactors_per_host: 1
EOF
;;
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
index 4f41cbef3e..5790fa7fd3 100644
---
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
@@ -42,7 +42,7 @@ public class ClusterConfigParser {
private static final String[] SECTIONS = new String[] {"manager", "monitor",
"gc", "tserver"};
private static final Set VALID_CONFIG_KEYS = Set.of("manager",
"monitor", "gc", "tserver",
- "tservers_per_host", "sservers_per_host", "compaction.coordinator");
+ "tservers_per_host", "sservers_per_host", "compaction.coordinator",
"compactors_per_host");
private static final Set VALID_CONFIG_PREFIXES =
Set.of("compaction.compactor.", "sserver.");
@@ -150,6 +150,9 @@ public class ClusterConfigParser {
String numSservers = config.getOrDefault("sservers_per_host", "1");
out.print("NUM_SSERVERS=\"${NUM_SSERVERS:=" + numSservers + "}\"\n");
+String numCompactors = config.getOrDefault("compactors_per_host", "1");
+out.print("NUM_COMPACTORS=\"${NUM_COMPACTORS:=" + numCompactors + "}\"\n");
+
out.flush();
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
index ef2c2382bc..1410dc569a 100644
---
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
@@ -73,6 +73,7 @@ public class ClusterConfigParserTest {
assertFalse(contents.containsKey("compaction.compactor.q2"));
assertFalse(contents.containsKey("tservers_per_host"));
assertFalse(contents.containsKey("sservers_per_host"));
+assertFalse(contents.containsKey("compactors_per_host"));
}
@Test
@@ -84,7 +85,7 @@ public class ClusterConfigParserTest {
Map contents =
ClusterConfigParser.parseConfiguration(new
File(configFile.toURI()).getAbsolutePath());
-assertEquals(12, con
(accumulo) branch elasticity updated: Fixed CompactionDriver after new column added to metadata (#4319)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 0528a71586 Fixed CompactionDriver after new column added to metadata
(#4319)
0528a71586 is described below
commit 0528a71586adcbf0696338ed7d32184ad7744b88
Author: Dave Marion
AuthorDate: Wed Feb 28 13:33:36 2024 -0500
Fixed CompactionDriver after new column added to metadata (#4319)
CompactionDriver.cleanupTabletMetadata was not fetching the
newly added metadata column USER_COMPACTION_REQUESTED causing
CompactionExecutorIT and ExternalCompaction_2_IT to fail. The
column was added in #4254.
---
.../apache/accumulo/manager/tableOps/compact/CompactionDriver.java | 7 +++
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
index 4f61047749..9fee825488 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java
@@ -342,10 +342,9 @@ class CompactionDriver extends ManagerRepo {
}
};
- try (
- var tablets =
ample.readTablets().forTable(tableId).overlapping(startRow, endRow)
- .fetch(PREV_ROW, COMPACTED, SELECTED).checkConsistency().build();
- var tabletsMutator =
ample.conditionallyMutateTablets(resultConsumer)) {
+ try (var tablets =
ample.readTablets().forTable(tableId).overlapping(startRow, endRow)
+ .fetch(PREV_ROW, COMPACTED, SELECTED,
USER_COMPACTION_REQUESTED).checkConsistency()
+ .build(); var tabletsMutator =
ample.conditionallyMutateTablets(resultConsumer)) {
Predicate needsUpdate =
tabletMetadata -> (tabletMetadata.getSelectedFiles() != null
&&
tabletMetadata.getSelectedFiles().getFateId().equals(fateId))
(accumulo) branch elasticity updated: Reduced default compaction configuration down to one group (#4299)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f9d79650fc Reduced default compaction configuration down to one group
(#4299)
f9d79650fc is described below
commit f9d79650fcbc5bf430c573242e43de7e05b2dbe9
Author: Dave Marion
AuthorDate: Tue Feb 27 09:26:09 2024 -0500
Reduced default compaction configuration down to one group (#4299)
Fixes #3546
---
assemble/bin/accumulo-cluster | 6 +--
.../org/apache/accumulo/core/conf/Property.java| 7 ++-
.../spi/compaction/DefaultCompactionPlanner.java | 14 +++---
.../compaction/DefaultCompactionPlannerTest.java | 53 +++---
.../minicluster/MiniAccumuloClusterTest.java | 2 +-
.../server/conf/CheckCompactionConfigTest.java | 38
.../compaction/BadCompactionServiceConfigIT.java | 8 ++--
.../test/compaction/CompactionConfigChangeIT.java | 6 +--
.../test/compaction/CompactionExecutorIT.java | 18
.../CompactionPriorityQueueMetricsIT.java | 5 +-
.../compaction/ExternalCompactionTestUtils.java| 16 +++
.../accumulo/test/functional/CompactionIT.java | 6 +--
.../test/functional/FateConcurrencyIT.java | 5 +-
.../accumulo/test/functional/FateStarvationIT.java | 9 ++--
.../test/functional/IdleProcessMetricsIT.java | 6 +--
.../test/functional/MemoryStarvedMajCIT.java | 11 +++--
16 files changed, 104 insertions(+), 106 deletions(-)
diff --git a/assemble/bin/accumulo-cluster b/assemble/bin/accumulo-cluster
index 69879d2061..5a78c6773c 100755
--- a/assemble/bin/accumulo-cluster
+++ b/assemble/bin/accumulo-cluster
@@ -526,11 +526,7 @@ tserver:
- localhost
compactor:
- accumulo_meta:
-- localhost
- user_small:
-- localhost
- user_large:
+ default:
- localhost
sserver:
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 394b79746e..f48253a812 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -65,7 +65,7 @@ public enum Property {
COMPACTION_SERVICE_ROOT_MAX_OPEN(COMPACTION_SERVICE_PREFIX +
"root.planner.opts.maxOpen", "30",
PropertyType.COUNT, "The maximum number of files a compaction will
open.", "4.0.0"),
COMPACTION_SERVICE_ROOT_GROUPS(COMPACTION_SERVICE_PREFIX +
"root.planner.opts.groups",
- "[{'name':'accumulo_meta'}]".replaceAll("'", "\""), PropertyType.STRING,
+ "[{'group':'default'}]".replaceAll("'", "\""), PropertyType.JSON,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"4.0.0"),
COMPACTION_SERVICE_META_PLANNER(COMPACTION_SERVICE_PREFIX + "meta.planner",
@@ -74,7 +74,7 @@ public enum Property {
COMPACTION_SERVICE_META_MAX_OPEN(COMPACTION_SERVICE_PREFIX +
"meta.planner.opts.maxOpen", "30",
PropertyType.COUNT, "The maximum number of files a compaction will
open.", "4.0.0"),
COMPACTION_SERVICE_META_GROUPS(COMPACTION_SERVICE_PREFIX +
"meta.planner.opts.groups",
- "[{'name':'accumulo_meta'}]".replaceAll("'", "\""), PropertyType.JSON,
+ "[{'group':'default'}]".replaceAll("'", "\""), PropertyType.JSON,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"4.0.0"),
COMPACTION_SERVICE_DEFAULT_PLANNER(COMPACTION_SERVICE_PREFIX +
"default.planner",
@@ -83,8 +83,7 @@ public enum Property {
COMPACTION_SERVICE_DEFAULT_MAX_OPEN(COMPACTION_SERVICE_PREFIX +
"default.planner.opts.maxOpen",
"10", PropertyType.COUNT, "The maximum number of files a compaction will
open.", "4.0.0"),
COMPACTION_SERVICE_DEFAULT_GROUPS(COMPACTION_SERVICE_PREFIX +
"default.planner.opts.groups",
- ("[{'name':'user_small','maxSize':'128M'},
{'name':'user_large'}]").replaceAll("'", "\""),
- PropertyType.STRING,
+ ("[{'group':'default'}]").replaceAll("'", "\""), PropertyType.JSON,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"4.0.0"),
COMPACTION_WARN_TIME(COMPACTION_PREFIX + "warn.time", "10m",
PropertyType.TIMEDURATION,
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/
(accumulo) branch elasticity updated: Added unit test for CompactionJobPriorityQueue (#4296)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 11f8a5095f Added unit test for CompactionJobPriorityQueue (#4296)
11f8a5095f is described below
commit 11f8a5095fc5e96c0ad016f40a98b1ce389382a7
Author: Dave Marion
AuthorDate: Tue Feb 27 09:01:46 2024 -0500
Added unit test for CompactionJobPriorityQueue (#4296)
Modified add method to return into instead of boolean
and added unit test for CompactionJobPriorityQueue
Fixes #3466
---
.../queue/CompactionJobPriorityQueue.java | 39 ++-
.../compaction/queue/CompactionJobQueues.java | 2 +-
.../queue/CompactionJobPriorityQueueTest.java | 322 +
3 files changed, 350 insertions(+), 13 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
index 12d82ff1f9..6c28fe5ecf 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobPriorityQueue.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -47,7 +48,7 @@ import com.google.common.base.Preconditions;
*
*/
public class CompactionJobPriorityQueue {
- // ELASTICITY_TODO unit test this class
+
private final CompactorGroupId groupId;
private class CjpqKey implements Comparable {
@@ -59,7 +60,7 @@ public class CompactionJobPriorityQueue {
CjpqKey(CompactionJob job) {
this.job = job;
- this.seq = nextSeq++;
+ this.seq = nextSeq.incrementAndGet();
}
@Override
@@ -102,9 +103,9 @@ public class CompactionJobPriorityQueue {
// jobs in the queue when new jobs are queued for a tablet.
private final Map> tabletJobs;
- private long nextSeq;
+ private final AtomicLong nextSeq = new AtomicLong(0);
- private boolean closed = false;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
public CompactionJobPriorityQueue(CompactorGroupId groupId, int maxSize) {
this.jobQueue = new TreeMap<>();
@@ -115,21 +116,23 @@ public class CompactionJobPriorityQueue {
this.dequeuedJobs = new AtomicLong(0);
}
- public synchronized boolean add(TabletMetadata tabletMetadata,
Collection jobs) {
-if (closed) {
- return false;
-}
-
+ public synchronized int add(TabletMetadata tabletMetadata,
Collection jobs) {
Preconditions.checkArgument(jobs.stream().allMatch(job ->
job.getGroup().equals(groupId)));
removePreviousSubmissions(tabletMetadata.getExtent());
List newEntries = new ArrayList<>(jobs.size());
+int jobsAdded = 0;
for (CompactionJob job : jobs) {
CjpqKey cjqpKey = addJobToQueue(tabletMetadata, job);
if (cjqpKey != null) {
newEntries.add(cjqpKey);
+jobsAdded++;
+ } else {
+// The priority for this job was lower than all other priorities and
not added
+// In this case we will return true even though a subset of the jobs,
or none,
+// were added
}
}
@@ -137,7 +140,7 @@ public class CompactionJobPriorityQueue {
checkState(tabletJobs.put(tabletMetadata.getExtent(), newEntries) ==
null);
}
-return true;
+return jobsAdded;
}
public long getMaxSize() {
@@ -178,9 +181,19 @@ public class CompactionJobPriorityQueue {
return first == null ? null : first.getValue();
}
+ // exists for tests
+ synchronized CompactionJobQueues.MetaJob peek() {
+var firstEntry = jobQueue.firstEntry();
+return firstEntry == null ? null : firstEntry.getValue();
+ }
+
+ public boolean isClosed() {
+return closed.get();
+ }
+
public synchronized boolean closeIfEmpty() {
if (jobQueue.isEmpty()) {
- closed = true;
+ closed.set(true);
return true;
}
@@ -205,7 +218,9 @@ public class CompactionJobPriorityQueue {
return null;
} else {
// the new job has a higher priority than the lowest job in the queue,
so remove the lowest
-jobQueue.pollLastEntry();
+if (jobQueue.pollLastEntry() != null) {
+ rejectedJobs.getAndIncrement();
+}
}
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java
b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/queue/CompactionJobQueues.java
index 2d6f
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9deb242ff63fd6dd2d0a9e72ac37ebd28cb7e84b Merge: dd7b749102 f2adaebba2 Author: Dave Marion AuthorDate: Thu Feb 22 17:50:39 2024 + Merge branch '2.1'
(accumulo) branch main updated (dd7b749102 -> 9deb242ff6)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from dd7b749102 Merge branch '2.1' add f2adaebba2 Removed extra braces in string added as part of #4289 new 9deb242ff6 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch 2.1 updated: Removed extra braces in string added as part of #4289
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new f2adaebba2 Removed extra braces in string added as part of #4289
f2adaebba2 is described below
commit f2adaebba28b5e47ea03a850254e901a0c7be371
Author: Dave Marion
AuthorDate: Thu Feb 22 17:50:03 2024 +
Removed extra braces in string added as part of #4289
---
core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
b/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
index ba3e7e543f..eba5cc420f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
@@ -57,7 +57,7 @@ public class ConfigCheckUtil {
} else if (prop == null) {
log.warn(PREFIX + "unrecognized property key ({}) for {}", key,
source);
} else if (prop.getType() == PropertyType.PREFIX) {
-fatal(PREFIX + "incomplete property key (" + key + ") for {}" +
source);
+fatal(PREFIX + "incomplete property key (" + key + ") for " + source);
} else if (!prop.getType().isValidFormat(value)) {
fatal(PREFIX + "improperly formatted value for key (" + key + ",
type=" + prop.getType()
+ ") : " + value + " for " + source);
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a4af4f0155dc0d3db17ae3cbb0ea9a3d8159c13d
Merge: d766696152 dd7b749102
Author: Dave Marion
AuthorDate: Thu Feb 22 17:42:09 2024 +
Merge branch 'main' into elasticity
.../manager/upgrade/UpgradeCoordinator.java| 37 +-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
index 8436a18737,34b362fa2e..162196a84b
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
@@@ -34,9 -33,13 +34,13 @@@ import java.util.concurrent.TimeUnit
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
+ import org.apache.accumulo.core.client.AccumuloSecurityException;
+ import org.apache.accumulo.core.client.NamespaceNotFoundException;
+ import org.apache.accumulo.core.client.TableNotFoundException;
+ import org.apache.accumulo.core.conf.ConfigCheckUtil;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.fate.ReadOnlyTStore;
+import org.apache.accumulo.core.fate.ReadOnlyFateStore;
import org.apache.accumulo.core.fate.ZooStore;
+import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.volume.Volume;
import org.apache.accumulo.manager.EventCoordinator;
@@@ -79,10 -82,19 +83,19 @@@ public class UpgradeCoordinator
*/
UPGRADED_ROOT {
@Override
- public boolean isParentLevelUpgraded(KeyExtent extent) {
-return extent.isMeta();
+ public boolean isParentLevelUpgraded(Ample.DataLevel level) {
+return level == Ample.DataLevel.METADATA || level ==
Ample.DataLevel.ROOT;
}
},
+ /**
+ * This signifies that zookeeper and the root and metadata tables have
been upgraded so far.
+ */
+ UPGRADED_METADATA {
+ @Override
- public boolean isParentLevelUpgraded(KeyExtent extent) {
-return extent.isMeta();
++ public boolean isParentLevelUpgraded(Ample.DataLevel level) {
++return level == Ample.DataLevel.METADATA || level ==
Ample.DataLevel.ROOT;
+ }
+ },
/**
* This signifies that everything (zookeeper, root table, metadata table)
is upgraded.
*/
(accumulo) branch elasticity updated (d766696152 -> a4af4f0155)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from d766696152 improves TabletServerGivesUpIT (#4290) add af50af77ce Added validation of configuration during upgrade (#4289) add dd7b749102 Merge branch '2.1' new a4af4f0155 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../manager/upgrade/UpgradeCoordinator.java| 37 +- 1 file changed, 36 insertions(+), 1 deletion(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit dd7b749102ad1718c1c0dc5cc46e0126c1a10023
Merge: e3d6204305 af50af77ce
Author: Dave Marion
AuthorDate: Thu Feb 22 17:28:33 2024 +
Merge branch '2.1'
.../manager/upgrade/UpgradeCoordinator.java| 37 +-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
index f5fd84c0e1,d62e824e05..34b362fa2e
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
@@@ -190,25 -198,19 +203,28 @@@ public class UpgradeCoordinator
.submit(() -> {
try {
for (int v = currentVersion; v < AccumuloDataVersion.get();
v++) {
-log.info("Upgrading Root from data version {}", v);
+log.info("Upgrading Root - current version {} as step towards
target version {}", v,
+AccumuloDataVersion.get());
+var upgrader = upgraders.get(v);
+Objects.requireNonNull(upgrader,
+"upgrade root: failed to find root upgrader for version "
+ currentVersion);
upgraders.get(v).upgradeRoot(context);
}
-
setStatus(UpgradeStatus.UPGRADED_ROOT, eventCoordinator);
for (int v = currentVersion; v < AccumuloDataVersion.get();
v++) {
-log.info("Upgrading Metadata from data version {}", v);
+log.info(
+"Upgrading Metadata - current version {} as step towards
target version {}", v,
+AccumuloDataVersion.get());
+var upgrader = upgraders.get(v);
+Objects.requireNonNull(upgrader,
+"upgrade metadata: failed to find upgrader for version "
+ currentVersion);
upgraders.get(v).upgradeMetadata(context);
}
+ setStatus(UpgradeStatus.UPGRADED_METADATA, eventCoordinator);
+
+ log.info("Validating configuration properties.");
+ validateProperties(context);
log.info("Updating persistent data version.");
updateAccumuloVersion(context.getServerDirs(),
context.getVolumeManager(),
(accumulo) branch main updated (e3d6204305 -> dd7b749102)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from e3d6204305 Merge remote-tracking branch 'upstream/2.1' add af50af77ce Added validation of configuration during upgrade (#4289) new dd7b749102 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../manager/upgrade/UpgradeCoordinator.java| 37 +- 1 file changed, 36 insertions(+), 1 deletion(-)
(accumulo) branch 2.1 updated: Added validation of configuration during upgrade (#4289)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new af50af77ce Added validation of configuration during upgrade (#4289)
af50af77ce is described below
commit af50af77cea24a8c4c8cfe62c09b4238577e1034
Author: Dave Marion
AuthorDate: Thu Feb 22 12:19:36 2024 -0500
Added validation of configuration during upgrade (#4289)
Backported second parameter to CheckConfigUtil.validate and updated
all call locations to supply the second parameter. Added method to
UpgradeCoordinator to call CheckConfigUtil.validate after the
upgradeMetadata method is called on all Upgraders
Fixes #3972
---
core/pom.xml | 4 +++
.../apache/accumulo/core/conf/ConfigCheckUtil.java | 11 ---
.../accumulo/core/conf/SiteConfiguration.java | 2 +-
.../accumulo/core/conf/ConfigCheckUtilTest.java| 16 +-
.../core/conf/DefaultConfigurationTest.java| 2 +-
.../server/conf/ServerConfigurationFactory.java| 4 +--
.../manager/upgrade/UpgradeCoordinator.java| 37 +-
7 files changed, 58 insertions(+), 18 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 43dcd6de30..d8cc050b33 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -135,6 +135,10 @@
org.apache.zookeeper
zookeeper-jute
+
+ org.checkerframework
+ checker-qual
+
org.slf4j
slf4j-api
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
b/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
index b9e7ecfff3..ba3e7e543f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigCheckUtil.java
@@ -23,6 +23,7 @@ import java.util.Map.Entry;
import java.util.Objects;
import org.apache.accumulo.core.spi.crypto.CryptoServiceFactory;
+import org.checkerframework.checker.nullness.qual.NonNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +46,7 @@ public class ConfigCheckUtil {
* @param entries iterable through configuration keys and values
* @throws ConfigCheckException if a fatal configuration error is found
*/
- public static void validate(Iterable> entries) {
+ public static void validate(Iterable> entries, @NonNull
String source) {
String instanceZkTimeoutValue = null;
for (Entry entry : entries) {
String key = entry.getKey();
@@ -54,12 +55,12 @@ public class ConfigCheckUtil {
if (prop == null && Property.isValidPropertyKey(key)) {
continue; // unknown valid property (i.e. has proper prefix)
} else if (prop == null) {
-log.warn(PREFIX + "unrecognized property key (" + key + ")");
+log.warn(PREFIX + "unrecognized property key ({}) for {}", key,
source);
} else if (prop.getType() == PropertyType.PREFIX) {
-fatal(PREFIX + "incomplete property key (" + key + ")");
+fatal(PREFIX + "incomplete property key (" + key + ") for {}" +
source);
} else if (!prop.getType().isValidFormat(value)) {
fatal(PREFIX + "improperly formatted value for key (" + key + ",
type=" + prop.getType()
-+ ") : " + value);
++ ") : " + value + " for " + source);
}
if (key.equals(Property.INSTANCE_ZK_TIMEOUT.getKey())) {
@@ -128,7 +129,7 @@ public class ConfigCheckUtil {
}
/**
- * The exception thrown when {@link ConfigCheckUtil#validate(Iterable)}
fails.
+ * The exception thrown when {@link ConfigCheckUtil#validate(Iterable,
String)} fails.
*/
public static class ConfigCheckException extends RuntimeException {
private static final long serialVersionUID = 1L;
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
index c20aab006b..196486f205 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/SiteConfiguration.java
@@ -212,7 +212,7 @@ public class SiteConfiguration extends
AccumuloConfiguration {
private final Map config;
private SiteConfiguration(Map config) {
-ConfigCheckUtil.validate(config.entrySet());
+ConfigCheckUtil.validate(config.entrySet(), "site config");
this.config = config;
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/conf/ConfigCheckUtilTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/ConfigCheckUtilTest.java
index 258e38075f..e204316c50 100644
--- a/core/src/test/java/org/apache/accumulo/cor
(accumulo) branch elasticity updated: Ensured that TabletMgmtStats are incremented in TabletGroupWatcher (#4272)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 36adc8e67e Ensured that TabletMgmtStats are incremented in
TabletGroupWatcher (#4272)
36adc8e67e is described below
commit 36adc8e67e9468e49d19102044315738659151d2
Author: Dave Marion
AuthorDate: Wed Feb 21 10:09:07 2024 -0500
Ensured that TabletMgmtStats are incremented in TabletGroupWatcher (#4272)
Fixes #4233
---
.../src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 37c12dd96c..9f1deac81e 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -361,7 +361,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
throws BadLocationStateException, TException, DistributedStoreException,
WalMarkerException,
IOException {
-TableMgmtStats tableMgmtStats = new TableMgmtStats();
+final TableMgmtStats tableMgmtStats = new TableMgmtStats();
final boolean shuttingDownAllTabletServers =
tableMgmtParams.getServersToShutdown().equals(currentTServers.keySet());
if (shuttingDownAllTabletServers && !isFullScan) {
@@ -446,6 +446,7 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
state = newState;
}
}
+ tableMgmtStats.counts[state.ordinal()]++;
// This is final because nothing in this method should change the goal.
All computation of the
// goal should be done in TabletGoalState.compute() so that all parts of
the Accumulo code
@@ -619,7 +620,6 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
break;
}
}
-tableMgmtStats.counts[state.ordinal()]++;
}
}
(accumulo) branch elasticity updated: Create accumulo.fate table during upgrade (#4284)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 17780f79d2 Create accumulo.fate table during upgrade (#4284)
17780f79d2 is described below
commit 17780f79d272668d059c354ea381e7249e4b9ab8
Author: Dave Marion
AuthorDate: Wed Feb 21 07:34:25 2024 -0500
Create accumulo.fate table during upgrade (#4284)
Use the same code as ZooKeeperInitializer to create
the new accumulo.fate table
Fixes #4167
---
.../apache/accumulo/manager/upgrade/Upgrader12to13.java | 15 +++
1 file changed, 15 insertions(+)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
index 6d0d51a3d9..179147edc3 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
@@ -30,12 +30,14 @@ import java.util.Map.Entry;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
+import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
@@ -49,6 +51,7 @@ import org.apache.accumulo.core.schema.Section;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.conf.store.TablePropKey;
+import org.apache.accumulo.server.tables.TableManager;
import org.apache.accumulo.server.util.PropUtil;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@@ -74,6 +77,8 @@ public class Upgrader12to13 implements Upgrader {
@Override
public void upgradeRoot(ServerContext context) {
+LOG.info("Creating table {}", AccumuloTable.FATE.tableName());
+createFateTable(context);
LOG.info("Looking for partial splits");
handlePartialSplits(context, AccumuloTable.ROOT.tableName());
LOG.info("setting metadata table hosting availability");
@@ -100,6 +105,16 @@ public class Upgrader12to13 implements Upgrader {
removeCompactColumnsFromTable(context, AccumuloTable.METADATA.tableName());
}
+ private void createFateTable(ServerContext context) {
+try {
+ TableManager.prepareNewTableState(context, AccumuloTable.FATE.tableId(),
+ Namespace.ACCUMULO.id(), AccumuloTable.FATE.tableName(),
TableState.ONLINE,
+ ZooUtil.NodeExistsPolicy.SKIP);
+} catch (KeeperException | InterruptedException e) {
+ throw new IllegalStateException("Error creating table: " +
AccumuloTable.FATE.tableName(), e);
+}
+ }
+
private void removeCompactColumnsFromRootTabletMetadata(ServerContext
context) {
var rootBase = ZooUtil.getRoot(context.getInstanceID()) + ZROOT_TABLET;
(accumulo-access) branch 1.0.0-beta-rc4 deleted (was 1c6051d)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-beta-rc4 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git was 1c6051d [maven-release-plugin] prepare release rel/1.0.0-beta The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(accumulo-access) annotated tag rel/1.0.0-beta updated (1c6051d -> e5d3dd6)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to annotated tag rel/1.0.0-beta in repository https://gitbox.apache.org/repos/asf/accumulo-access.git *** WARNING: tag rel/1.0.0-beta was modified! *** from 1c6051d (commit) to e5d3dd6 (tag) tagging 1c6051d4f450d620e3594659ed8f00c107348003 (commit) by Dave Marion on Sat Feb 17 15:16:07 2024 + - Log - Apache Accumulo Access 1.0.0-beta -BEGIN PGP SIGNATURE- iQIzBAABCAAdFiEEz0pbbhMhhE62WHKDBsycofm/JI4FAmXQzbcACgkQBsycofm/ JI63lw//Vt3Y4I75sPQgxfPeX8F/teyeA0t/W9Ub2jw+UAsyFQGn99rVDwX71CCG lH8OhbUqszURO6RnrhK44VWpteOa8Rr8eXHaVcgAngpDg0DIk2mgpN7pxhYFqWyf Lqyh91ZP8qYU7np4262clgdm8aMzeplm2ksSbMhzyFNKGhF63KulV668sJAuyHv5 6piz218qiaZVfNwHbjk0+EG9TZQe+bUYWMoJUIvHtceKolZFGz3zUn69qYz7ArSf k2m5eHaoYorMPgVScdO8/mXkt5dllOCHkWSgeP1UYj50agZmYf4rCY3+SJyGkREF i1K50/7i5jEWDuxG0Er7RNH79PFMx/J0dhkwsvH0G8LJ7J0hjPyNZJU9wjnV8K4O 8Kgn+yNrKwXE068/6z951oiCqTbetd7A0XMrLYvjm9iSO/dtgGjJPaKTKSKNlJqB m+iZmV/XgjRoQv7anqx6LAbkhGCoH/sGuOXHjYwD81WxBkf7O5iz4/yjbbeD/4/K ZISsBjCuFlH8L7qbUgrwlc8W20k8fsKy9TbVL4XrU7yQlq34waBdM7kZbqYle2wA 4ulSgSMXAHwu+QRB3lQbX58H5tALIxxNdA2u8cMdaJmg/Rcj4mJtVZgz/JORhDdh lps3qTmh63/rPNrEdGfwi2fkvh/OkYXQZPwhFlumsqiq9+vXpL8= =EIhP -END PGP SIGNATURE- --- No new revisions were added by this update. Summary of changes:
(accumulo-access) 01/02: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit afe80ddb43a7891f859257d71da38d499e11da6d Author: Dave Marion AuthorDate: Wed Feb 14 14:21:53 2024 + [maven-release-plugin] prepare for next development iteration --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 4f2e1b4..70a1ca5 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.accumulo accumulo-access - 1.0.0-beta + 1.0.0-SNAPSHOT Apache Accumulo Access Project Apache Accumulo Access is a library that provides the same functionality, semantics, and syntax as the Apache Accumulo ColumnVisibility and VisibilityEvaluator classes. This functionality is provided in a @@ -75,7 +75,7 @@ scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git -rel/1.0.0-beta +HEAD https://gitbox.apache.org/repos/asf?p=accumulo-access.git @@ -100,7 +100,7 @@ .+-SNAPSHOT,(?i).*(alpha|beta)[0-9.-]*,(?i).*[.-](m|rc)[0-9]+ 17 - 2024-02-14T14:21:47Z + 2024-02-14T14:21:53Z UTF-8 true source-release-tar
(accumulo-access) 02/02: Merge branch '1.0.0-beta-rc4-next'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit e0abc5d5a62f7847bf6c93886f6910daad18a0f3 Merge: 202a51c afe80dd Author: Dave Marion AuthorDate: Sat Feb 17 15:18:52 2024 + Merge branch '1.0.0-beta-rc4-next' pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(accumulo-access) branch main updated (202a51c -> e0abc5d)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git from 202a51c Added post-vote-checklist issue templates (#44) add 1c6051d [maven-release-plugin] prepare release rel/1.0.0-beta new afe80dd [maven-release-plugin] prepare for next development iteration new e0abc5d Merge branch '1.0.0-beta-rc4-next' The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(accumulo-access) branch main updated: Added post-vote-checklist issue templates (#44)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git The following commit(s) were added to refs/heads/main by this push: new 202a51c Added post-vote-checklist issue templates (#44) 202a51c is described below commit 202a51cc23fe3157d9e34dc0243634beac94ecb9 Author: Dave Marion AuthorDate: Sat Feb 17 09:24:52 2024 -0500 Added post-vote-checklist issue templates (#44) Co-authored-by: Christopher Tubbs --- .github/ISSUE_TEMPLATE/post_vote_checklist.md | 41 +++ pom.xml | 1 + 2 files changed, 42 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/post_vote_checklist.md b/.github/ISSUE_TEMPLATE/post_vote_checklist.md new file mode 100644 index 000..e984596 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/post_vote_checklist.md @@ -0,0 +1,41 @@ +--- +name: Post Vote Checklist +about: A checklist for tracking post-vote release tasks +title: 'Post-vote release checklist for version [e.g. 1.0.0]' +labels: +assignees: '' + +--- + +- [ ] Label this issue with the GitHub [milestone](https://github.com/apache/accumulo-access/milestones) that corresponds to this release version +- [Git](https://github.com/apache/accumulo-access) tasks + - [ ] Create a signed `rel/` tag (and push it) + - [ ] Merge `-rc-next` branch into a maintenance branch (if maintenance is expected), +and then into the `main` branch (and push them) + - [ ] Remove `*-rc*` branches +- [Nexus](https://repository.apache.org) tasks + - [ ] Release the staging repository corresponding to the successful release candidate (use "release" button) + - [ ] Drop any other staging repositories for Accumulo-Access (do *not* "release" or "promote" them) +- [SVN / dist-release](https://dist.apache.org/repos/dist/release/accumulo-access) tasks + - [ ] Upload the release artifacts (tarballs, signatures, and `.sha512` files) for the mirrors + - [ ] Remove old artifacts from the mirrors (**after** updating the website) +- [Board reporting tool](https://reporter.apache.org/addrelease?accumulo) + - [ ] Add the date of release (the date the release artifacts were uploaded to SVN `UTC+`) +- Verify published artifacts + - [ ] In [Maven Central](https://repo1.maven.org/maven2/org/apache/accumulo/accumulo-access/) + - [ ] In [ASF Downloads](https://downloads.apache.org/accumulo) + - [ ] In [several mirrors or CDN](https://www.apache.org/dyn/closer.lua/accumulo) +- Update the [website](https://accumulo.apache.org/) + - [ ] Release notes + - [ ] Navigation + - [ ] Downloads page + - [ ] If LTM release, update previous LTM release entry on downloads page and release notes with an EOL date 1 year from the current release date + - [ ] DOAP file +- Announcement email + - [ ] Prepare and get review on dev list (see examples [from previous announcement messages](https://lists.apache.org/list.html?annou...@apache.org:gte=1d:accumulo)) + - [ ] Send to annou...@apache.org and u...@accumulo.apache.org (use plain text mode only; html email will be rejected) +- GitHub wrap-up + - [ ] Close this issue + - [ ] Create a new GitHub milestone for the next version (if necessary) and move any open issues not completed in this release to that project + - [ ] Close the milestone that corresponds to this release + diff --git a/pom.xml b/pom.xml index 2e22b9a..f48f702 100644 --- a/pom.xml +++ b/pom.xml @@ -504,6 +504,7 @@ src/it/antlr4-example/src/main/antlr4/Abnf.g4 src/test/resources/testdata.json +.github/**
(accumulo) branch elasticity updated: Fixed RecoveryIT timeout issue (#4271)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new da7b3b005e Fixed RecoveryIT timeout issue (#4271)
da7b3b005e is described below
commit da7b3b005e13276c202b4f337496f9ecf2c039e5
Author: Dave Marion
AuthorDate: Thu Feb 15 16:33:07 2024 -0500
Fixed RecoveryIT timeout issue (#4271)
Test was waiting for 60s to confirm that recovery logs were sorted. However,
the default interval for checking if recovery logs needed to be sorted is
1m.
Increased the timeout in the test so that it would not fail.
---
test/src/main/java/org/apache/accumulo/test/RecoveryIT.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java
b/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java
index c8fb440eea..6da4cd9d77 100644
--- a/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.test;
+import static
org.apache.accumulo.core.util.compaction.ExternalCompactionUtil.getCompactorAddrs;
import static org.apache.accumulo.harness.AccumuloITBase.MINI_CLUSTER_ONLY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
@@ -142,7 +143,10 @@ public class RecoveryIT extends AccumuloClusterHarness {
// Stop any running Compactors and ScanServers
control.stopAllServers(ServerType.COMPACTOR);
+ Wait.waitFor(() ->
getCompactorAddrs(getCluster().getServerContext()).size() == 0, 60_000);
+
control.stopAllServers(ServerType.SCAN_SERVER);
+ Wait.waitFor(() -> ((ClientContext) c).getScanServers().size() == 0,
60_000);
// Kill the TabletServer in resource group that is hosting the table
List procs = control.getTabletServers(RESOURCE_GROUP);
@@ -181,7 +185,7 @@ public class RecoveryIT extends AccumuloClusterHarness {
}
// Confirm sorting completed
- Wait.waitFor(() -> logSortingCompleted(c, tid) == true, 60_000);
+ Wait.waitFor(() -> logSortingCompleted(c, tid) == true, 120_000);
// Start the tablet servers so that the Manager
// can assign the table and so that recovery can be completed.
(accumulo) branch elasticity updated: Fixed MemoryStarvedScanIT timeout (#4270)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5cf9953142 Fixed MemoryStarvedScanIT timeout (#4270)
5cf9953142 is described below
commit 5cf99531425ccde5199bc2de344a5a6cce26baa5
Author: Dave Marion
AuthorDate: Thu Feb 15 16:22:35 2024 -0500
Fixed MemoryStarvedScanIT timeout (#4270)
MemoryStarvedScanIT was timing out because the IT was deleting
tables as part of the test and scans on the new FATE table were
being blocked because of the low memory condition. Adjusted the
check for whether the table is a user table or not to include
all tables in the accumulo namespace.
---
.../src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
index 28ad0e0744..ff67ed0469 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
@@ -42,6 +42,7 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.YieldCallback;
import
org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException;
import org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
@@ -92,7 +93,7 @@ public abstract class TabletBase {
this.context = server.getContext();
this.server = server;
this.extent = extent;
-this.isUserTable = !extent.isMeta();
+this.isUserTable = !AccumuloTable.allTableIds().contains(extent.tableId());
TableConfiguration tblConf =
context.getTableConfiguration(extent.tableId());
if (tblConf == null) {
(accumulo) branch elasticity updated: Resolved timeout issue in MemoryStarvedMajCIT (#4267)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 335592c68d Resolved timeout issue in MemoryStarvedMajCIT (#4267)
335592c68d is described below
commit 335592c68deb4fa59e00decc4d3fd72251f2d963
Author: Dave Marion
AuthorDate: Thu Feb 15 14:33:06 2024 -0500
Resolved timeout issue in MemoryStarvedMajCIT (#4267)
It appears that the test was stopping the Compactors, but not
waiting for their ZooKeeper locks to be removed, before moving
on and checking the number of zookeeper locks. I think what was
happening is that the test code moved forward thinking that the
Compactors were stopped, but the Compactors were actually stopping
after this point and during the next step in the test.
---
.../util/compaction/ExternalCompactionUtil.java | 2 +-
.../miniclusterImpl/MiniAccumuloClusterControl.java | 4
.../test/functional/MemoryStarvedMajCIT.java| 21 ++---
3 files changed, 23 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index 697f079d40..0d42f659c8 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@ -114,7 +114,6 @@ public class ExternalCompactionUtil {
ZooReader zooReader = context.getZooReader();
List groups = zooReader.getChildren(compactorGroupsPath);
for (String group : groups) {
-groupsAndAddresses.putIfAbsent(group, new ArrayList<>());
try {
List compactors = zooReader.getChildren(compactorGroupsPath
+ "/" + group);
for (String compactor : compactors) {
@@ -124,6 +123,7 @@ public class ExternalCompactionUtil {
zooReader.getChildren(compactorGroupsPath + "/" + group + "/"
+ compactor);
if (!children.isEmpty()) {
LOG.trace("Found live compactor {} ", compactor);
+ groupsAndAddresses.putIfAbsent(group, new ArrayList<>());
groupsAndAddresses.get(group).add(HostAndPort.fromString(compactor));
}
}
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index 21e831c3e0..9bf50cff2e 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@ -519,6 +519,10 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
stop(server, hostname);
}
+ public List getCompactors(String resourceGroup) {
+return compactorProcesses.get(resourceGroup);
+ }
+
public List getTabletServers(String resourceGroup) {
return tabletServerProcesses.get(resourceGroup);
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
index 98222a5fa6..19723a5391 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java
@@ -22,6 +22,7 @@ import static org.apache.accumulo.test.util.Wait.waitFor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import java.util.List;
import java.util.Map;
@@ -66,6 +67,7 @@ public class MemoryStarvedMajCIT extends
SharedMiniClusterBase {
cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1);
cfg.getClusterServerConfiguration().setNumDefaultScanServers(0);
cfg.getClusterServerConfiguration().setNumDefaultCompactors(1);
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_INTERVAL, "5s");
cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_THRESHOLD,
Double.toString(MemoryStarvedScanIT.FREE_MEMORY_THRESHOLD));
@@ -132,10 +134,23 @@ public class MemoryStarvedMajCIT extends
SharedMiniClusterBase {
ClientContext ctx = (ClientContext) client;
- // Stop the normal compactors and start the version that will consume
- // and free memory when we need it to
- getCluster().getClusterControl().st
(accumulo-access) branch 1.0.0-beta-rc4 created (now 1c6051d)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-beta-rc4 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git at 1c6051d [maven-release-plugin] prepare release rel/1.0.0-beta This branch includes the following new commits: new 1c6051d [maven-release-plugin] prepare release rel/1.0.0-beta The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(accumulo-access) 01/01: [maven-release-plugin] prepare release rel/1.0.0-beta
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1.0.0-beta-rc4 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit 1c6051d4f450d620e3594659ed8f00c107348003 Author: Dave Marion AuthorDate: Wed Feb 14 14:21:53 2024 + [maven-release-plugin] prepare release rel/1.0.0-beta --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2e22b9a..4f2e1b4 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.accumulo accumulo-access - 1.0.0-beta-SNAPSHOT + 1.0.0-beta Apache Accumulo Access Project Apache Accumulo Access is a library that provides the same functionality, semantics, and syntax as the Apache Accumulo ColumnVisibility and VisibilityEvaluator classes. This functionality is provided in a @@ -75,7 +75,7 @@ scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git -HEAD +rel/1.0.0-beta https://gitbox.apache.org/repos/asf?p=accumulo-access.git @@ -100,7 +100,7 @@ .+-SNAPSHOT,(?i).*(alpha|beta)[0-9.-]*,(?i).*[.-](m|rc)[0-9]+ 17 - 2023-08-14T08:11:10Z + 2024-02-14T14:21:47Z UTF-8 true source-release-tar
(accumulo) branch elasticity updated: Add LogSorter to Compactor and ScanServer (#4239)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5d12597b4d Add LogSorter to Compactor and ScanServer (#4239)
5d12597b4d is described below
commit 5d12597b4dedb60524775d0411405bafc3ee86ce
Author: Dave Marion
AuthorDate: Wed Feb 14 08:06:49 2024 -0500
Add LogSorter to Compactor and ScanServer (#4239)
Adds LogSorter to ScanServer and Compactor such that
all available processes will participate when a failure
occurs that leaves a Tablet with walogs. Modified LogSorter
and DistributedWorkQueue so that the Compactor could call
the LogSorter and have its tasks execute serially in the
Compactor thread.
Fixes #4232
---
.../org/apache/accumulo/core/conf/Property.java| 3 +
.../MiniAccumuloClusterControl.java| 4 +
.../server/zookeeper/DistributedWorkQueue.java | 97 ++---
server/compactor/pom.xml | 4 +
.../org/apache/accumulo/compactor/Compactor.java | 11 +
.../org/apache/accumulo/tserver/ScanServer.java| 17 ++
.../org/apache/accumulo/tserver/TabletServer.java | 20 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 25 ++-
.../java/org/apache/accumulo/test/RecoveryIT.java | 231 +
9 files changed, 371 insertions(+), 41 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 12f69d2ff9..22c2606c85 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -520,6 +520,9 @@ public enum Property {
@Experimental
SSERV_THREADCHECK("sserver.server.threadcheck.time", "1s",
PropertyType.TIMEDURATION,
"The time between adjustments of the thrift server thread pool.",
"2.1.0"),
+ @Experimental
+ SSERV_WAL_SORT_MAX_CONCURRENT("sserver.wal.sort.concurrent.max", "2",
PropertyType.COUNT,
+ "The maximum number of threads to use to sort logs during recovery.",
"4.0.0"),
// properties that are specific to tablet server behavior
TSERV_PREFIX("tserver.", null, PropertyType.PREFIX,
"Properties in this category affect the behavior of the tablet
servers.", "1.3.5"),
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
index b3977dab00..21e831c3e0 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java
@@ -519,4 +519,8 @@ public class MiniAccumuloClusterControl implements
ClusterControl {
stop(server, hostname);
}
+ public List getTabletServers(String resourceGroup) {
+return tabletServerProcesses.get(resourceGroup);
+ }
+
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
index cd667909d0..2263e02af4 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -57,7 +58,6 @@ public class DistributedWorkQueue {
private static final Logger log =
LoggerFactory.getLogger(DistributedWorkQueue.class);
- private ThreadPoolExecutor threadPool;
private ZooReaderWriter zoo;
private String path;
private ServerContext context;
@@ -65,12 +65,20 @@ public class DistributedWorkQueue {
private AtomicInteger numTask = new AtomicInteger(0);
- private void lookForWork(final Processor processor, List children) {
+ /**
+ * Finds a child in {@code children} that is not currently being processed
and adds a Runnable to
+ * the {@code executor} that invokes the {@code processor}. The Runnable
will recursively call
+ * {@code lookForWork} after it invokes the {@code processor} such that it
will continue to look
+ * for children that need work until that condition is exhausted. This
method will return early if
+ * the number of currently running tasks is larger than {@code maxThreads}.
+ */
+ private void lookForWor
(accumulo-access) branch main updated: Removed contrib .gitignore files, use top level .gitignore (#48)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git The following commit(s) were added to refs/heads/main by this push: new 91423c6 Removed contrib .gitignore files, use top level .gitignore (#48) 91423c6 is described below commit 91423c6b07938b5ff0fc73081748001acdc6d8ce Author: Dave Marion AuthorDate: Tue Feb 13 13:28:52 2024 -0500 Removed contrib .gitignore files, use top level .gitignore (#48) --- .gitignore | 28 ++-- contrib/.gitignore | 33 - contrib/getting-started/.gitignore | 1 - 3 files changed, 14 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 55d7f58..7b0fced 100644 --- a/.gitignore +++ b/.gitignore @@ -18,19 +18,19 @@ # # Maven ignores -/target/ +target/ # IDE ignores -/.settings/ -/.project -/.classpath -/.pydevproject -/.idea -/*.iml -/*.ipr -/*.iws -/nbproject/ -/nbactions.xml -/nb-configuration.xml -/.vscode/ -/.factorypath +.settings/ +.project +.classpath +.pydevproject +.idea +*.iml +*.ipr +*.iws +nbproject/ +nbactions.xml +nb-configuration.xml +.vscode/ +.factorypath diff --git a/contrib/.gitignore b/contrib/.gitignore deleted file mode 100644 index 7cd1fdc..000 --- a/contrib/.gitignore +++ /dev/null @@ -1,33 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -/antlr4/target/ - -/antlr4/.settings/ -/antlr4/.project -/antlr4/.classpath -/antlr4/.pydevproject -/antlr4/.idea -/antlr4/*.iml -/antlr4/*.ipr -/antlr4/*.iws -/antlr4/nbproject/ -/antlr4/nbactions.xml -/antlr4/nb-configuration.xml -/antlr4/.vscode/ -/antlr4/.factorypath diff --git a/contrib/getting-started/.gitignore b/contrib/getting-started/.gitignore deleted file mode 100644 index eb5a316..000 --- a/contrib/getting-started/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target
(accumulo-access) branch main updated: Removed non-applicable information from LICENSE file (#47)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git The following commit(s) were added to refs/heads/main by this push: new 936e3cc Removed non-applicable information from LICENSE file (#47) 936e3cc is described below commit 936e3cc2bd3156170432033fe7193a3e16f7b01a Author: Dave Marion AuthorDate: Tue Feb 13 11:02:38 2024 -0500 Removed non-applicable information from LICENSE file (#47) --- LICENSE | 132 1 file changed, 132 deletions(-) diff --git a/LICENSE b/LICENSE index 411ba86..d645695 100644 --- a/LICENSE +++ b/LICENSE @@ -200,135 +200,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -** - -# APACHE ACCUMULO SUBCOMPONENTS - -The Apache Accumulo project contains subcomponents with separate -copyright notices and license terms. Your use of these is subject -to the terms and conditions of the following licenses. - -## Software from the European Commission project OneLab - -Files: -* core/src/main/java/org/apache/accumulo/core/bloomfilter/* - -Copyright (c) 2005, European Commission project OneLab under contract 034819 -(http://www.one-lab.org) - -All rights reserved. -Redistribution and use in source and binary forms, with or -without modification, are permitted provided that the following -conditions are met: - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the distribution. - - Neither the name of the University Catholique de Louvain - UCL - nor the names of its contributors may be used to endorse or - promote products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -## JQuery 3.6.1 (https://jquery.com/) - -Files: -* server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/jquery/* - -jQuery JavaScript Library v3.6.1 -https://jquery.com/ - -Includes Sizzle.js -https://sizzlejs.com/ - -Copyright JS Foundation and other contributors -Released under the MIT license -https://jquery.org/license - -Date: 2022-08-26T17:52Z - -Text of the MIT License: - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -## Flot 4.2.3 (https://github.com/flot/flot) - -Files: -* server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/flot/* - -Copyright (c) 2007-2014 IOLA and Ole Laursen - -Available under the MIT License -(see server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/external/flot/LICENSE.txt) - -
(accumulo-access) branch 1.0.0-rc3 deleted (was 276dfe1)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc3 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git was 276dfe1 [maven-release-plugin] prepare release rel/1.0.0 This change permanently discards the following revisions: discard 276dfe1 [maven-release-plugin] prepare release rel/1.0.0
(accumulo-access) 01/01: [maven-release-plugin] prepare release rel/1.0.0
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1.0.0-rc3 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit 276dfe1bfb3868812da5414d79f4d995bade9a36 Author: Dave Marion AuthorDate: Mon Feb 12 14:45:50 2024 + [maven-release-plugin] prepare release rel/1.0.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5edf059..e071f30 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.accumulo accumulo-access - 1.0.0-SNAPSHOT + 1.0.0 Apache Accumulo Access Project Apache Accumulo Access is a library that provides the same functionality, semantics, and syntax as the Apache Accumulo ColumnVisibility and VisibilityEvaluator classes. This functionality is provided in a @@ -75,7 +75,7 @@ scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git -HEAD +rel/1.0.0 https://gitbox.apache.org/repos/asf?p=accumulo-access.git
(accumulo-access) branch 1.0.0-rc3 created (now 276dfe1)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc3 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git at 276dfe1 [maven-release-plugin] prepare release rel/1.0.0 This branch includes the following new commits: new 276dfe1 [maven-release-plugin] prepare release rel/1.0.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(accumulo-access) branch main updated: Changes to rc script because we don't build a tarball (#41)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-access.git
The following commit(s) were added to refs/heads/main by this push:
new a8bc6a1 Changes to rc script because we don't build a tarball (#41)
a8bc6a1 is described below
commit a8bc6a12ee1bbb93bf927b722a5c5f270a6a5b1b
Author: Dave Marion
AuthorDate: Mon Feb 12 09:44:15 2024 -0500
Changes to rc script because we don't build a tarball (#41)
---
src/build/create-release-candidate.sh | 18 +-
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/build/create-release-candidate.sh
b/src/build/create-release-candidate.sh
index d368630..a467e15 100755
--- a/src/build/create-release-candidate.sh
+++ b/src/build/create-release-candidate.sh
@@ -23,10 +23,10 @@ scriptname=$(basename "$0")
projectroot="$(git rev-parse --show-toplevel)" || exit 1
cd "$projectroot" || exit 1
export tlpName=accumulo
-export projName="${tlpName}-access"
+export projName="${tlpName}-Access"
export projNameLong="Apache ${projName^}"
export
stagingRepoPrefix="https://repository.apache.org/content/repositories/orgapache$tlpName;
-export srcQualifier="src"
+export srcQualifier="source-release"
export
relTestingUrl="https://$tlpName.apache.org/contributor/verifying-release;
export tagPrefix="rel/"
@@ -141,7 +141,7 @@ createEmail() {
local stagingrepo
[[ -n $3 ]] && stagingrepo=$3 || stagingrepo=$(prompter 'staging repository
number from https://repository.apache.org/#stagingRepositories' '[0-9]+')
local srcSha
- [[ -n $4 ]] && srcSha=$4 || srcSha=$(prompter 'SHA512 for source tarball'
'[0-9a-f]{128}')
+ [[ -n $4 ]] && srcSha=$4 || srcSha=$(prompter 'SHA512 for source release zip
file' '[0-9a-f]{128}')
local branch
branch=$ver-rc$rc
@@ -206,7 +206,7 @@ If this vote passes, a gpg-signed tag will be created using:
$(green "$commit")
Staging repo: $(green "$stagingRepoPrefix-$stagingrepo")
-Source (official release artifact): $(green
"$stagingRepoPrefix-$stagingrepo/org/apache/$tlpName/$projName/$ver/$projName-$ver-$srcQualifier.tar.gz")
+Source (official release artifact): $(green
"$stagingRepoPrefix-$stagingrepo/org/apache/$tlpName/$projName/$ver/$projName-$ver-$srcQualifier.zip")
Append ".asc" to download the cryptographic signature for a given artifact.
(You can also append ".sha1" or ".md5" instead in order to verify the checksums
@@ -215,10 +215,10 @@ generated by Maven to verify the integrity of the Nexus
repository staging area.
Signing keys are available at https://www.apache.org/dist/$tlpName/KEYS
(Expected fingerprint: $(green "$SELECTED_FINGERPRINT"))
-In addition to the tarballs and their signatures, the following checksum
+In addition to the zipfiles and their signatures, the following checksum
files will be added to the dist/release SVN area after release:
-$(yellow "$projName-$ver-$srcQualifier.tar.gz.sha512") will contain:
-SHA512 ($(green "$projName-$ver-$srcQualifier.tar.gz")) = $(yellow "$srcSha")
+$(yellow "$projName-$ver-$srcQualifier.zip.sha512") will contain:
+SHA512 ($(green "$projName-$ver-$srcQualifier.zip")) = $(yellow "$srcSha")
Issues and pull requests related to this release can be found at: $(green
"https://github.com/apache/accumulo-access/issues?q=milestone%3A$ver;)
@@ -413,11 +413,11 @@ createReleaseCandidate() {
fi
local numSrc
- numSrc=$(find target/checkout/ -type f -name
"$projName-$ver-source-release.tar.gz" | wc -l)
+ numSrc=$(find target/checkout/ -type f -name
"$projName-$ver-source-release.zip" | wc -l)
shopt -s globstar
local srcSha
srcSha=""
- [[ $numSrc == "1" ]] && srcSha=$(sha512sum
target/checkout/**/"$projName-$ver-source-release.tar.gz" | cut -f1 -d" ")
+ [[ $numSrc == "1" ]] && srcSha=$(sha512sum
target/checkout/**/"$projName-$ver-source-release.zip" | cut -f1 -d" ")
# continue to creating email notification
echo "$(red Running)" "$(yellow "$scriptname" --email "$ver" "$rc")"
(accumulo-access) branch 1.0.0-rc2-next deleted (was 4dc198f)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc2-next in repository https://gitbox.apache.org/repos/asf/accumulo-access.git was 4dc198f [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard 4dc198f [maven-release-plugin] prepare for next development iteration discard 157b185 [maven-release-plugin] prepare release rel/1.0.0
(accumulo-access) branch 1.0.0-rc2 deleted (was 157b185)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc2 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git was 157b185 [maven-release-plugin] prepare release rel/1.0.0 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(accumulo-access) branch 1.0.0-rc2 created (now 157b185)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc2 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git at 157b185 [maven-release-plugin] prepare release rel/1.0.0 No new revisions were added by this update.
(accumulo-access) 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1.0.0-rc2-next in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit 4dc198f4c781b8f412465e1f50de0ac3b2db1fea Author: Dave Marion AuthorDate: Mon Feb 12 14:23:02 2024 + [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e071f30..508813e 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.accumulo accumulo-access - 1.0.0 + 1.0.1-SNAPSHOT Apache Accumulo Access Project Apache Accumulo Access is a library that provides the same functionality, semantics, and syntax as the Apache Accumulo ColumnVisibility and VisibilityEvaluator classes. This functionality is provided in a @@ -75,7 +75,7 @@ scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git -rel/1.0.0 +HEAD https://gitbox.apache.org/repos/asf?p=accumulo-access.git
(accumulo-access) branch 1.0.0-rc2-next created (now 4dc198f)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc2-next in repository https://gitbox.apache.org/repos/asf/accumulo-access.git at 4dc198f [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new 4dc198f [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(accumulo-access) branch main updated: Changes based on testing release candidate creation (#40)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-access.git
The following commit(s) were added to refs/heads/main by this push:
new ec2f0ef Changes based on testing release candidate creation (#40)
ec2f0ef is described below
commit ec2f0efade3b084a04190449617d48ead8a2da15
Author: Dave Marion
AuthorDate: Mon Feb 12 09:20:30 2024 -0500
Changes based on testing release candidate creation (#40)
---
pom.xml | 2 +-
src/build/create-release-candidate.sh | 8 ++--
2 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/pom.xml b/pom.xml
index 0836585..5edf059 100644
--- a/pom.xml
+++ b/pom.xml
@@ -440,7 +440,7 @@
clean deploy
clean package
rel/@{project.version}
- apache-release,accumulo-release
+ apache-release
false
false
true
diff --git a/src/build/create-release-candidate.sh
b/src/build/create-release-candidate.sh
index 6fb94ad..d368630 100755
--- a/src/build/create-release-candidate.sh
+++ b/src/build/create-release-candidate.sh
@@ -142,8 +142,6 @@ createEmail() {
[[ -n $3 ]] && stagingrepo=$3 || stagingrepo=$(prompter 'staging repository
number from https://repository.apache.org/#stagingRepositories' '[0-9]+')
local srcSha
[[ -n $4 ]] && srcSha=$4 || srcSha=$(prompter 'SHA512 for source tarball'
'[0-9a-f]{128}')
- local binSha
- [[ -n $5 ]] && binSha=$5 || binSha=$(prompter 'SHA512 for binary tarball'
'[0-9a-f]{128}')
local branch
branch=$ver-rc$rc
@@ -222,9 +220,7 @@ files will be added to the dist/release SVN area after
release:
$(yellow "$projName-$ver-$srcQualifier.tar.gz.sha512") will contain:
SHA512 ($(green "$projName-$ver-$srcQualifier.tar.gz")) = $(yellow "$srcSha")
-Release notes (in progress) can be found at: $(green
"https://$tlpName.apache.org/release/$projName-$ver;)
-
-Release testing instructions: $relTestingUrl
+Issues and pull requests related to this release can be found at: $(green
"https://github.com/apache/accumulo-access/issues?q=milestone%3A$ver;)
Please vote one of:
[ ] +1 - I have verified and accept...
@@ -425,7 +421,7 @@ createReleaseCandidate() {
# continue to creating email notification
echo "$(red Running)" "$(yellow "$scriptname" --email "$ver" "$rc")"
- createEmail "$ver" "$rc" "" "$srcSha" "$binSha"
+ createEmail "$ver" "$rc" "" "$srcSha"
}
SELECTED_FINGERPRINT=""
(accumulo-access) branch 1.0.0-rc1-next deleted (was 4b89b6c)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc1-next in repository https://gitbox.apache.org/repos/asf/accumulo-access.git was 4b89b6c [maven-release-plugin] prepare for next development iteration This change permanently discards the following revisions: discard 4b89b6c [maven-release-plugin] prepare for next development iteration discard 88702a4 [maven-release-plugin] prepare release rel/1.0.0
(accumulo-access) branch 1.0.0-rc1 deleted (was 88702a4)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc1 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git was 88702a4 [maven-release-plugin] prepare release rel/1.0.0 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(accumulo-access) 01/01: [maven-release-plugin] prepare for next development iteration
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1.0.0-rc1-next in repository https://gitbox.apache.org/repos/asf/accumulo-access.git commit 4b89b6c6a96b29a3187fbfc5c8b51ef794f41de2 Author: Dave Marion AuthorDate: Mon Feb 12 13:54:08 2024 + [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index da15e4c..1763b62 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.accumulo accumulo-access - 1.0.0 + 1.0.1-SNAPSHOT Apache Accumulo Access Project Apache Accumulo Access is a library that provides the same functionality, semantics, and syntax as the Apache Accumulo ColumnVisibility and VisibilityEvaluator classes. This functionality is provided in a @@ -75,7 +75,7 @@ scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git scm:git:https://gitbox.apache.org/repos/asf/accumulo-access.git -rel/1.0.0 +HEAD https://gitbox.apache.org/repos/asf?p=accumulo-access.git
(accumulo-access) branch 1.0.0-rc1-next created (now 4b89b6c)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc1-next in repository https://gitbox.apache.org/repos/asf/accumulo-access.git at 4b89b6c [maven-release-plugin] prepare for next development iteration This branch includes the following new commits: new 4b89b6c [maven-release-plugin] prepare for next development iteration The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(accumulo-access) branch 1.0.0-rc1 created (now 88702a4)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch 1.0.0-rc1 in repository https://gitbox.apache.org/repos/asf/accumulo-access.git at 88702a4 [maven-release-plugin] prepare release rel/1.0.0 No new revisions were added by this update.
(accumulo-access) branch main updated: Added description to pom, added initial release candidate script (#39)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-access.git
The following commit(s) were added to refs/heads/main by this push:
new f315807 Added description to pom, added initial release candidate
script (#39)
f315807 is described below
commit f315807509b26a1e1f46cb428d56d285853f805a
Author: Dave Marion
AuthorDate: Mon Feb 12 08:49:49 2024 -0500
Added description to pom, added initial release candidate script (#39)
---
pom.xml | 3 +
src/build/create-release-candidate.sh | 440 ++
2 files changed, 443 insertions(+)
diff --git a/pom.xml b/pom.xml
index d455c3e..0836585 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,6 +30,9 @@
accumulo-access
1.0.0-SNAPSHOT
Apache Accumulo Access Project
+ Apache Accumulo Access is a library that provides the same
functionality, semantics, and syntax
+ as the Apache Accumulo ColumnVisibility and VisibilityEvaluator classes.
This functionality is provided in a
+ standalong Java library that has no runtime dependencies.
https://accumulo.apache.org
The Apache Software Foundation
diff --git a/src/build/create-release-candidate.sh
b/src/build/create-release-candidate.sh
new file mode 100755
index 000..6fb94ad
--- /dev/null
+++ b/src/build/create-release-candidate.sh
@@ -0,0 +1,440 @@
+#! /usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cd "$(dirname "$0")/.." || exit 1
+scriptname=$(basename "$0")
+projectroot="$(git rev-parse --show-toplevel)" || exit 1
+cd "$projectroot" || exit 1
+export tlpName=accumulo
+export projName="${tlpName}-access"
+export projNameLong="Apache ${projName^}"
+export
stagingRepoPrefix="https://repository.apache.org/content/repositories/orgapache$tlpName;
+export srcQualifier="src"
+export
relTestingUrl="https://$tlpName.apache.org/contributor/verifying-release;
+export tagPrefix="rel/"
+
+# check if running in a color terminal
+terminalSupportsColor() {
+ local c
+ c=$(tput colors 2>/dev/null) || c=-1
+ [[ -t 1 ]] && [[ $c -ge 8 ]]
+}
+terminalSupportsColor && doColor=1 || doColor=0
+
+color() {
+ local c
+ c=$1
+ shift
+ [[ $doColor -eq 1 ]] && echo -e "\\e[0;${c}m${*}\\e[0m" || echo "$@"
+}
+red() { color 31 "$@"; }
+green() { color 32 "$@"; }
+yellow() { color 33 "$@"; }
+
+fail() {
+ echo -e ' ' "$@"
+ exit 1
+}
+runLog() {
+ local o
+ o=$1 && shift && echo "$(green Running) $(yellow "$@" '>>' "$o")" && echo
Running "$*" >>"$o" && "$@" >>"$o"
+}
+run() { echo "$(green Running) $(yellow "$@")" && "$@"; }
+
+currentBranch() {
+ local b
+ b=$(git symbolic-ref -q HEAD) && echo "${b##refs/heads/}"
+}
+
+cacheGPG() {
+ # make sure gpg agent has key cached
+ # first clear cache, to reset timeouts (best attempt)
+ { hash gpg-connect-agent && gpg-connect-agent reloadagent /bye; } &>/dev/null
+ # determine fingerprint to use
+ until selectFingerprint; do
+red 'ERROR - Invalid selection'
+ done
+ local TESTFILE
+ TESTFILE=$(mktemp --tmpdir "${USER}-gpgTestFile-.txt")
+ [[ -r $TESTFILE ]] && gpg --local-user "$SELECTED_FINGERPRINT" --sign
"$TESTFILE" && rm -f "$TESTFILE" "$TESTFILE.gpg"
+}
+
+prompter() {
+ # $1 description; $2 pattern to validate against
+ local x
+ read -r -p "Enter the $1: " x
+ until eval "[[ \$x =~ ^$2\$ ]]"; do
+echo " $(red "$x") is not a proper $1" 1>&2
+read -r -p "Enter the $1: " x
+ done
+ echo "$x"
+}
+
+pretty() {
+ local f
+ f=$1
+ shift
+ git log "--pretty=tformat:$f" "$@"
+}
+gitCommits() { pretty %H "$@"; }
+gitCommit() { g
