[accumulo] branch main updated: Validate table/namespace names in public API (#1971)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 58a559f Validate table/namespace names in public API (#1971)
58a559f is described below
commit 58a559fa6ac7c5137b5cd1418b0de457f7cad2a9
Author: slackwinner <50567198+slackwin...@users.noreply.github.com>
AuthorDate: Mon Jun 7 17:02:05 2021 -0400
Validate table/namespace names in public API (#1971)
Throw IllegalArgumentException in Public API consistently
after validating table and namespace names
In response to issue apache/accumulo#1953 this commit
contains changes to allow fast failure and achieve
consistent behavior when a table or namespace name
is invalid.
---
.../java/org/apache/accumulo/core/Constants.java | 2 -
.../accumulo/core/clientImpl/ClientContext.java| 33
.../accumulo/core/clientImpl/ConnectorImpl.java| 29 +++
.../core/clientImpl/NamespaceOperationsImpl.java | 56 +-
.../accumulo/core/clientImpl/Namespaces.java | 5 +-
.../core/clientImpl/SecurityOperationsImpl.java| 26 +++
.../core/clientImpl/TableOperationsHelper.java | 38
.../core/clientImpl/TableOperationsImpl.java | 216 -
.../apache/accumulo/core/clientImpl/Tables.java| 24 ++-
.../accumulo/manager/FateServiceHandler.java | 4 +-
.../accumulo/manager/util/TableValidators.java | 4 +-
.../shell/commands/CreateTableCommand.java | 2 +-
.../org/apache/accumulo/test/NamespacesIT.java | 2 +-
.../apache/accumulo/test/TableOperationsIT.java| 2 +-
14 files changed, 367 insertions(+), 76 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java
b/core/src/main/java/org/apache/accumulo/core/Constants.java
index e2fac29..8559656 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -117,6 +117,4 @@ public class Constants {
public static final int DEFAULT_VISIBILITY_CACHE_SIZE = 1000;
- public static final int MAX_TABLE_NAME_LEN = 1024;
- public static final int MAX_NAMESPACE_LEN = 1024;
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
index 6219188..60c0c5b 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java
@@ -98,6 +98,7 @@ public class ClientContext implements AccumuloClient {
private final ClientInfo info;
private String instanceId;
private final ZooCache zooCache;
+ private static final String VALID_TABLENAME_REGEX =
"^(\\w{1,1024}[.])?(\\w{1,1024})$";
private Credentials creds;
private BatchWriterConfig batchWriterConfig;
@@ -509,6 +510,10 @@ public class ClientContext implements AccumuloClient {
}
TableId getTableId(String tableName) throws TableNotFoundException {
+checkArgument(tableName.matches(VALID_TABLENAME_REGEX),
+"tableName must only contain word characters (letters, digits, and
underscores)"
++ " and cannot exceed 1024 characters");
+
TableId tableId = Tables.getTableId(this, tableName);
if (Tables.getTableState(this, tableId) == TableState.OFFLINE)
throw new TableOfflineException(Tables.getTableOfflineMsg(this,
tableId));
@@ -520,6 +525,10 @@ public class ClientContext implements AccumuloClient {
int numQueryThreads) throws TableNotFoundException {
checkArgument(tableName != null, "tableName is null");
checkArgument(authorizations != null, "authorizations is null");
+checkArgument(tableName.matches(VALID_TABLENAME_REGEX),
+"tableName must only contain word characters (letters, digits, and
underscores)"
++ " and cannot exceed 1024 characters");
+
ensureOpen();
return new TabletServerBatchReader(this, getTableId(tableName),
authorizations,
numQueryThreads);
@@ -547,6 +556,10 @@ public class ClientContext implements AccumuloClient {
int numQueryThreads, BatchWriterConfig config) throws
TableNotFoundException {
checkArgument(tableName != null, "tableName is null");
checkArgument(authorizations != null, "authorizations is null");
+checkArgument(tableName.matches(VALID_TABLENAME_REGEX),
+"tableName must only contain word characters (letters, digits, and
underscores)"
++ " and cannot exceed 1024 characters");
+
ensureOpen();
return new TabletServerBatchDeleter(this, getTableId(tableName),
authorizations,
numQueryThreads, config.merge(getBatchWriterConfig()));
@@ -563,6 +576,10 @@ public class ClientContext implements AccumuloClient {
public BatchWriter createBatchWriter(String
[accumulo-website] branch asf-site updated (21fadb0 -> df2e3a1)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from 21fadb0 Automatic Site Publish by Buildbot add 2daf54d Automatic Site Publish by Buildbot add df2e3a1 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 ++-- output/people/index.html | 10 ++ 2 files changed, 12 insertions(+), 2 deletions(-)
[accumulo] branch main updated: Remove unused code residual from #2122
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3fd4874 Remove unused code residual from #2122
3fd4874 is described below
commit 3fd48747970710a119d870b76baadd8d9d69e181
Author: Christopher Tubbs
AuthorDate: Mon Jun 7 14:46:31 2021 -0400
Remove unused code residual from #2122
Remove two private methods to convert an iterable into a map that are no
longer necessary and an unused import.
---
.../accumulo/core/clientImpl/NamespaceOperationsImpl.java | 1 -
.../main/java/org/apache/accumulo/test/TableOperationsIT.java | 8
.../java/org/apache/accumulo/test/functional/DurabilityIT.java | 10 --
3 files changed, 19 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
index b812e3a..678cee0 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/NamespaceOperationsImpl.java
@@ -30,7 +30,6 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
diff --git a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
index be25e69..3182e35 100644
--- a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java
@@ -259,14 +259,6 @@ public class TableOperationsIT extends
AccumuloClusterHarness {
expectedSplits.isEmpty());
}
- private Map propsToMap(Iterable>
props) {
-Map map = new HashMap<>();
-for (Map.Entry prop : props) {
- map.put(prop.getKey(), prop.getValue());
-}
-return map;
- }
-
/** Compare only the row, column family and column qualifier. */
static class KeyRowColFColQComparator implements Comparator {
@Override
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
index c139954..afe5288 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@ -21,9 +21,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.util.HashMap;
import java.util.Map;
-import java.util.Map.Entry;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
@@ -152,14 +150,6 @@ public class DurabilityIT extends ConfigurableMacBase {
}
}
- private static Map map(Iterable>
entries) {
-Map result = new HashMap<>();
-for (Entry entry : entries) {
- result.put(entry.getKey(), entry.getValue());
-}
-return result;
- }
-
@Test
public void testMetaDurability() throws Exception {
try (AccumuloClient c =
Accumulo.newClient().from(getClientProperties()).build()) {
Buildbot success in on jekyll_websites
The Buildbot has detected a passing build on builder jekyll_websites while building accumulo. Full details are available at: https://ci2.apache.org/#builders/7/builds/501 Buildbot URL: https://ci2.apache.org/ Worker for this Build: bb_slave10_ubuntu Build Reason: Triggered jekyll auto-build via .asf.yaml by ctubbsii Blamelist: asfinfra, commits@accumulo.apache.org Build succeeded! Sincerely, -The Buildbot
[accumulo-website] branch asf-staging updated: Automatic Site Publish by Buildbot
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-staging
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/asf-staging by this push:
new df2e3a1 Automatic Site Publish by Buildbot
df2e3a1 is described below
commit df2e3a11bed0c10a2104ffea233d136dcbac0a00
Author: buildbot
AuthorDate: Mon Jun 7 18:18:22 2021 +
Automatic Site Publish by Buildbot
---
output/feed.xml | 4 ++--
output/people/index.html | 5 +
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/output/feed.xml b/output/feed.xml
index fa77fb6..da32720 100644
--- a/output/feed.xml
+++ b/output/feed.xml
@@ -6,8 +6,8 @@
https://accumulo.apache.org/
https://accumulo.apache.org/feed.xml; rel="self"
type="application/rss+xml"/>
-Thu, 20 May 2021 14:21:16 +
-Thu, 20 May 2021 14:21:16 +
+Mon, 07 Jun 2021 18:18:16 +
+Mon, 07 Jun 2021 18:18:16 +
Jekyll v4.2.0
diff --git a/output/people/index.html b/output/people/index.html
index a21f96d..a00d4c8 100644
--- a/output/people/index.html
+++ b/output/people/index.html
@@ -735,6 +735,11 @@ $(function() {
https://www.timeanddate.com/time/zones/et;>ET
+ Luke Foster
+ https://www.asrc.com;>Arctic Slope Regional Corp.
+ https://www.timeanddate.com/time/zones/et;>ET
+
+
Mandar Inamdar
https://www.microsoft.com;>Microsoft
https://www.timeanddate.com/time/zones/pt;>PT
[accumulo-website] branch main updated: Add Luke Foster to people.md (#284)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new 0900900 Add Luke Foster to people.md (#284) 0900900 is described below commit 0900900d69556105e472697abbb477de78347a9e Author: Luke Foster <84727868+foste...@users.noreply.github.com> AuthorDate: Mon Jun 7 14:17:56 2021 -0400 Add Luke Foster to people.md (#284) Co-authored-by: Luke Foster --- pages/people.md | 1 + 1 file changed, 1 insertion(+) diff --git a/pages/people.md b/pages/people.md index 80e142c..26b82f5 100644 --- a/pages/people.md +++ b/pages/people.md @@ -132,6 +132,7 @@ GitHub also has a [contributor list][github-contributors] based on commits. | Laura Peaslee | [Objective Solutions, Inc.][OBJECTIVE] | [ET][ET] | | Laura Schanno | [Arctic Slope Regional Corp.][ASRC] | [ET][ET] | | Luke Brassard | [sqrrl][SQRRL] | [ET][ET] | +| Luke Foster | [Arctic Slope Regional Corp.][ASRC] | [ET][ET] | | Mandar Inamdar | [Microsoft][MICROSOFT] | [PT][PT] | | Mario Pastorelli| [Teralytics AG][TERALYTICS] | [CEST][CEST] | | Markus Cozowicz | [Microsoft][MICROSOFT] | [CET][CEST] |
[accumulo] branch main updated (5d3344d -> 4ea9bf1)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 5d3344d Remove redundant information in tablets external compaction entry (#2136) add 4ea9bf1 Return table/namespace config as map in API (#2122) No new revisions were added by this update. Summary of changes: .../core/client/admin/NamespaceOperations.java | 27 +-- .../core/client/admin/NewTableConfiguration.java | 2 +- .../core/client/admin/TableOperations.java | 25 -- .../core/clientImpl/InstanceOperationsImpl.java| 2 +- .../core/clientImpl/NamespaceOperationsHelper.java | 5 +-- .../core/clientImpl/NamespaceOperationsImpl.java | 8 ++--- .../accumulo/core/clientImpl/OfflineIterator.java | 2 +- .../core/clientImpl/TableOperationsHelper.java | 9 ++--- .../core/clientImpl/TableOperationsImpl.java | 7 ++-- .../accumulo/core/util/LocalityGroupUtil.java | 2 +- .../java/org/apache/accumulo/core/util/Merge.java | 2 +- .../core/clientImpl/TableOperationsHelperTest.java | 12 +++ .../org/apache/accumulo/server/util/Admin.java | 12 +++ .../accumulo/server/util/ReplicationTableUtil.java | 6 ++-- .../server/util/ReplicationTableUtilTest.java | 2 +- .../rest/replication/ReplicationResource.java | 15 .../main/java/org/apache/accumulo/shell/Shell.java | 10 +++--- .../accumulo/shell/commands/ConfigCommand.java | 21 +--- .../shell/commands/CreateNamespaceCommand.java | 7 ++-- .../shell/commands/CreateTableCommand.java | 6 ++-- .../commands/ShellPluginConfigurationCommand.java | 23 ++--- .../org/apache/accumulo/test/NamespacesIT.java | 10 +++--- .../accumulo/test/NewTableConfigurationIT.java | 9 ++--- .../org/apache/accumulo/test/ShellServerIT.java| 17 + .../apache/accumulo/test/TableOperationsIT.java| 4 +-- .../accumulo/test/functional/CloneTestIT.java | 5 +-- .../accumulo/test/functional/DurabilityIT.java | 4 +-- .../accumulo/test/functional/PermissionsIT.java| 40 +- .../test/replication/StatusCombinerMacIT.java | 8 ++--- 29 files changed, 148 insertions(+), 154 deletions(-)
[accumulo] branch main updated: Remove redundant information in tablets external compaction entry (#2136)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 5d3344d Remove redundant information in tablets external compaction
entry (#2136)
5d3344d is described below
commit 5d3344dbb488d602025ce8f2acb5f807969f45ec
Author: Dave Marion
AuthorDate: Mon Jun 7 08:26:34 2021 -0400
Remove redundant information in tablets external compaction entry (#2136)
Remove redundant information in tablets external compaction entry
Closes #2110
---
.../schema/ExternalCompactionMetadata.java | 21
.../accumulo/tserver/tablet/CompactableImpl.java | 18 +-
.../accumulo/tserver/tablet/CompactableUtils.java | 18 --
.../accumulo/tserver/tablet/DatafileManager.java | 7 ++--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 5 +++
.../tserver/compaction/CompactableUtilsTest.java | 39 ++
6 files changed, 77 insertions(+), 31 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index e2efc63..eada2c7 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -42,7 +42,6 @@ public class ExternalCompactionMetadata {
private final Set jobFiles;
private final Set nextFiles;
private final TabletFile compactTmpName;
- private final TabletFile newFile;
private final String compactorId;
private final CompactionKind kind;
private final short priority;
@@ -52,13 +51,12 @@ public class ExternalCompactionMetadata {
private final Long compactionId;
public ExternalCompactionMetadata(Set jobFiles,
Set nextFiles,
- TabletFile compactTmpName, TabletFile newFile, String compactorId,
CompactionKind kind,
- short priority, CompactionExecutorId ceid, boolean propagateDeletes,
- boolean initiallySelectedAll, Long compactionId) {
+ TabletFile compactTmpName, String compactorId, CompactionKind kind,
short priority,
+ CompactionExecutorId ceid, boolean propagateDeletes, boolean
initiallySelectedAll,
+ Long compactionId) {
this.jobFiles = Objects.requireNonNull(jobFiles);
this.nextFiles = Objects.requireNonNull(nextFiles);
this.compactTmpName = Objects.requireNonNull(compactTmpName);
-this.newFile = Objects.requireNonNull(newFile);
this.compactorId = Objects.requireNonNull(compactorId);
this.kind = Objects.requireNonNull(kind);
this.priority = priority;
@@ -80,10 +78,6 @@ public class ExternalCompactionMetadata {
return compactTmpName;
}
- public TabletFile getNewFile() {
-return newFile;
- }
-
public String getCompactorId() {
return compactorId;
}
@@ -118,7 +112,6 @@ public class ExternalCompactionMetadata {
List inputs;
List nextFiles;
String tmp;
-String dest;
String compactor;
String kind;
String executorId;
@@ -135,7 +128,6 @@ public class ExternalCompactionMetadata {
jData.nextFiles =
nextFiles.stream().map(StoredTabletFile::getMetaUpdateDelete).collect(toList());
jData.tmp = compactTmpName.getMetaInsert();
-jData.dest = newFile.getMetaInsert();
jData.compactor = compactorId;
jData.kind = kind.name();
jData.executorId = ((CompactionExecutorIdImpl) ceid).getExternalName();
@@ -152,10 +144,9 @@ public class ExternalCompactionMetadata {
return new ExternalCompactionMetadata(
jData.inputs.stream().map(StoredTabletFile::new).collect(toSet()),
jData.nextFiles.stream().map(StoredTabletFile::new).collect(toSet()),
-new TabletFile(new Path(jData.tmp)), new TabletFile(new
Path(jData.dest)), jData.compactor,
-CompactionKind.valueOf(jData.kind), jData.priority,
-CompactionExecutorIdImpl.externalId(jData.executorId), jData.propDels,
jData.selectedAll,
-jData.compactionId);
+new TabletFile(new Path(jData.tmp)), jData.compactor,
CompactionKind.valueOf(jData.kind),
+jData.priority, CompactionExecutorIdImpl.externalId(jData.executorId),
jData.propDels,
+jData.selectedAll, jData.compactionId);
}
@Override
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index a7b3ded..ece9f44 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -74,7 +74,6 @@ import org.apache.accumulo.tserver.compactions.Compactable;
import
[accumulo] branch main updated: Modify default compaction planner opts to include internal type (#2140)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 7cc8607 Modify default compaction planner opts to include internal
type (#2140)
7cc8607 is described below
commit 7cc8607a7ba36d8637034fbaa45a340d6f42ba1f
Author: Dave Marion
AuthorDate: Mon Jun 7 08:22:28 2021 -0400
Modify default compaction planner opts to include internal type (#2140)
Modify default compaction planner opts to include internal type. This was
causing CompactionIT to fail as seen in #2137
Co-authored-by: Christopher Tubbs
---
.../accumulo/core/spi/compaction/DefaultCompactionPlanner.java | 7 +--
.../org/apache/accumulo/tserver/compactions/CompactionManager.java | 2 +-
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index 2e1ae90..2cd1b96 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -161,8 +161,11 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
CompactionExecutorId ceid;
- Objects.requireNonNull(executorConfig.type,
- "'type' is a required and must be 'internal' or 'external'");
+ // If not supplied, GSON will leave type null. Default to internal
+ if (executorConfig.type == null) {
+executorConfig.type = "internal";
+ }
+
switch (executorConfig.type) {
case "internal":
Preconditions.checkArgument(null == executorConfig.queue,
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
index a15684e..ae567b8 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
@@ -148,7 +148,7 @@ public class CompactionManager {
Map defaultServiceConfigs =
Map.of(defaultServicePrefix + "planner",
DefaultCompactionPlanner.class.getName(),
defaultServicePrefix + "planner.opts.executors",
- "[{'name':'deprecated','numThreads':" + numThreads + "}]");
+ "[{'name':'deprecated', 'numThreads':" + numThreads + "}]");
configsCopy.putAll(defaultServiceConfigs);
[accumulo] branch main updated: Make DeadCompactionDetector handle network hiccups (#2132)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3dc2bb7 Make DeadCompactionDetector handle network hiccups (#2132)
3dc2bb7 is described below
commit 3dc2bb736995ef9a4bd78eedaeda2e2c59585338
Author: Dave Marion
AuthorDate: Mon Jun 7 08:18:24 2021 -0400
Make DeadCompactionDetector handle network hiccups (#2132)
Modified the DeadCompactionDetector to fail compactions if
they are dead for more than two cycles. This should handle
the case where there is a transient network issue talking
to another component.
Closes #2125
---
.../coordinator/DeadCompactionDetector.java| 43 +++---
1 file changed, 38 insertions(+), 5 deletions(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
index 1b5bd28..3b6a3b4 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
@@ -21,8 +21,12 @@ package org.apache.accumulo.coordinator;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -40,13 +44,15 @@ public class DeadCompactionDetector {
private final ServerContext context;
private final CompactionCoordinator coordinator;
- private ScheduledThreadPoolExecutor schedExecutor;
+ private final ScheduledThreadPoolExecutor schedExecutor;
+ private final ConcurrentHashMap deadCompactions;
public DeadCompactionDetector(ServerContext context, CompactionCoordinator
coordinator,
ScheduledThreadPoolExecutor stpe) {
this.context = context;
this.coordinator = coordinator;
this.schedExecutor = stpe;
+this.deadCompactions = new ConcurrentHashMap<>();
}
private void detectDeadCompactions() {
@@ -66,6 +72,9 @@ public class DeadCompactionDetector {
});
if (tabletCompactions.isEmpty()) {
+ // Clear out dead compactions, tservers don't think anything is running
+ log.trace("Clearing the dead compaction map, no tablets have compactions
running");
+ this.deadCompactions.clear();
// no need to look for dead compactions when tablets don't have anything
recorded as running
return;
}
@@ -74,6 +83,10 @@ public class DeadCompactionDetector {
tabletCompactions.forEach((ecid, extent) -> log.trace("Saw {} for {}",
ecid, extent));
}
+// Remove from the dead map any compactions that the Tablet's
+// do not think are running any more.
+this.deadCompactions.keySet().retainAll(tabletCompactions.keySet());
+
// Determine what compactions are currently running and remove those.
//
// In order for this overall algorithm to be correct and avoid race
conditions, the compactor
@@ -84,21 +97,41 @@ public class DeadCompactionDetector {
running.forEach((ecid) -> {
if (tabletCompactions.remove(ecid) != null) {
-log.trace("Removed {} running on a compactor", ecid);
+log.trace("Removed compaction {} running on a compactor", ecid);
+ }
+ if (this.deadCompactions.remove(ecid) != null) {
+log.trace("Removed {} from the dead compaction map, it's running on a
compactor", ecid);
}
});
// Determine which compactions are currently committing and remove those
context.getAmple().getExternalCompactionFinalStates()
-.map(ecfs ->
ecfs.getExternalCompactionId()).forEach(tabletCompactions::remove);
+.map(ecfs -> ecfs.getExternalCompactionId()).forEach(ecid -> {
+ if (tabletCompactions.remove(ecid) != null) {
+log.trace("Removed compaction {} that is committing", ecid);
+ }
+ if (this.deadCompactions.remove(ecid) != null) {
+log.trace("Removed {} from the dead compaction map, it's
committing", ecid);
+ }
+});
-tabletCompactions
-.forEach((ecid, extent) -> log.debug("Detected dead compaction {} {}",
ecid, extent));
+tabletCompactions.forEach((ecid, extent) -> {
+ log.debug("Possible dead compaction detected {} {}", ecid, extent);
+ this.deadCompactions.merge(ecid, 1L, Long::sum);
+});
// Everything left in tabletCompactions is no longer running anywhere and
should be failed.
// Its
