[accumulo] branch 1451-external-compactions-feature updated: Make compactor sleep time a function of # of compactors
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new 022f4b9 Make compactor sleep time a function of # of compactors
022f4b9 is described below
commit 022f4b9e25c34cabeb2562b678cc9f065b4d12ed
Author: Keith Turner
AuthorDate: Mon May 3 17:09:15 2021 -0400
Make compactor sleep time a function of # of compactors
---
.../core/util/compaction/ExternalCompactionUtil.java | 18 ++
.../java/org/apache/accumulo/compactor/Compactor.java | 16 ++--
2 files changed, 32 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index cdf19e7..56114ef 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@ -250,4 +250,22 @@ public class ExternalCompactionUtil {
return runningIds;
}
+
+ public static int countCompactors(String queueName, ClientContext context) {
+String queueRoot = context.getZooKeeperRoot() + Constants.ZCOMPACTORS +
"/" + queueName;
+List children = context.getZooCache().getChildren(queueRoot);
+if (children == null)
+ return 0;
+
+int count = 0;
+
+for (String child : children) {
+ List children2 = context.getZooCache().getChildren(queueRoot +
"/" + child);
+ if (children2 != null && !children2.isEmpty()) {
+count++;
+ }
+}
+
+return count;
+ }
}
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index e1fc0db..17cc08d 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -479,7 +479,7 @@ public class Compactor extends AbstractServer
getContext().rpcCreds(), queueName,
ExternalCompactionUtil.getHostPortString(compactorAddress.getAddress()),
eci.toString());
-} catch (TException e) {
+} catch (Exception e) {
currentCompactionId.set(null);
throw e;
} finally {
@@ -603,7 +603,19 @@ public class Compactor extends AbstractServer
}
protected long getWaitTimeBetweenCompactionChecks() {
-return 3000;
+// get the total number of compactors assigned to this queue
+int numCompactors = ExternalCompactionUtil.countCompactors(queueName,
getContext());
+// Aim for around 3 compactors checking in every second
+long sleepTime = numCompactors * 1000 / 3;
+// Ensure a compactor sleeps at least around a second
+sleepTime = Math.max(1000, sleepTime);
+// Ensure a compactor sleep not too much more than 5 mins
+sleepTime = Math.min(300_000L, sleepTime);
+// Add some random jitter to the sleep time, that averages out to sleep
time. This will spread
+// compactors out evenly over time.
+sleepTime = (long) (.9 * sleepTime + sleepTime * .2 * Math.random());
+LOG.trace("Sleeping {}ms based on {} compactors", sleepTime,
numCompactors);
+return sleepTime;
}
@Override
[accumulo] branch main updated: Fix for Constraint API (#2074)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 6094a53 Fix for Constraint API (#2074)
6094a53 is described below
commit 6094a537421f7b2d328c8d1d016a2bf830743d4c
Author: Mike Miller
AuthorDate: Mon May 3 17:13:04 2021 -0400
Fix for Constraint API (#2074)
* Add method to deprecated API to prevent errors
* Add test to DeprecatedConstraintExtendTest
---
.../accumulo/core/constraints/Constraint.java | 11 +++
.../DeprecatedConstraintExtendTest.java| 38 --
2 files changed, 47 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
b/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
index 3414970..b24d613 100644
--- a/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
+++ b/core/src/main/java/org/apache/accumulo/core/constraints/Constraint.java
@@ -84,4 +84,15 @@ public interface Constraint extends
org.apache.accumulo.core.data.constraints.Co
* @return list of violation codes, or null if none
*/
List check(Environment env, Mutation mutation);
+
+ /**
+ * Implemented for backwards compatibility.
+ *
+ * @since 2.1.0
+ */
+ @Override
+ default List
check(org.apache.accumulo.core.data.constraints.Constraint.Environment env,
+ Mutation mutation) {
+return check((Environment) env, mutation);
+ }
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
b/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
index aa7ef99..dac943d 100644
---
a/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/constraints/DeprecatedConstraintExtendTest.java
@@ -40,13 +40,12 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@SuppressWarnings("deprecation")
public class DeprecatedConstraintExtendTest {
- Constraint constraint = new MinKeySizeConstraint();
-
byte[] min = new byte[1024];
byte[] oversized = new byte[1048577];
@Test
public void testMinKeySizeConstraint() {
+Constraint constraint = new MinKeySizeConstraint();
// pass constraints
Mutation m = new Mutation(min);
@@ -67,6 +66,20 @@ public class DeprecatedConstraintExtendTest {
constraint.check(null, m));
}
+ @Test
+ public void testFoo() {
+FooConstraint fc = new FooConstraint();
+// pass constraints
+Mutation m = new Mutation("blah");
+m.put("colf", "colq", new Value(new byte[] {}));
+assertEquals(null, fc.check(null, m));
+
+// test fail constraint
+m = new Mutation("foo");
+m.put("colf", "colq", new Value(new byte[] {}));
+assertEquals(Collections.singletonList(Short.valueOf("1")), fc.check(null,
m));
+ }
+
/**
* Limit the size of 1mb but also a minimum of 1KB
*/
@@ -94,4 +107,25 @@ public class DeprecatedConstraintExtendTest {
return violations;
}
}
+
+ /**
+ * Test previously defined constraint.
+ */
+ public class FooConstraint implements Constraint {
+public String getViolationDescription(short violationCode) {
+ switch (violationCode) {
+case 1:
+ return "Contains foo";
+ }
+ throw new IllegalArgumentException();
+}
+
+public List check(Constraint.Environment env, Mutation mutation) {
+ if (new String(mutation.getRow()).contains("foo")) {
+return Collections.singletonList(Short.valueOf("1"));
+ }
+ return null;
+}
+ }
+
}
[accumulo] branch 1451-external-compactions-feature updated (857231c -> 6daea69)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a change to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from 857231c Resolved some more CBUG markers
add 9afab2c Made private fields in Violations.CVSKey final (#2036)
add 032bfdc Fix typo in ConstraintChecker.java (#2037)
add da2b099 Add bulk import option to ignore empty dirs (#2039)
add d3b41fc Migrate Constraints to API (#1985)
add 35319d4 Move ignoreEmptyDir opt to ImportOptions interface (#2045)
add 4383b41 Fixed issue #2048 wrong name in CompactableFileImpl method
(#2050)
add 2831c29 Fix #2054 typo in ClientOps (conection-> connection)
add 4213d76 Fix redundant TableId creation in TableLoadBalancer (#2060)
add 323bbea #2049 Renamed method in SingletonManager (#2061)
add c180b34 Fix #2062 - Fix typo in VolumeImpl.java exception message.
add 747c48a Fix javadoc for LongCombiner (#2069)
add 97fe15c Fix formatting from #2069
add 17c0174 Fixes #2066 - Declaration of map in MapCounter.java as final
(#2071)
add f35ca73 Fix NumUtilTests to set Locale to US (#2072)
add c74e2ea #2040: replace immutable HashSet by Set.of() to reduce code *
adjusted imports * closes #2040
add 4f35e6a #2040: replace immutable HashSet by Set.of() to reduce code *
set locale in NumUtilTest to US * adjusted imports * closes #2040
add 7778203 Fix #2053 Fix typo -in TabletServerResourceManager
new 6daea69 Merge branch 'main' into 1451-external-compactions-feature
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
core/pom.xml | 2 +
.../org/apache/accumulo/core/cli/ClientOpts.java | 2 +-
.../core/client/admin/TableOperations.java | 7 ++
.../core/clientImpl/NamespaceOperationsImpl.java | 2 +-
.../core/clientImpl/TableOperationsImpl.java | 2 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 18 +++-
.../apache/accumulo/core/conf/IterConfigUtil.java | 2 +-
.../org/apache/accumulo/core/conf/Property.java| 2 +-
.../accumulo/core/constraints/Constraint.java | 26 ++
.../core/constraints/DefaultKeySizeConstraint.java | 13 ++-
.../core/constraints/NoDeleteConstraint.java | 12 ++-
.../accumulo/core/constraints/Violations.java | 4 +-
.../core/constraints/VisibilityConstraint.java | 13 ++-
.../core/{ => data}/constraints/Constraint.java| 19 +++--
.../constraints/DefaultKeySizeConstraint.java | 4 +-
.../{ => data}/constraints/NoDeleteConstraint.java | 4 +-
.../constraints/VisibilityConstraint.java | 4 +-
.../apache/accumulo/core/file/FileOperations.java | 12 ++-
.../accumulo/core/iterators/LongCombiner.java | 4 +-
.../core/metadata/CompactableFileImpl.java | 2 +-
.../accumulo/core/singletons/SingletonManager.java | 2 +-
.../core/singletons/SingletonReservation.java | 2 +-
.../core/spi/balancer/TableLoadBalancer.java | 6 +-
.../org/apache/accumulo/core/util/MapCounter.java | 2 +-
.../apache/accumulo/core/volume/VolumeImpl.java| 2 +-
.../client/security/tokens/KerberosTokenTest.java | 4 +-
.../constraints/DefaultKeySizeConstraintTest.java | 1 +
.../DeprecatedConstraintExtendTest.java| 97 ++
.../core/constraints/NoDeleteConstraintTest.java | 1 +
.../VisibilityConstraintTest.java | 11 +--
.../constraints/DefaultKeySizeConstraintTest.java | 7 +-
.../constraints/NoDeleteConstraintTest.java| 2 +-
.../constraints}/VisibilityConstraintTest.java | 7 +-
.../core/iterators/user/RowFilterTest.java | 25 +++---
.../org/apache/accumulo/core/util/NumUtilTest.java | 3 +
.../org/apache/accumulo/fate/AgeOffStoreTest.java | 23 +++--
.../server/constraints/MetadataConstraints.java| 16 ++--
.../server/constraints/SystemEnvironment.java | 2 +-
.../server/metadata/RootTabletMutatorImpl.java | 15 +++-
.../security/handler/KerberosAuthenticator.java| 7 +-
.../accumulo/server/util/FileSystemMonitor.java| 8 +-
.../tserver/TabletServerResourceManager.java | 8 +-
.../accumulo/tserver/TservConstraintEnv.java | 21 +++--
.../tserver/constraints/ConstraintChecker.java | 32 +--
.../tserver/constraints/SystemConstraint.java | 2 +-
.../constraints/UnsatisfiableConstraint.java | 2 +-
.../accumulo/tserver/tablet/CompactableImpl.java | 2 +-
.../tserver/constraints/ConstraintCheckerTest.java | 34
.../accumulo/shell/commands/ConstraintCommand.java | 2 +-
.../shell/commands/CreateTableCommand.java | 2 +-
.../shell/commands/ImportDirectoryCommand.java | 13 ++-
.../ap
[accumulo] 01/01: Merge branch 'main' into 1451-external-compactions-feature
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6daea692c0838d769a42657ba432385c0f69c035
Merge: 857231c 7778203
Author: Dave Marion
AuthorDate: Mon May 3 19:30:51 2021 +
Merge branch 'main' into 1451-external-compactions-feature
Conflicts:
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
core/pom.xml | 2 +
.../org/apache/accumulo/core/cli/ClientOpts.java | 2 +-
.../core/client/admin/TableOperations.java | 7 ++
.../core/clientImpl/NamespaceOperationsImpl.java | 2 +-
.../core/clientImpl/TableOperationsImpl.java | 2 +-
.../accumulo/core/clientImpl/bulk/BulkImport.java | 18 +++-
.../apache/accumulo/core/conf/IterConfigUtil.java | 2 +-
.../org/apache/accumulo/core/conf/Property.java| 2 +-
.../accumulo/core/constraints/Constraint.java | 26 ++
.../core/constraints/DefaultKeySizeConstraint.java | 13 ++-
.../core/constraints/NoDeleteConstraint.java | 12 ++-
.../accumulo/core/constraints/Violations.java | 4 +-
.../core/constraints/VisibilityConstraint.java | 13 ++-
.../core/{ => data}/constraints/Constraint.java| 19 +++--
.../constraints/DefaultKeySizeConstraint.java | 4 +-
.../{ => data}/constraints/NoDeleteConstraint.java | 4 +-
.../constraints/VisibilityConstraint.java | 4 +-
.../apache/accumulo/core/file/FileOperations.java | 12 ++-
.../accumulo/core/iterators/LongCombiner.java | 4 +-
.../core/metadata/CompactableFileImpl.java | 2 +-
.../accumulo/core/singletons/SingletonManager.java | 2 +-
.../core/singletons/SingletonReservation.java | 2 +-
.../core/spi/balancer/TableLoadBalancer.java | 6 +-
.../org/apache/accumulo/core/util/MapCounter.java | 2 +-
.../apache/accumulo/core/volume/VolumeImpl.java| 2 +-
.../client/security/tokens/KerberosTokenTest.java | 4 +-
.../constraints/DefaultKeySizeConstraintTest.java | 1 +
.../DeprecatedConstraintExtendTest.java| 97 ++
.../core/constraints/NoDeleteConstraintTest.java | 1 +
.../VisibilityConstraintTest.java | 11 +--
.../constraints/DefaultKeySizeConstraintTest.java | 7 +-
.../constraints/NoDeleteConstraintTest.java| 2 +-
.../constraints}/VisibilityConstraintTest.java | 7 +-
.../core/iterators/user/RowFilterTest.java | 25 +++---
.../org/apache/accumulo/core/util/NumUtilTest.java | 3 +
.../org/apache/accumulo/fate/AgeOffStoreTest.java | 23 +++--
.../server/constraints/MetadataConstraints.java| 16 ++--
.../server/constraints/SystemEnvironment.java | 2 +-
.../server/metadata/RootTabletMutatorImpl.java | 15 +++-
.../security/handler/KerberosAuthenticator.java| 7 +-
.../accumulo/server/util/FileSystemMonitor.java| 8 +-
.../tserver/TabletServerResourceManager.java | 8 +-
.../accumulo/tserver/TservConstraintEnv.java | 21 +++--
.../tserver/constraints/ConstraintChecker.java | 32 +--
.../tserver/constraints/SystemConstraint.java | 2 +-
.../constraints/UnsatisfiableConstraint.java | 2 +-
.../accumulo/tserver/tablet/CompactableImpl.java | 2 +-
.../tserver/constraints/ConstraintCheckerTest.java | 34
.../accumulo/shell/commands/ConstraintCommand.java | 2 +-
.../shell/commands/CreateTableCommand.java | 2 +-
.../shell/commands/ImportDirectoryCommand.java | 13 ++-
.../apache/accumulo/shell/commands/OptUtil.java| 20 +
.../shell/commands/DeleteTableCommandTest.java | 9 +-
.../shell/commands/ImportDirectoryCommandTest.java | 6 ++
.../org/apache/accumulo/test/OrIteratorIT.java | 3 +-
.../org/apache/accumulo/test/ShellServerIT.java| 75 ++---
.../apache/accumulo/test/TableOperationsIT.java| 2 +-
.../test/constraints/AlphaNumKeyConstraint.java| 2 +-
.../accumulo/test/constraints/MaxMutationSize.java | 2 +-
.../test/constraints/NumericValueConstraint.java | 2 +-
.../apache/accumulo/test/functional/BulkIT.java| 18
.../apache/accumulo/test/functional/BulkNewIT.java | 29 +++
.../accumulo/test/functional/ManagerMetricsIT.java | 11 +--
.../accumulo/test/functional/PermissionsIT.java| 3 +-
.../accumulo/test/functional/SlowConstraint.java | 2 +-
65 files changed, 495 insertions(+), 204 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index a398a71,fe61b55..0c598d7
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -96,8 -95,7 +96,8
[accumulo] branch 1451-external-compactions-feature updated: Resolved some more CBUG markers
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new 857231c Resolved some more CBUG markers
857231c is described below
commit 857231cae25f04bbb51ecfdbcb4ab7a4b8ca24d4
Author: Dave Marion
AuthorDate: Mon May 3 18:56:08 2021 +
Resolved some more CBUG markers
---
.../accumulo/core/util/compaction/ExternalCompactionUtil.java | 3 +--
.../java/org/apache/accumulo/coordinator/CompactionFinalizer.java | 2 +-
.../src/main/java/org/apache/accumulo/compactor/Compactor.java | 6 +++---
3 files changed, 5 insertions(+), 6 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index 0c5d45b..cdf19e7 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@ -110,8 +110,7 @@ public class ExternalCompactionUtil {
}
}
} catch (NoNodeException e) {
- // CBUG change to trace
- LOG.debug("Ignoring node that went missing", e);
+ LOG.trace("Ignoring node that went missing", e);
}
}
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
index 573105c..7e673b0 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionFinalizer.java
@@ -150,7 +150,7 @@ public class CompactionFinalizer {
List statusesToDelete = new ArrayList<>();
for (ExternalCompactionFinalState ecfs : batch) {
- // CBUG use #1974 for more efficient metadata reads
+ // TODO: use #1974 for more efficient metadata reads
TabletMetadata tabletMetadata = context.getAmple().readTablets()
.forTablet(ecfs.getExtent()).fetch(ColumnType.LOCATION,
ColumnType.PREV_ROW).build()
.stream().findFirst().orElse(null);
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 6ed7fa4..e1fc0db 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -378,9 +378,6 @@ public class Compactor extends AbstractServer
*/
protected void updateCompactionState(TExternalCompactionJob job,
TCompactionState state,
String message) throws RetriesExceededException {
-// CBUG the return type was changed from Void to String just to make this
work. When type was
-// Void and returned null, it would retry forever. Could specialize
RetryableThriftCall for case
-// w/ no return type.
RetryableThriftCall thriftCall = new RetryableThriftCall<>(1000,
RetryableThriftCall.MAX_WAIT_TIME, 25, new
RetryableThriftFunction() {
@Override
@@ -390,6 +387,9 @@ public class Compactor extends AbstractServer
coordinatorClient.get().updateCompactionStatus(TraceUtil.traceInfo(),
getContext().rpcCreds(), job.getExternalCompactionId(),
state, message,
System.currentTimeMillis());
+ // Note: the return type was changed from Void to String just to
make this work. When
+ // type was
+ // Void and returned null, it would retry forever.
return "";
} finally {
ThriftUtil.returnClient(coordinatorClient.getAndSet(null));
[accumulo] branch 1451-external-compactions-feature updated: Don't do work if no external compaction ids
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new ea0a604 Don't do work if no external compaction ids
ea0a604 is described below
commit ea0a6049f364dfd6ba5cc8e35a65a80eb5f6adc7
Author: Dave Marion
AuthorDate: Mon May 3 18:30:00 2021 +
Don't do work if no external compaction ids
---
.../org/apache/accumulo/coordinator/DeadCompactionDetector.java | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
index 46c6de5..afd2e2d 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/DeadCompactionDetector.java
@@ -123,10 +123,11 @@ public class DeadCompactionDetector {
.stream().flatMap(tm -> tm.getExternalCompactions().keySet().stream())
.forEach(danglingEcids::remove);
-danglingEcids.forEach(
-ecid -> log.debug("Detected dangling external compaction final state
marker {}", ecid));
-
-context.getAmple().deleteExternalCompactionFinalStates(danglingEcids);
+if (!danglingEcids.isEmpty()) {
+ danglingEcids.forEach(
+ ecid -> log.debug("Detected dangling external compaction final state
marker {}", ecid));
+ context.getAmple().deleteExternalCompactionFinalStates(danglingEcids);
+}
}
public void start() {
[accumulo] branch 1451-external-compactions-feature updated: Resolved some CBUG comments in Coordinator
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new 5ddd0cf Resolved some CBUG comments in Coordinator
5ddd0cf is described below
commit 5ddd0cfad2873ec2b961ba0e5cf8726947fa5d15
Author: Dave Marion
AuthorDate: Mon May 3 18:04:35 2021 +
Resolved some CBUG comments in Coordinator
---
.../org/apache/accumulo/core/conf/Property.java| 31 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 2 ++
.../coordinator/CompactionCoordinator.java | 20 +++---
.../accumulo/coordinator/CompactionFinalizer.java | 18 -
.../coordinator/DeadCompactionDetector.java| 8 +++---
.../accumulo/coordinator/QueueAndPriority.java | 2 +-
.../apache/accumulo/test/ExternalCompactionIT.java | 1 +
7 files changed, 48 insertions(+), 34 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 2cc6b23..d5a605e 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -1040,23 +1040,28 @@ public enum Property {
// CompactionCoordinator properties
COORDINATOR_PREFIX("coordinator.", null, PropertyType.PREFIX,
"Properties in this category affect the behavior of the accumulo
compaction coordinator server."),
- COORDINATOR_PORTSEARCH("coordinator.port.search", "false",
PropertyType.BOOLEAN,
+ COORDINATOR_THRIFTCLIENT_PORTSEARCH("coordinator.thrift.port.search",
"false",
+ PropertyType.BOOLEAN,
"if the ports above are in use, search higher ports until one is
available"),
- COORDINATOR_CLIENTPORT("coordinator.port.client", "9100", PropertyType.PORT,
+ COORDINATOR_THRIFTCLIENT_CLIENTPORT("coordinator.thrift.port.client",
"9100", PropertyType.PORT,
"The port used for handling Thrift client connections on the compaction
coordinator server"),
- COORDINATOR_MINTHREADS("coordinator.server.threads.minimum", "1",
PropertyType.COUNT,
- "The minimum number of threads to use to handle incoming requests."),
- COORDINATOR_MINTHREADS_TIMEOUT("coordinator.server.threads.timeout", "0s",
+
COORDINATOR_THRIFTCLIENT_MINTHREADS("coordinator.server.thrift.threads.minimum",
"1",
+ PropertyType.COUNT, "The minimum number of threads to use to handle
incoming requests."),
+
COORDINATOR_THRIFTCLIENT_MINTHREADS_TIMEOUT("coordinator.server.thrift.threads.timeout",
"0s",
PropertyType.TIMEDURATION,
"The time after which incoming request threads terminate with no work
available. Zero (0) will keep the threads alive indefinitely."),
- COORDINATOR_THREADCHECK("coordinator.server.threadcheck.time", "1s",
PropertyType.TIMEDURATION,
- "The time between adjustments of the server thread pool."),
- COORDINATOR_MAX_MESSAGE_SIZE("coordinator.server.message.size.max", "10M",
PropertyType.BYTES,
- "The maximum size of a message that can be sent to a tablet server."),
// deprecated
-
// properties grouped
-
// at the end to
-
// reference property
-
// that replaces them
+
COORDINATOR_THRIFTCLIENT_THREADCHECK("coordinator.server.thrift.threadcheck.time",
"1s",
+ PropertyType.TIMEDURATION, "The time between adjustments of the server
thread pool."),
+
COORDINATOR_THRIFTCLIENT_MAX_MESSAGE_SIZE("coordinator.server.thrift.message.size.max",
"10M",
+ PropertyType.BYTES, "The maximum size of a message that can be sent to a
tablet server."),
+
COORDINATOR_DEAD_COMPACTOR_CHECK_INTERVAL("coordinator.server.compactor.dead.check.interval",
+ "5m", PropertyType.TIMEDURATION, "The interval at which to check for
dead compactors."),
+
COORDINATOR_FINALIZER_TSERVER_NOTIFIER_MAXTHREADS("coordinator.server.finalizer.threads.maximum",
+ "5", PropertyType.COUNT,
+ "The maximum number of threads to use for notifying tablet servers that
an external compaction has completed."),
+
COORDINATOR_FINALIZER_COMPLETION_CHECK_INTERVAL("coordinator.server.finalizer.check.interval",
+ "60s", PropertyType.TIMEDURATION,
+ "The interval at which to check for external compaction final state
markers in the metadata table."),
// deprecated properties grouped at the end to reference property that
replaces them
@Deprecated(since = "1.6.0")
@ReplacedBy(property = INSTANCE_VOLUMES)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/met
[accumulo] branch 1451-external-compactions-feature updated: closes #2059 - add IT for coordinator restart during compaction
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new 10a295d closes #2059 - add IT for coordinator restart during
compaction
10a295d is described below
commit 10a295d6f8be22cdf5d015bfab58fd6672594393
Author: Dave Marion
AuthorDate: Mon May 3 16:53:50 2021 +
closes #2059 - add IT for coordinator restart during compaction
---
.../apache/accumulo/test/ExternalCompactionIT.java | 55 ++
1 file changed, 55 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
b/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
index df34c25..9ab5f55 100644
--- a/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
@@ -263,6 +263,61 @@ public class ExternalCompactionIT extends
ConfigurableMacBase {
}
@Test
+ public void testCoordinatorRestartsDuringCompaction() throws Exception {
+try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ String table1 = "ectt9";
+ createTable(client, table1, "cs1", 2);
+ writeData(client, table1);
+ cluster.exec(ExternalDoNothingCompactor.class, "-q", "DCQ1");
+ ProcessInfo process = cluster.exec(CompactionCoordinator.class);
+ compact(client, table1, 2, "DCQ1", false);
+ TableId tid = Tables.getTableId(getCluster().getServerContext(), table1);
+ // Wait for the compaction to start by waiting for 1 external compaction
column
+ Set ecids = new HashSet<>();
+ do {
+UtilWaitThread.sleep(50);
+try (TabletsMetadata tm =
getCluster().getServerContext().getAmple().readTablets()
+.forTable(tid).fetch(ColumnType.ECOMP).build()) {
+ tm.stream().flatMap(t ->
t.getExternalCompactions().keySet().stream())
+ .forEach(ecids::add);
+}
+ } while (ecids.isEmpty());
+
+ // Stop the Coordinator
+ Process coord = process.getProcess();
+ if (coord.supportsNormalTermination()) {
+cluster.stopProcessWithTimeout(coord, 60, TimeUnit.SECONDS);
+ } else {
+LOG.info("Stopping tserver manually");
+new ProcessBuilder("kill", Long.toString(coord.pid())).start();
+coord.waitFor();
+ }
+
+ // Start the TestCompactionCoordinator so that we have
+ // access to the metrics.
+ cluster.exec(TestCompactionCoordinator.class);
+
+ // Wait for coordinator to start
+ ExternalCompactionMetrics metrics = null;
+ while (null == metrics) {
+try {
+ metrics = getCoordinatorMetrics();
+} catch (Exception e) {
+ UtilWaitThread.sleep(250);
+}
+ }
+
+ // wait for failure or test timeout
+ metrics = getCoordinatorMetrics();
+ while (metrics.getRunning() == 0) {
+UtilWaitThread.sleep(250);
+metrics = getCoordinatorMetrics();
+ }
+
+}
+ }
+
+ @Test
public void testCompactionAndCompactorDies() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
// Stop the TabletServer so that it does not commit the compaction and
remove
[accumulo] branch 1451-external-compactions-feature updated: closes #2058 - added IT for Compactor dying during external compaction
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new b5214f6 closes #2058 - added IT for Compactor dying during external
compaction
b5214f6 is described below
commit b5214f6231acfc56480bddb8ccfb24260737a6e3
Author: Dave Marion
AuthorDate: Mon May 3 16:17:29 2021 +
closes #2058 - added IT for Compactor dying during external compaction
---
.../apache/accumulo/test/ExternalCompactionIT.java | 55 +-
1 file changed, 54 insertions(+), 1 deletion(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
b/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
index e790f96..df34c25 100644
--- a/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ExternalCompactionIT.java
@@ -79,6 +79,7 @@ import
org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState;
+import
org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState.FinalState;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
@@ -262,6 +263,59 @@ public class ExternalCompactionIT extends
ConfigurableMacBase {
}
@Test
+ public void testCompactionAndCompactorDies() throws Exception {
+try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
+ // Stop the TabletServer so that it does not commit the compaction and
remove
+ // the final state from the metadata table.
+ getCluster().getProcesses().get(TABLET_SERVER).forEach(p -> {
+try {
+ getCluster().killProcess(TABLET_SERVER, p);
+} catch (Exception e) {
+ fail("Failed to shutdown tablet server");
+}
+ });
+ // Start our TServer that will not commit the compaction
+ cluster.exec(ExternalCompactionTServer.class);
+
+ String table1 = "ectt8";
+ createTable(client, table1, "cs1", 2);
+ writeData(client, table1);
+ ProcessInfo process = cluster.exec(ExternalDoNothingCompactor.class,
"-q", "DCQ1");
+ cluster.exec(CompactionCoordinator.class);
+ compact(client, table1, 2, "DCQ1", false);
+ TableId tid = Tables.getTableId(getCluster().getServerContext(), table1);
+ // Wait for the compaction to start by waiting for 1 external compaction
column
+ Set ecids = new HashSet<>();
+ do {
+UtilWaitThread.sleep(50);
+try (TabletsMetadata tm =
getCluster().getServerContext().getAmple().readTablets()
+.forTable(tid).fetch(ColumnType.ECOMP).build()) {
+ tm.stream().flatMap(t ->
t.getExternalCompactions().keySet().stream())
+ .forEach(ecids::add);
+}
+ } while (ecids.isEmpty());
+
+ // Stop the Compactor
+ Process comp = process.getProcess();
+ if (comp.supportsNormalTermination()) {
+cluster.stopProcessWithTimeout(comp, 60, TimeUnit.SECONDS);
+ } else {
+LOG.info("Stopping tserver manually");
+new ProcessBuilder("kill", Long.toString(comp.pid())).start();
+comp.waitFor();
+ }
+}
+
+// DeadCompactionDetector in the CompactionCoordinator should fail the
compaction.
+long count = 0;
+while (count == 0) {
+ count =
getCluster().getServerContext().getAmple().getExternalCompactionFinalStates()
+ .filter(state ->
state.getFinalState().equals(FinalState.FAILED)).count();
+ UtilWaitThread.sleep(250);
+}
+ }
+
+ @Test
public void testMergeDuringExternalCompaction() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProperties()).build()) {
String table1 = "ectt7";
@@ -291,7 +345,6 @@ public class ExternalCompactionIT extends
ConfigurableMacBase {
.forEach(ecids::add);
}
} while (ecids.isEmpty());
- ;
var md = new ArrayList();
try (TabletsMetadata tm =
getCluster().getServerContext().getAmple().readTablets()
