This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new 4d4abab closes #2093, fix typo
4d4abab is described below
commit 4d4abab7e884c61d5ac6d657d8c028be88cdbed0
Author: Dave Marion <dlmar...@apache.org>
AuthorDate: Mon May 10 18:11:55 2021 +0000
closes #2093, fix typo
---
.../metadata/schema/ExternalCompactionMetadata.java | 2 +-
.../core/util/compaction/CompactionExecutorIdImpl.java | 6 +++---
.../org/apache/accumulo/server/init/Initialize.java | 1 -
.../apache/accumulo/manager/upgrade/Upgrader9to10.java | 17 +++++++++++++++++
.../accumulo/tserver/compactions/CompactionService.java | 2 +-
.../tserver/compactions/ExternalCompactionExecutor.java | 2 +-
.../apache/accumulo/tserver/tablet/CompactableImpl.java | 2 +-
7 files changed, 24 insertions(+), 8 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index 2295fc8..db70b62 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -138,7 +138,7 @@ public class ExternalCompactionMetadata {
jData.dest = newFile.getMetaInsert();
jData.compactor = compactorId;
jData.kind = kind.name();
- jData.executorId = ((CompactionExecutorIdImpl) ceid).getExernalName();
+ jData.executorId = ((CompactionExecutorIdImpl) ceid).getExternalName();
jData.priority = priority;
jData.propDels = propogateDeletes;
jData.selectedAll = selectedAll;
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java
index fa205b4..2dcb44c 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java
@@ -31,12 +31,12 @@ public class CompactionExecutorIdImpl extends
CompactionExecutorId {
private static final long serialVersionUID = 1L;
- public boolean isExernalId() {
+ public boolean isExternalId() {
return canonical().startsWith("e.");
}
- public String getExernalName() {
- Preconditions.checkState(isExernalId());
+ public String getExternalName() {
+ Preconditions.checkState(isExternalId());
return canonical().substring("e.".length());
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
index 24ebf49..ae6448a 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
@@ -660,7 +660,6 @@ public class Initialize implements KeywordExecutable {
NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + WalStateManager.ZWALS,
EMPTY_BYTE_ARRAY,
NodeExistsPolicy.FAIL);
- // TODO upgrade code should create following paths in ZK
zoo.putPersistentData(zkInstanceRoot + Constants.ZCOORDINATOR,
EMPTY_BYTE_ARRAY,
NodeExistsPolicy.FAIL);
zoo.putPersistentData(zkInstanceRoot + Constants.ZCOORDINATOR_LOCK,
EMPTY_BYTE_ARRAY,
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
index 41347a5..8f7c284 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
@@ -128,6 +128,7 @@ public class Upgrader9to10 implements Upgrader {
setMetaTableProps(ctx);
upgradeRootTabletMetadata(ctx);
renameOldMasterPropsinZK(ctx);
+ createExternalCompactionNodes(ctx);
}
@Override
@@ -162,6 +163,22 @@ public class Upgrader9to10 implements Upgrader {
}
}
+ private void createExternalCompactionNodes(ServerContext ctx) {
+
+ final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+ try {
+ ctx.getZooReaderWriter().putPersistentData(ctx.getZooKeeperRoot() +
Constants.ZCOORDINATOR,
+ EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ ctx.getZooReaderWriter().putPersistentData(
+ ctx.getZooKeeperRoot() + Constants.ZCOORDINATOR_LOCK,
EMPTY_BYTE_ARRAY,
+ NodeExistsPolicy.FAIL);
+ ctx.getZooReaderWriter().putPersistentData(ctx.getZooKeeperRoot() +
Constants.ZCOMPACTORS,
+ EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL);
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException("Unable to create external compaction paths",
e);
+ }
+ }
+
private void upgradeRootTabletMetadata(ServerContext ctx) {
String rootMetaSer = getFromZK(ctx, ZROOT_TABLET);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
index dc33fa0..e83b238 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
@@ -320,7 +320,7 @@ public class CompactionService {
plan = convertPlan(plan, kind, files.get().allFiles,
files.get().candidates);
if (compactable.getExtent().isMeta() && plan.getJobs().stream().map(cj ->
cj.getExecutor())
- .anyMatch(ceid -> ((CompactionExecutorIdImpl) ceid).isExernalId())) {
+ .anyMatch(ceid -> ((CompactionExecutorIdImpl) ceid).isExternalId())) {
log.error(
"Compacting metadata tablets on external compactors is not
supported, please change "
+ "config for compaction service ({}) and/or table ASAP. {} is
not compacting, "
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java
index 336978b..63d91b3 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java
@@ -199,7 +199,7 @@ public class ExternalCompactionExecutor implements
CompactionExecutor {
priority = topJob.getJob().getPriority();
}
- return new TCompactionQueueSummary(((CompactionExecutorIdImpl)
ceid).getExernalName(),
+ return new TCompactionQueueSummary(((CompactionExecutorIdImpl)
ceid).getExternalName(),
priority);
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 695ccae..6ff27f6 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -1172,7 +1172,7 @@ public class CompactableImpl implements Compactable {
// wait while internal jobs are running or external compactions are
committing, but do not
// wait on external compactions that are running
while (runnningJobs.stream()
- .anyMatch(job -> !((CompactionExecutorIdImpl)
job.getExecutor()).isExernalId())
+ .anyMatch(job -> !((CompactionExecutorIdImpl)
job.getExecutor()).isExternalId())
|| !externalCompactionsCommitting.isEmpty()) {
try {
wait(50);
