This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch 1451-external-compactions-feature in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1451-external-compactions-feature by this push: new 4d4abab closes #2093, fix typo 4d4abab is described below commit 4d4abab7e884c61d5ac6d657d8c028be88cdbed0 Author: Dave Marion <dlmar...@apache.org> AuthorDate: Mon May 10 18:11:55 2021 +0000 closes #2093, fix typo --- .../metadata/schema/ExternalCompactionMetadata.java | 2 +- .../core/util/compaction/CompactionExecutorIdImpl.java | 6 +++--- .../org/apache/accumulo/server/init/Initialize.java | 1 - .../apache/accumulo/manager/upgrade/Upgrader9to10.java | 17 +++++++++++++++++ .../accumulo/tserver/compactions/CompactionService.java | 2 +- .../tserver/compactions/ExternalCompactionExecutor.java | 2 +- .../apache/accumulo/tserver/tablet/CompactableImpl.java | 2 +- 7 files changed, 24 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java index 2295fc8..db70b62 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java @@ -138,7 +138,7 @@ public class ExternalCompactionMetadata { jData.dest = newFile.getMetaInsert(); jData.compactor = compactorId; jData.kind = kind.name(); - jData.executorId = ((CompactionExecutorIdImpl) ceid).getExernalName(); + jData.executorId = ((CompactionExecutorIdImpl) ceid).getExternalName(); jData.priority = priority; jData.propDels = propogateDeletes; jData.selectedAll = selectedAll; diff --git a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java index fa205b4..2dcb44c 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionExecutorIdImpl.java @@ -31,12 +31,12 @@ public class CompactionExecutorIdImpl extends CompactionExecutorId { private static final long serialVersionUID = 1L; - public boolean isExernalId() { + public boolean isExternalId() { return canonical().startsWith("e."); } - public String getExernalName() { - Preconditions.checkState(isExernalId()); + public String getExternalName() { + Preconditions.checkState(isExternalId()); return canonical().substring("e.".length()); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java index 24ebf49..ae6448a 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java @@ -660,7 +660,6 @@ public class Initialize implements KeywordExecutable { NodeExistsPolicy.FAIL); zoo.putPersistentData(zkInstanceRoot + WalStateManager.ZWALS, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL); - // TODO upgrade code should create following paths in ZK zoo.putPersistentData(zkInstanceRoot + Constants.ZCOORDINATOR, EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL); zoo.putPersistentData(zkInstanceRoot + Constants.ZCOORDINATOR_LOCK, EMPTY_BYTE_ARRAY, diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java index 41347a5..8f7c284 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java @@ -128,6 +128,7 @@ public class Upgrader9to10 implements Upgrader { setMetaTableProps(ctx); upgradeRootTabletMetadata(ctx); renameOldMasterPropsinZK(ctx); + createExternalCompactionNodes(ctx); } @Override @@ -162,6 +163,22 @@ public class Upgrader9to10 implements Upgrader { } } + private void createExternalCompactionNodes(ServerContext ctx) { + + final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + try { + ctx.getZooReaderWriter().putPersistentData(ctx.getZooKeeperRoot() + Constants.ZCOORDINATOR, + EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL); + ctx.getZooReaderWriter().putPersistentData( + ctx.getZooKeeperRoot() + Constants.ZCOORDINATOR_LOCK, EMPTY_BYTE_ARRAY, + NodeExistsPolicy.FAIL); + ctx.getZooReaderWriter().putPersistentData(ctx.getZooKeeperRoot() + Constants.ZCOMPACTORS, + EMPTY_BYTE_ARRAY, NodeExistsPolicy.FAIL); + } catch (KeeperException | InterruptedException e) { + throw new RuntimeException("Unable to create external compaction paths", e); + } + } + private void upgradeRootTabletMetadata(ServerContext ctx) { String rootMetaSer = getFromZK(ctx, ZROOT_TABLET); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java index dc33fa0..e83b238 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java @@ -320,7 +320,7 @@ public class CompactionService { plan = convertPlan(plan, kind, files.get().allFiles, files.get().candidates); if (compactable.getExtent().isMeta() && plan.getJobs().stream().map(cj -> cj.getExecutor()) - .anyMatch(ceid -> ((CompactionExecutorIdImpl) ceid).isExernalId())) { + .anyMatch(ceid -> ((CompactionExecutorIdImpl) ceid).isExternalId())) { log.error( "Compacting metadata tablets on external compactors is not supported, please change " + "config for compaction service ({}) and/or table ASAP. {} is not compacting, " diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java index 336978b..63d91b3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/ExternalCompactionExecutor.java @@ -199,7 +199,7 @@ public class ExternalCompactionExecutor implements CompactionExecutor { priority = topJob.getJob().getPriority(); } - return new TCompactionQueueSummary(((CompactionExecutorIdImpl) ceid).getExernalName(), + return new TCompactionQueueSummary(((CompactionExecutorIdImpl) ceid).getExternalName(), priority); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java index 695ccae..6ff27f6 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java @@ -1172,7 +1172,7 @@ public class CompactableImpl implements Compactable { // wait while internal jobs are running or external compactions are committing, but do not // wait on external compactions that are running while (runnningJobs.stream() - .anyMatch(job -> !((CompactionExecutorIdImpl) job.getExecutor()).isExernalId()) + .anyMatch(job -> !((CompactionExecutorIdImpl) job.getExecutor()).isExternalId()) || !externalCompactionsCommitting.isEmpty()) { try { wait(50);