This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 3a0d8e5c72 Added compaction service configuration validation to
upgrade --start (#5807)
3a0d8e5c72 is described below
commit 3a0d8e5c72915fb4c7ce26d6156d15b0a444fb71
Author: Dave Marion <[email protected]>
AuthorDate: Thu Aug 21 09:11:56 2025 -0400
Added compaction service configuration validation to upgrade --start (#5807)
Added compaction service configuration validation to UpgradeUtil.
When `accumulo upgrade --start` is run it will fail if any of the
configured compaction services do not exist for tables other than
the root and metadata table. For the root and metadata table, it will
log a warning that, if not fixed, the compaction service configurations
will be removed from the root and metadata tables on the upgrade so
that they can use the default compaction service configuration.
Modified the Upgrader to implement this change.
Closes #5805
---
.../compaction/RatioBasedCompactionPlanner.java | 8 +-
.../apache/accumulo/server/util/UpgradeUtil.java | 115 +++++++++++++++++++++
.../accumulo/manager/upgrade/Upgrader12to13.java | 40 ++++++-
3 files changed, 158 insertions(+), 5 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
index fb44d038e6..11fcfb9a8d 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/RatioBasedCompactionPlanner.java
@@ -64,10 +64,10 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* <ul>
* <li>Note that the CompactionCoordinator and at least one running Compactor
must be assigned to
* the "large" compactor group.
- * <li>{@code compaction.service.<service>.opts.maxOpen} This determines the
maximum number of files
- * that will be included in a single compaction.
- * <li>{@code compaction.service.<service>.opts.groups} This is a json array
of compactor group
- * objects which have the following fields:
+ * <li>{@code compaction.service.<service>.planner.opts.maxOpen} This
determines the maximum number
+ * of files that will be included in a single compaction.
+ * <li>{@code compaction.service.<service>.planner.opts.groups} This is a json
array of compactor
+ * group objects which have the following fields:
* <table>
* <caption>Default Compaction Planner Group options</caption>
* <tr>
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/UpgradeUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/UpgradeUtil.java
index 86abc4a42c..352de5de26 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/UpgradeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/UpgradeUtil.java
@@ -18,23 +18,39 @@
*/
package org.apache.accumulo.server.util;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.accumulo.core.Constants.ZFATE;
import static org.apache.accumulo.core.Constants.ZPREPARE_FOR_UPGRADE;
+import static org.apache.accumulo.core.Constants.ZTABLES;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.cli.ConfigOpts;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.fate.zookeeper.ZooReader;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy;
+import org.apache.accumulo.core.metadata.SystemTables;
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
+import org.apache.accumulo.core.spi.compaction.CompactionDispatcher;
+import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.spi.compaction.CompactionServiceId;
+import org.apache.accumulo.core.spi.compaction.CompactionServices;
+import org.apache.accumulo.core.util.compaction.CompactionServicesConfig;
import org.apache.accumulo.core.zookeeper.ZooSession;
import org.apache.accumulo.server.AccumuloDataVersion;
import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.conf.NamespaceConfiguration;
+import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.util.upgrade.PreUpgradeValidation;
import org.apache.accumulo.server.util.upgrade.UpgradeProgress;
@@ -53,6 +69,7 @@ import com.google.auto.service.AutoService;
public class UpgradeUtil implements KeywordExecutable {
private static final Logger LOG = LoggerFactory.getLogger(UpgradeUtil.class);
+ private static final String ZTABLE_NAME = "/name";
static class Opts extends ConfigOpts {
@Parameter(names = "--prepare",
@@ -250,6 +267,14 @@ public class UpgradeUtil implements KeywordExecutable {
}
}
+ try {
+ validateCompactionServiceConfiguration(context);
+ LOG.info("Validated compaction service configuration");
+ } catch (KeeperException | InterruptedException e) {
+ LOG.error("Error validating compaction service configuration", e);
+ throw new IllegalStateException("Error validating compaction service
configuration", e);
+ }
+
// Run the PreUpgradeValidation code to validate the ZooKeeper ACLs
try {
new PreUpgradeValidation().validate(context);
@@ -304,4 +329,94 @@ public class UpgradeUtil implements KeywordExecutable {
}
+ private void validateCompactionServiceConfiguration(ServerContext ctx)
+ throws KeeperException, InterruptedException {
+
+ boolean configurationError = false;
+
+ final CompactionServicesConfig servicesConfig =
+ new CompactionServicesConfig(ctx.getConfiguration());
+ final Set<CompactionServiceId> definedServiceIds =
servicesConfig.getPlanners().keySet()
+
.stream().map(CompactionServiceId::of).collect(Collectors.toUnmodifiableSet());
+
+ LOG.info("Defined compaction service ids: {}", definedServiceIds);
+
+ final ZooReader zr = ctx.getZooSession().asReader();
+ List<String> zooTableIds = zr.getChildren(ZTABLES);
+
+ for (String tableId : zooTableIds) {
+
+ final String tableName =
+ new String(zr.getData(Constants.ZTABLES + "/" + tableId +
ZTABLE_NAME), UTF_8);
+ final String namespaceId = new String(
+ zr.getData(Constants.ZTABLES + "/" + tableId +
Constants.ZTABLE_NAMESPACE), UTF_8);
+
+ final NamespaceId nsid = NamespaceId.of(namespaceId);
+ final TableId tid = TableId.of(tableId);
+
+ final NamespaceConfiguration nsConf =
+ new NamespaceConfiguration(ctx, nsid, ctx.getConfiguration());
+ final TableConfiguration tconf = new TableConfiguration(ctx, tid,
nsConf);
+
+ final CompactionDispatcher dispatcher = tconf.getCompactionDispatcher();
+
+ for (CompactionKind kind : CompactionKind.values()) {
+ final CompactionDispatcher.DispatchParameters dispatchParams =
+ new CompactionDispatcher.DispatchParameters() {
+ @Override
+ public CompactionServices getCompactionServices() {
+ return () -> definedServiceIds;
+ }
+
+ @Override
+ public ServiceEnvironment getServiceEnv() {
+ return (ServiceEnvironment) ctx;
+ }
+
+ @Override
+ public CompactionKind getCompactionKind() {
+ return kind;
+ }
+
+ @Override
+ public Map<String,String> getExecutionHints() {
+ return Map.of();
+ }
+ };
+ final CompactionServiceId expectedCompactionService =
+ dispatcher.dispatch(dispatchParams).getService();
+ LOG.info("Table {} is configured to use service \"{}\" for compaction
kind {}", tableName,
+ expectedCompactionService, kind);
+ if
(!servicesConfig.getPlanners().containsKey(expectedCompactionService.canonical()))
{
+ if ((tid.equals(SystemTables.ROOT.tableId())
+ && expectedCompactionService.canonical().equals("root"))
+ || (tid.equals(SystemTables.METADATA.tableId())
+ && expectedCompactionService.canonical().equals("meta"))) {
+ LOG.warn(
+ "Table {} is using a default compaction service configuration
from a prior version."
+ + " The \"{}\" compaction service configuration is no
longer defined. You can either define"
+ + " it now in the accumulo.properties file, or the
compaction service configuration will"
+ + " be removed to adopt the new default in this version
during the upgrade.",
+ tableName, expectedCompactionService);
+ } else {
+ LOG.error(
+ "Table {} returned non-existent compaction service \"{}\" for
compaction type {}.",
+ tid, expectedCompactionService, kind);
+ configurationError = true;
+ }
+ }
+ }
+ }
+ if (configurationError) {
+ LOG.error("Compaction configuration is incorrect. One or more tables is
configured to use a"
+ + " compaction service that does not exist in the configuration.
Configured compaction"
+ + " services are: {}", definedServiceIds);
+ throw new IllegalStateException(
+ "Compaction configuration is not correct. Continuing with upgrade"
+ + " will leave the instance in a state where compactions will
not start for some tables. Please fix the system"
+ + " configuration by defining the expected compaction services
in accumulo.properties and run"
+ + " --start again.");
+ }
+ }
+
}
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
index 8bdd200f3e..ddb076e120 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader12to13.java
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
@@ -57,7 +58,9 @@ import
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.schema.Section;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.compaction.CompactionServicesConfig;
import org.apache.accumulo.server.ServerContext;
+import org.apache.accumulo.server.conf.codec.VersionedProperties;
import org.apache.accumulo.server.conf.store.TablePropKey;
import org.apache.accumulo.server.init.FileSystemInitializer;
import org.apache.accumulo.server.init.InitialConfiguration;
@@ -82,7 +85,9 @@ public class Upgrader12to13 implements Upgrader {
public void upgradeZookeeper(ServerContext context) {
LOG.info("Ensuring all worker server processes are down.");
validateEmptyZKWorkerServerPaths(context);
- LOG.info("setting root table stored hosting availability");
+ LOG.info("Validating root and metadata compaction services");
+ validateCompactionServiceConfiguration(context);
+ LOG.info("Setting root table stored hosting availability");
addHostingGoals(context, TabletAvailability.HOSTED, DataLevel.ROOT);
LOG.info("Removing nodes no longer used from ZooKeeper");
removeUnusedZKNodes(context);
@@ -441,4 +446,37 @@ public class Upgrader12to13 implements Upgrader {
"Could not read or write metadata in ZooKeeper because of ZooKeeper
exception", ex);
}
}
+
+ private void validateCompactionServiceConfiguration(ServerContext ctx) {
+
+ final String compactionSvcKey =
Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() + "service";
+ final Map<String,String> compactionPlanners =
+ new CompactionServicesConfig(ctx.getConfiguration()).getPlanners();
+
+ for (TableId tid : new TableId[] {SystemTables.ROOT.tableId(),
+ SystemTables.METADATA.tableId()}) {
+
+ final TablePropKey tpk = TablePropKey.of(tid);
+ final VersionedProperties tableProps = ctx.getPropStore().get(tpk);
+ final String value = tableProps.asMap().get(compactionSvcKey);
+
+ if (value != null) {
+ if (tid.equals(SystemTables.ROOT.tableId()) && value.equals("root")
+ && !compactionPlanners.containsKey(value)) {
+ LOG.warn(
+ "Compaction service \"root\" in configuration for root table,
but is not defined. "
+ + "Modifying root table configuration to use the default
compaction service configuration");
+ ctx.getPropStore().removeProperties(tpk, Set.of(compactionSvcKey));
+ } else if (tid.equals(SystemTables.METADATA.tableId()) &&
value.equals("meta")
+ && !compactionPlanners.containsKey(value)) {
+ LOG.warn(
+ "Compaction service \"meta\" in configuration for metadata
table, but is not defined. "
+ + "Modifying metadata table configuration to use the default
compaction service configuration");
+ ctx.getPropStore().removeProperties(tpk, Set.of(compactionSvcKey));
+ }
+ }
+
+ }
+ }
+
}
