This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5f52a5f7059691be915d781e9370d5295273dded Author: Murtadha Hubail <[email protected]> AuthorDate: Thu Jan 27 23:17:03 2022 +0300 [NO ISSUE][CONF] Removed unused active partitions config - user model changes: no - storage format changes: no - interface changes: no Details: - Remove unused active partitions config. - Improved node partitions logging. - Logging fixes. Change-Id: I4f2a611e2846405738401310f485db0e72844031 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/15024 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../apache/asterix/app/nc/NCAppRuntimeContext.java | 2 +- .../org/apache/asterix/app/nc/ReplicaManager.java | 11 +++-------- .../asterix/app/nc/task/UpdateNodeStatusTask.java | 4 ++-- .../app/replication/NcLifecycleCoordinator.java | 5 +++-- .../message/NCLifecycleTaskReportMessage.java | 6 ++++++ .../message/RegistrationTasksRequestMessage.java | 20 +++++++++++++------- .../message/RegistrationTasksResponseMessage.java | 8 ++------ .../asterix/hyracks/bootstrap/NCApplication.java | 4 ++-- .../asterix/common/config/MetadataProperties.java | 4 ++-- .../apache/asterix/common/config/NodeProperties.java | 3 +-- .../asterix/common/config/PropertiesAccessor.java | 18 +++++++----------- .../apache/asterix/common/utils/NcLocalCounters.java | 6 ++++++ 12 files changed, 48 insertions(+), 43 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java index f532352..1a89168 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java @@ -224,7 +224,7 @@ public class NCAppRuntimeContext implements INcApplicationContext { new DatasetLifecycleManager(storageProperties, localResourceRepository, txnSubsystem.getLogManager(), virtualBufferCache, indexCheckpointManagerProvider, ioManager.getIODevices().size()); final String nodeId = getServiceContext().getNodeId(); - final Set<Integer> nodePartitions = metadataProperties.getNodeActivePartitions(nodeId); + final Set<Integer> nodePartitions = metadataProperties.getNodePartitions(nodeId); replicaManager = new ReplicaManager(this, nodePartitions); isShuttingdown = false; activeManager = new ActiveManager(threadExecutor, getServiceContext().getNodeId(), diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java index 7c4b59c..7b52df6 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java @@ -19,7 +19,6 @@ package org.apache.asterix.app.nc; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -31,7 +30,6 @@ import java.util.stream.Stream; import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.api.INcApplicationContext; -import org.apache.asterix.common.cluster.ClusterPartition; import org.apache.asterix.common.replication.IPartitionReplica; import org.apache.asterix.common.storage.IReplicaManager; import org.apache.asterix.common.storage.ReplicaIdentifier; @@ -188,11 +186,8 @@ public class ReplicaManager implements IReplicaManager { } private void setNodeOwnedPartitions(INcApplicationContext appCtx) { - ClusterPartition[] clusterPartitions = - appCtx.getMetadataProperties().getNodePartitions().get(appCtx.getServiceContext().getNodeId()); - if (clusterPartitions != null) { - nodeOwnedPartitions.addAll(Arrays.stream(clusterPartitions).map(ClusterPartition::getPartitionId) - .collect(Collectors.toList())); - } + Set<Integer> nodePartitions = + appCtx.getMetadataProperties().getNodePartitions(appCtx.getServiceContext().getNodeId()); + nodeOwnedPartitions.addAll(nodePartitions); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java index fe579ad..17eff4a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java @@ -36,7 +36,7 @@ public class UpdateNodeStatusTask implements INCLifecycleTask { private static final Logger LOGGER = LogManager.getLogger(); private static final long serialVersionUID = 2L; private final NodeStatus status; - private Set<Integer> activePartitions; + private final Set<Integer> activePartitions; public UpdateNodeStatusTask(NodeStatus status, Set<Integer> activePartitions) { this.status = status; @@ -61,6 +61,6 @@ public class UpdateNodeStatusTask implements INCLifecycleTask { @Override public String toString() { - return "{ \"class\" : \"" + getClass().getSimpleName() + "\" }"; + return "UpdateNodeStatusTask{" + "status=" + status + ", activePartitions=" + activePartitions + '}'; } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java index 22a0a84..06005a9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java @@ -168,8 +168,9 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator { protected List<INCLifecycleTask> buildNCRegTasks(String nodeId, NodeStatus nodeStatus, SystemState state, Set<Integer> activePartitions) { - LOGGER.info("Building registration tasks for node {} with status {} and system state: {}", nodeId, nodeStatus, - state); + LOGGER.info( + "Building registration tasks for node {} with status {} and system state: {} and active partitions {}", + nodeId, nodeStatus, state, activePartitions); final boolean isMetadataNode = nodeId.equals(metadataNodeId); switch (nodeStatus) { case ACTIVE: diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java index 1309369..b2a2dd7 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java @@ -76,4 +76,10 @@ public class NCLifecycleTaskReportMessage implements INCLifecycleMessage, ICcAdd public Set<Integer> getActivePartitions() { return activePartitions; } + + @Override + public String toString() { + return "NCLifecycleTaskReportMessage{" + "nodeId='" + nodeId + '\'' + ", success=" + success + ", exception=" + + exception + ", localCounters=" + localCounters + ", activePartitions=" + activePartitions + '}'; + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java index fb50b3e..9e95ac6 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.app.replication.message; -import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -39,18 +38,18 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc private static final Logger LOGGER = LogManager.getLogger(); private static final long serialVersionUID = 2L; - protected final SystemState state; - protected final String nodeId; - protected final NodeStatus nodeStatus; - protected final Map<String, Object> secrets; - protected final Set<Integer> activePartitions; + private final SystemState state; + private final String nodeId; + private final NodeStatus nodeStatus; + private final Map<String, Object> secrets; + private final Set<Integer> activePartitions; public RegistrationTasksRequestMessage(String nodeId, NodeStatus nodeStatus, SystemState state, Map<String, Object> secretsEphemeral, Set<Integer> activePartitions) { this.state = state; this.nodeId = nodeId; this.nodeStatus = nodeStatus; - this.secrets = new HashMap<>(secretsEphemeral); + this.secrets = secretsEphemeral; this.activePartitions = activePartitions; } @@ -59,6 +58,7 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc try { RegistrationTasksRequestMessage msg = new RegistrationTasksRequestMessage(cs.getId(), nodeStatus, systemState, secretsEphemeral, activePartitions); + LOGGER.info("sending {} to CC", msg); ((INCMessageBroker) cs.getContext().getMessageBroker()).sendMessageToCC(ccId, msg); } catch (Exception e) { LOGGER.log(Level.ERROR, "Unable to send RegistrationTasksRequestMessage to CC", e); @@ -95,4 +95,10 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc public Set<Integer> getActivePartitions() { return activePartitions; } + + @Override + public String toString() { + return "RegistrationTasksRequestMessage{" + "state=" + state + ", nodeId='" + nodeId + '\'' + ", nodeStatus=" + + nodeStatus + ", activePartitions=" + activePartitions + '}'; + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java index f0a4a7c..f0f0470 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java @@ -59,13 +59,9 @@ public class RegistrationTasksResponseMessage extends CcIdentifiedMessage Throwable exception = null; try { for (INCLifecycleTask task : tasks) { - if (LOGGER.isInfoEnabled()) { - LOGGER.log(Level.INFO, "Starting startup task: " + task); - } + LOGGER.log(Level.INFO, "Starting startup task: {}", task); task.perform(getCcId(), cs); - if (LOGGER.isInfoEnabled()) { - LOGGER.log(Level.INFO, "Completed startup task: " + task); - } + LOGGER.log(Level.INFO, "Completed startup task: {}", task); } } catch (Throwable e) { //NOSONAR all startup failures should be reported to CC LOGGER.log(Level.ERROR, "Failed during startup task", e); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index be1cc7c..f60349f 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -300,12 +300,12 @@ public class NCApplication extends BaseNCApplication { final NodeStatus currentStatus = ncs.getNodeStatus(); final SystemState systemState = isPendingStartupTasks(currentStatus, ncs.getPrimaryCcId(), ccId) ? getCurrentSystemState() : SystemState.HEALTHY; - final Map httpSecrets = + final Map<String, Object> httpSecrets = apiServer != null ? Collections.singletonMap(SYS_AUTH_HEADER, apiServer.ctx().get(SYS_AUTH_HEADER)) : Collections.emptyMap(); RegistrationTasksRequestMessage.send(ccId, (NodeControllerService) ncServiceCtx.getControllerService(), currentStatus, systemState, httpSecrets, - runtimeContext.getMetadataProperties().getNodeActivePartitions(nodeId)); + runtimeContext.getMetadataProperties().getNodePartitions(nodeId)); } @Override diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java index 31708d3..252017f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java @@ -124,8 +124,8 @@ public class MetadataProperties extends AbstractProperties { return accessor.getClusterPartitions(); } - public Set<Integer> getNodeActivePartitions(String nodeId) { - return accessor.getActivePartitions(nodeId); + public Set<Integer> getNodePartitions(String nodeId) { + return accessor.getNodePartitions(nodeId); } public Map<String, String> getTransactionLogDirs() { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java index afb103d..522cabd 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/NodeProperties.java @@ -46,8 +46,7 @@ public class NodeProperties extends AbstractProperties { STARTING_PARTITION_ID( OptionTypes.INTEGER, -1, - "The first partition id to assign to iodevices on this node (-1 == auto-assign)"), - ACTIVE_PARTITIONS(OptionTypes.STRING_ARRAY, null, "List of node active partitions"); + "The first partition id to assign to iodevices on this node (-1 == auto-assign)"); private final IOptionType type; private final Object defaultValue; diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java index 5ba378d..80f9a17 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java @@ -41,6 +41,7 @@ import java.util.stream.Collectors; import org.apache.asterix.common.cluster.ClusterPartition; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.utils.PrintUtil; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.config.IApplicationConfig; @@ -69,7 +70,7 @@ public class PropertiesAccessor implements IApplicationConfig { /** * Constructor which wraps an IApplicationConfig. */ - private PropertiesAccessor(IApplicationConfig cfg) throws AsterixException, IOException { + private PropertiesAccessor(IApplicationConfig cfg) throws AsterixException { this.cfg = cfg; nodePartitionsMap = new ConcurrentHashMap<>(); clusterPartitions = Collections.synchronizedSortedMap(new TreeMap<>()); @@ -80,6 +81,7 @@ public class PropertiesAccessor implements IApplicationConfig { for (String ncName : cfg.getNCNames()) { configureNc(configManager, ncName, uniquePartitionId); } + LOGGER.info("configured partitions: {} from config {}", () -> PrintUtil.toString(nodePartitionsMap), () -> cfg); for (String section : cfg.getSectionNames()) { if (section.startsWith(AsterixProperties.SECTION_PREFIX_EXTENSION)) { String className = AsterixProperties.getSectionId(AsterixProperties.SECTION_PREFIX_EXTENSION, section); @@ -194,22 +196,16 @@ public class PropertiesAccessor implements IApplicationConfig { return clusterPartitions; } - public Set<Integer> getActivePartitions(String nodeId) { - // by default, node actives partitions are the partitions assigned to the node - String[] activePartitions = cfg.getStringArray(NodeProperties.Option.ACTIVE_PARTITIONS); - if (activePartitions == null) { - ClusterPartition[] nodeClusterPartitions = nodePartitionsMap.get(nodeId); - return Arrays.stream(nodeClusterPartitions).map(ClusterPartition::getPartitionId) - .collect(Collectors.toSet()); - } - return Arrays.stream(activePartitions).map(Integer::parseInt).collect(Collectors.toSet()); + public Set<Integer> getNodePartitions(String nodeId) { + ClusterPartition[] nodeClusterPartitions = nodePartitionsMap.get(nodeId); + return Arrays.stream(nodeClusterPartitions).map(ClusterPartition::getPartitionId).collect(Collectors.toSet()); } public List<AsterixExtension> getExtensions() { return extensions; } - public static PropertiesAccessor getInstance(IApplicationConfig cfg) throws IOException, AsterixException { + public static PropertiesAccessor getInstance(IApplicationConfig cfg) throws AsterixException { PropertiesAccessor accessor = instances.get(cfg); if (accessor == null) { accessor = new PropertiesAccessor(cfg); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java index 5cf6724..8956b93 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/NcLocalCounters.java @@ -59,4 +59,10 @@ public class NcLocalCounters implements Serializable { public long getMaxJobId() { return maxJobId; } + + @Override + public String toString() { + return "NcLocalCounters{" + "maxResourceId=" + maxResourceId + ", maxTxnId=" + maxTxnId + ", maxJobId=" + + maxJobId + '}'; + } }
