>From Michael Blow <[email protected]>: Michael Blow has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21986?usp=email )
Change subject: [NO ISSUE][FUN] Skip inactive partitions in library job splits ...................................................................... [NO ISSUE][FUN] Skip inactive partitions in library job splits - user model changes: no - storage format changes: no - interface changes: no Details: ExternalLibraryJobUtils built the create/drop library jobs over every registered cluster partition, active or not. A failed node's partitions stay registered, deactivated, until the node is removed or rejoins, and in a cloud deployment, where compute partitions are fixed, nothing takes them over. So after a failover the jobs were scheduled on the failed node and rejected with HYR0010 (node does not exist), the abort job the same way, and the library could not be created, replaced or dropped until the node was dealt with, though the cluster was ACTIVE. Ext-ref: MB-74378 Generated-by: Claude Opus 5.5 <[email protected]> Change-Id: I45e60290961bdd325bf9734bb8806e2c9357afa9 --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java A asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalLibraryJobUtilsTest.java 2 files changed, 72 insertions(+), 1 deletion(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/86/21986/1 diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java index fd829c9..68cac36 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryJobUtils.java @@ -126,12 +126,17 @@ return StoragePathUtil.splitProviderAndPartitionConstraints(splits); } - private static FileSplit[] getSplits(IClusterStateManager clusterStateManager) { + static FileSplit[] getSplits(IClusterStateManager clusterStateManager) { ClusterPartition[] clusterPartitions = clusterStateManager.getClusterPartitons(); Arrays.sort(clusterPartitions, Comparator.comparing(ClusterPartition::getNodeId).thenComparingInt(ClusterPartition::getIODeviceNum)); List<FileSplit> splits = new ArrayList<>(); for (ClusterPartition partition : clusterPartitions) { + // a failed node's partitions stay registered, deactivated, until it is removed or rejoins; when + // compute partitions are fixed (cloud) nothing takes them over, so a job sent there cannot start + if (!partition.isActive()) { + continue; + } String activeNodeId = partition.getActiveNodeId(); int n = splits.size(); if (n > 0 && splits.get(n - 1).getNodeName().equals(activeNodeId)) { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalLibraryJobUtilsTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalLibraryJobUtilsTest.java new file mode 100644 index 0000000..6c7007f --- /dev/null +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/external/ExternalLibraryJobUtilsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.external; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.asterix.common.cluster.ClusterPartition; +import org.apache.asterix.common.cluster.IClusterStateManager; +import org.apache.hyracks.api.io.FileSplit; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class ExternalLibraryJobUtilsTest { + + @Test + public void oneSplitPerActiveNode() { + IClusterStateManager csm = clusterOf(partition(0, "nc1", 0, true), partition(1, "nc1", 1, true), + partition(2, "nc2", 0, true), partition(3, "nc2", 1, true)); + Assert.assertEquals(List.of("nc1", "nc2"), splitNodes(csm)); + } + + @Test + public void inactivePartitionsAreSkipped() { + // nc3 has failed: its partition is deactivated but still registered against it + IClusterStateManager csm = + clusterOf(partition(0, "nc1", 0, true), partition(1, "nc2", 0, true), partition(2, "nc3", 0, false)); + Assert.assertEquals(List.of("nc1", "nc2"), splitNodes(csm)); + } + + private static List<String> splitNodes(IClusterStateManager csm) { + return Arrays.stream(ExternalLibraryJobUtils.getSplits(csm)).map(FileSplit::getNodeName) + .collect(Collectors.toList()); + } + + private static ClusterPartition partition(int id, String node, int ioDevice, boolean active) { + ClusterPartition partition = new ClusterPartition(id, node, ioDevice); + partition.setActiveNodeId(node); + partition.setActive(active); + return partition; + } + + private static IClusterStateManager clusterOf(ClusterPartition... partitions) { + IClusterStateManager csm = Mockito.mock(IClusterStateManager.class); + Mockito.when(csm.getClusterPartitons()).thenReturn(partitions); + return csm; + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21986?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I45e60290961bdd325bf9734bb8806e2c9357afa9 Gerrit-Change-Number: 21986 Gerrit-PatchSet: 1 Gerrit-Owner: Michael Blow <[email protected]>
