Repository: asterixdb Updated Branches: refs/heads/master c8ba268f7 -> 31589f960
[NO ISSUE][CLUS] Fail Tasks When NC Not Active - user model changes: no - storage format changes: no - interface changes: no Details: - Fail StartTasksWork if NC is not in active state (i.e. hasn't completed startup tasks). - Set NC state to active only after completing startup tasks. - Add test case. Change-Id: I447d4f1e255211e9026b68eb9d931f25846ed153 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2460 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Michael Blow <mb...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/31589f96 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/31589f96 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/31589f96 Branch: refs/heads/master Commit: 31589f9604be0ed5026ceca696934b4a612d42ea Parents: c8ba268 Author: Murtadha Hubail <mhub...@apache.org> Authored: Thu Mar 8 00:28:04 2018 +0300 Committer: Murtadha Hubail <mhub...@apache.org> Committed: Thu Mar 8 13:44:33 2018 -0800 ---------------------------------------------------------------------- .../app/replication/NcLifecycleCoordinator.java | 3 +- .../hyracks/bootstrap/NCApplication.java | 4 +- .../apache/asterix/runtime/NcLifecycleTest.java | 64 ++++++++++++++++++++ .../apache/hyracks/api/client/NodeStatus.java | 7 ++- .../hyracks/api/exceptions/ErrorCode.java | 1 + .../src/main/resources/errormsg/en.properties | 1 + .../control/nc/NodeControllerService.java | 6 +- .../hyracks/control/nc/work/StartTasksWork.java | 5 ++ 8 files changed, 83 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java index 844851a..5c497aa 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java @@ -146,7 +146,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator { return buildActiveNCRegTasks(isMetadataNode); } final List<INCLifecycleTask> tasks = new ArrayList<>(); - tasks.add(new UpdateNodeStatusTask(NodeStatus.ACTIVE)); + tasks.add(new UpdateNodeStatusTask(NodeStatus.BOOTING)); if (state == SystemState.CORRUPTED) { //need to perform local recovery for node partitions LocalRecoveryTask rt = new LocalRecoveryTask(Arrays.asList(clusterManager.getNodePartitions(nodeId)) @@ -167,6 +167,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator { tasks.add(new BindMetadataNodeTask()); } tasks.add(new ReportLocalCountersTask()); + tasks.add(new UpdateNodeStatusTask(NodeStatus.ACTIVE)); return tasks; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java index 0cd01ae..494198b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java @@ -89,6 +89,8 @@ public class NCApplication extends BaseNCApplication { @Override public void init(IServiceContext serviceCtx) throws Exception { ncServiceCtx = (INCServiceContext) serviceCtx; + // set the node status initially to idle to indicate that it is pending booting + ((NodeControllerService) serviceCtx.getControllerService()).setNodeStatus(NodeStatus.IDLE); ncServiceCtx.setThreadFactory( new AsterixThreadFactory(ncServiceCtx.getThreadFactory(), ncServiceCtx.getLifeCycleComponentManager())); } @@ -253,7 +255,7 @@ public class NCApplication extends BaseNCApplication { } private boolean isPendingStartupTasks(NodeStatus nodeStatus, CcId primaryCc, CcId registeredCc) { - return nodeStatus == NodeStatus.BOOTING && (primaryCc == null || primaryCc.equals(registeredCc)); + return nodeStatus == NodeStatus.IDLE && (primaryCc == null || primaryCc.equals(registeredCc)); } private SystemState getCurrentSystemState() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java new file mode 100644 index 0000000..df0740b --- /dev/null +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NcLifecycleTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime; + +import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil; +import org.apache.asterix.common.TestDataUtil; +import org.apache.asterix.common.config.GlobalConfig; +import org.apache.asterix.test.common.TestExecutor; +import org.apache.asterix.testframework.context.TestCaseContext; +import org.apache.hyracks.api.client.NodeStatus; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class NcLifecycleTest { + + protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf"; + private static final TestExecutor testExecutor = new TestExecutor(); + private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); + + @Before + public void setUp() throws Exception { + System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME); + integrationUtil.init(true, TEST_CONFIG_FILE_NAME); + } + + @After + public void tearDown() throws Exception { + integrationUtil.deinit(true); + } + + @Test + public void nodeNotActiveTest() throws Exception { + final String datasetName = "ds"; + TestDataUtil.createIdOnlyDataset(datasetName); + integrationUtil.ncs[0].setNodeStatus(NodeStatus.BOOTING); + boolean thrown = false; + try { + testExecutor.executeSqlppUpdateOrDdl("select count(*) from " + datasetName + ";", + TestCaseContext.OutputFormat.CLEAN_JSON); + } catch (Exception e) { + thrown = true; + Assert.assertTrue(e.getMessage().contains(integrationUtil.ncs[0].getId())); + } + Assert.assertTrue(thrown); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java index 10a9a3c..076dd10 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeStatus.java @@ -19,7 +19,8 @@ package org.apache.hyracks.api.client; public enum NodeStatus { - ACTIVE, - BOOTING, - DEAD + ACTIVE, // node is ready to process tasks + BOOTING, // node is executing startup tasks + DEAD, + IDLE } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java index 0dca782..0691005 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java @@ -147,6 +147,7 @@ public class ErrorCode { public static final int CANNOT_CONTINUE_TEXT_SEARCH_BUFFER_MANAGER_IS_NULL = 111; public static final int CANNOT_ADD_ELEMENT_TO_INVERTED_INDEX_SEARCH_RESULT = 112; public static final int UNDEFINED_INVERTED_LIST_MERGE_TYPE = 113; + public static final int NODE_IS_NOT_ACTIVE = 114; // Compilation error codes. public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties index 50e1ad4..8635efd 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties @@ -130,5 +130,6 @@ 111 = To conduct an inverted-index search, the given buffer manager cannot be null. 112 = Cannot add an element to an inverted-index search result. 113 = Undefined inverted-list merge type: %1$s +114 = Node (%1$s) is not active 10000 = The given rule collection %1$s is not an instance of the List class. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index b67bfac..2c73df1 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -170,7 +170,7 @@ public class NodeControllerService implements IControllerService { private final Map<CcId, AtomicLong> maxJobIds = new ConcurrentHashMap<>(); - private NodeStatus status = NodeStatus.BOOTING; + private volatile NodeStatus status = NodeStatus.ACTIVE; private NodeRegistration nodeRegistration; @@ -633,11 +633,11 @@ public class NodeControllerService implements IControllerService { return workQueue; } - public synchronized NodeStatus getNodeStatus() { + public NodeStatus getNodeStatus() { return status; } - public synchronized void setNodeStatus(NodeStatus status) { + public void setNodeStatus(NodeStatus status) { this.status = status; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/31589f96/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java index e229149..6a5785a 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/StartTasksWork.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import org.apache.hyracks.api.application.INCServiceContext; +import org.apache.hyracks.api.client.NodeStatus; import org.apache.hyracks.api.comm.IFrameWriter; import org.apache.hyracks.api.comm.IPartitionCollector; import org.apache.hyracks.api.comm.IPartitionWriterFactory; @@ -45,6 +46,7 @@ import org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.deployment.DeploymentId; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.job.ActivityCluster; @@ -116,6 +118,9 @@ public class StartTasksWork extends AbstractWork { ncs.updateMaxJobId(jobId); NCServiceContext serviceCtx = ncs.getContext(); Joblet joblet = getOrCreateLocalJoblet(deploymentId, serviceCtx, acgBytes); + if (ncs.getNodeStatus() != NodeStatus.ACTIVE) { + throw HyracksException.create(ErrorCode.NODE_IS_NOT_ACTIVE, ncs.getId()); + } final ActivityClusterGraph acg = joblet.getActivityClusterGraph(); IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() { @Override