caishunfeng commented on code in PR #16327: URL: https://github.com/apache/dolphinscheduler/pull/16327#discussion_r1728550236
########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/TriggerWorkflowExecutorDelegate.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.workflow; + +import org.apache.dolphinscheduler.api.validator.workflow.TriggerWorkflowDTO; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam; +import org.apache.dolphinscheduler.service.process.TriggerRelationService; + +import java.util.Date; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TriggerWorkflowExecutorDelegate implements IExecutorDelegate<TriggerWorkflowDTO, Void> { + + @Autowired + private CommandDao commandDao; + + @Autowired + private TriggerRelationService triggerRelationService; + + @Override + public Void execute(TriggerWorkflowDTO triggerWorkflowDTO) { + final RunWorkflowCommandParam runWorkflowCommandParam = + RunWorkflowCommandParam.builder() + .commandParams(triggerWorkflowDTO.getStartParamList()) + .startNodes(triggerWorkflowDTO.getStartNodes()) + .timeZone(DateUtils.getTimezone()) + .build(); + final Command command = Command.builder() + .commandType(triggerWorkflowDTO.getExecType()) Review Comment: Is it necessary to add type check? ########## dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/TaskGroupCoordinator.java: ########## @@ -421,8 +421,11 @@ public boolean needToReleaseTaskGroupSlot(TaskInstance taskInstance) { * @throws IllegalArgumentException If the taskInstance is null or the task doesn't use task group. */ public void releaseTaskGroupSlot(TaskInstance taskInstance) { - if (taskInstance == null || taskInstance.getTaskGroupId() <= 0) { - throw new IllegalArgumentException("The current TaskInstance does not use task group"); + if (taskInstance == null) { + throw new IllegalArgumentException("The TaskInstance is null"); + } + if (taskInstance.getTaskGroupId() <= 0) { Review Comment: It's better to add some warn log if dirty data. ########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/workflow/TriggerWorkflowRequestValidator.java: ########## @@ -15,18 +15,20 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.server.master.runner; +package org.apache.dolphinscheduler.api.validator.workflow; -import java.util.concurrent.Callable; +import org.apache.dolphinscheduler.api.validator.IValidator; -public interface IWorkflowExecuteRunnable extends Callable<WorkflowStartStatus> { - // todo: add control method to manage the workflow runnable e.g. pause/stop .... +import lombok.extern.slf4j.Slf4j; - @Override - default WorkflowStartStatus call() { - return startWorkflow(); - } +import org.springframework.stereotype.Component; - WorkflowStartStatus startWorkflow(); +@Slf4j +@Component +public class TriggerWorkflowRequestValidator implements IValidator<TriggerWorkflowDTO> { + @Override + public void validate(final TriggerWorkflowDTO triggerWorkflowDTO) { Review Comment: It seems need to add some check. ########## dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cluster/MasterSlotManager.java: ########## @@ -89,13 +89,19 @@ public void doReBalance(List<MasterServerMetadata> normalMasterServers) { } if (tmpCurrentSlot == -1) { log.warn( - "Do re balance failed, cannot found the current master: {} in the normal master clusters: {}. Please check the current master server status", + "Do rebalance failed, cannot found the current master: {} in the normal master clusters: {}. Please check the current master server status", masterConfig.getMasterAddress(), normalMasterServers); currentSlot = -1; return; } + if (totalSlots == normalMasterServers.size() && currentSlot == tmpCurrentSlot) { + log.debug("No need to rebalance, the currentSlot: {}, totalSlots: {} doesn't changed", currentSlot, + totalSlots); + return; + } totalSlots = normalMasterServers.size(); currentSlot = tmpCurrentSlot; + log.info("DO rebalance success, current master slot: {}, total master slots: {}", currentSlot, totalSlots); Review Comment: ```suggestion log.info("Do rebalance success, current master slot: {}, total master slots: {}", currentSlot, totalSlots); ``` ########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/PauseWorkflowInstanceExecutorDelegate.java: ########## @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.workflow; + +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; +import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory; +import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceController; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstancePauseRequest; +import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstancePauseResponse; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class PauseWorkflowInstanceExecutorDelegate + implements + IExecutorDelegate<PauseWorkflowInstanceExecutorDelegate.PauseWorkflowInstanceOperation, Void> { + + @Autowired + private ProcessInstanceDao workflowInstanceDao; + + @Override + public Void execute(PauseWorkflowInstanceOperation workflowInstanceControlRequest) { + final ProcessInstance workflowInstance = workflowInstanceControlRequest.workflowInstance; + exceptionIfWorkflowInstanceCannotPause(workflowInstance); + if (ifWorkflowInstanceCanDirectPauseInDB(workflowInstance)) { + directPauseInDB(workflowInstance); + } else { + pauseInMaster(workflowInstance); + } + return null; + } + + private void exceptionIfWorkflowInstanceCannotPause(ProcessInstance workflowInstance) { + WorkflowExecutionStatus workflowInstanceState = workflowInstance.getState(); + if (workflowInstanceState.canPause()) { + return; + } + throw new ServiceException( + "The workflow instance: " + workflowInstance.getName() + " status is " + workflowInstanceState + + ", can not pause"); + } + + private boolean ifWorkflowInstanceCanDirectPauseInDB(ProcessInstance workflowInstance) { + return workflowInstance.getState().canDirectPauseInDB(); + } + + private void directPauseInDB(ProcessInstance workflowInstance) { + workflowInstanceDao.updateWorkflowInstanceState( + workflowInstance.getId(), + workflowInstance.getState(), + WorkflowExecutionStatus.PAUSE); + log.info("Update workflow instance {} state from: {} to {} success", + workflowInstance.getName(), + workflowInstance.getState().name(), + WorkflowExecutionStatus.PAUSE.name()); + } + + private void pauseInMaster(ProcessInstance workflowInstance) { + try { + final WorkflowInstancePauseResponse pauseResponse = SingletonJdkDynamicRpcClientProxyFactory + .withService(IWorkflowInstanceController.class) + .withHost(workflowInstance.getHost()) + .pauseWorkflowInstance(new WorkflowInstancePauseRequest(workflowInstance.getId())); + + if (pauseResponse != null && pauseResponse.isSuccess()) { + log.info("WorkflowInstance: {} pause success", workflowInstance.getName()); + } else { + throw new ServiceException( + "WorkflowInstance: " + workflowInstance.getName() + " stop failed: " + pauseResponse); Review Comment: ```suggestion "WorkflowInstance: " + workflowInstance.getName() + " pause failed: " + pauseResponse); ``` ########## dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/executor/workflow/StopWorkflowInstanceExecuteFunctionTest.java: ########## @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.workflow; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class StopWorkflowInstanceExecuteFunctionTest { Review Comment: :+1: ########## dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEngine.java: ########## @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.master.engine; + +import org.apache.dolphinscheduler.server.master.engine.command.CommandEngine; +import org.apache.dolphinscheduler.server.master.runner.MasterTaskExecutorBootstrap; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class WorkflowEngine implements AutoCloseable { Review Comment: :+1: ########## dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskExecutionEvent.java: ########## @@ -17,7 +17,11 @@ package org.apache.dolphinscheduler.extract.master.transportor; -public interface ITaskInstanceExecutionEvent { +public interface ITaskExecutionEvent { Review Comment: Suggest to add `event source`, which can help troubleshoot. ########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/WorkflowInstanceControlRequest.java: ########## @@ -15,19 +15,22 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.api.executor.workflow.instance.stop; +package org.apache.dolphinscheduler.api.executor.workflow; -import org.apache.dolphinscheduler.api.executor.ExecuteRequest; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.User; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; -import lombok.NonNull; @Data +@Builder @AllArgsConstructor -public class StopRequest implements ExecuteRequest { +public class WorkflowInstanceControlRequest { Review Comment: The mean of `xxxxControlRequest` is too broad, please add some docs. ########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/TriggerWorkflowExecutorDelegate.java: ########## @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.workflow; + +import org.apache.dolphinscheduler.api.validator.workflow.TriggerWorkflowDTO; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.repository.CommandDao; +import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam; +import org.apache.dolphinscheduler.service.process.TriggerRelationService; + +import java.util.Date; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class TriggerWorkflowExecutorDelegate implements IExecutorDelegate<TriggerWorkflowDTO, Void> { + + @Autowired + private CommandDao commandDao; + + @Autowired + private TriggerRelationService triggerRelationService; + + @Override + public Void execute(TriggerWorkflowDTO triggerWorkflowDTO) { Review Comment: It's better to check field in `TriggerWorkflowDTO`, such as `getWorkflowDefinition` is not null.... ########## dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml: ########## @@ -152,10 +152,10 @@ </foreach> </if> </update> - <update id="updateProcessInstanceByState"> + <update id="updateWorkflowInstanceState"> update t_ds_process_instance set state = #{destState} Review Comment: ```suggestion set state = #{targetState} ``` ########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/RecoverSuspendedWorkflowInstanceExecutorDelegate.java: ########## @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.workflow; + +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.CommandDao; + +import java.util.Date; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class RecoverSuspendedWorkflowInstanceExecutorDelegate + implements + IExecutorDelegate<RecoverSuspendedWorkflowInstanceExecutorDelegate.RecoverSuspendedWorkflowInstanceOperation, Void> { + + @Autowired + private CommandDao commandDao; + + @Override + public Void execute(RecoverSuspendedWorkflowInstanceOperation workflowInstanceControlRequest) { Review Comment: use void is better and do not need to return null? ```suggestion public void execute(RecoverSuspendedWorkflowInstanceOperation workflowInstanceControlRequest) { ``` ########## dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/command/ICommandParam.java: ########## @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.master.command; + +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.plugin.task.api.model.Property; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, visible = true, property = "commandType") +@JsonSubTypes({ + @JsonSubTypes.Type(value = ScheduleWorkflowCommandParam.class, name = "SCHEDULER"), + @JsonSubTypes.Type(value = RunWorkflowCommandParam.class, name = "START_PROCESS"), + @JsonSubTypes.Type(value = BackfillWorkflowCommandParam.class, name = "COMPLEMENT_DATA"), + @JsonSubTypes.Type(value = ReRunWorkflowCommandParam.class, name = "REPEAT_RUNNING"), + @JsonSubTypes.Type(value = RecoverFailureTaskCommandParam.class, name = "START_FAILURE_TASK_PROCESS"), + @JsonSubTypes.Type(value = WorkflowFailoverCommandParam.class, name = "RECOVER_TOLERANCE_FAULT"), +}) +public interface ICommandParam { + + /** + * The task which need to be as the beginning of the workflow. + */ + List<Long> getStartNodes(); + + /** + * The command params. + */ + List<Property> getCommandParams(); + + /** + * Get the time zone. + * todo: we should remove this field. + */ + String getTimeZone(); + + /** + * Whether the command is used to trigger a sub workflow instance. + */ + boolean isSubWorkflowInstance(); Review Comment: Why need to distinguish whether it is a sub workflow instance? ########## dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskExecutionEvent.java: ########## @@ -38,9 +42,13 @@ public interface ITaskInstanceExecutionEvent { TaskInstanceExecutionEventType getEventType(); enum TaskInstanceExecutionEventType { + DISPATCH, RUNNING, + PAUSED, + KILLED, + FAILED, + SUCCESS, FINISH, Review Comment: Is it still need `FINISH` event type? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
