This is an automated email from the ASF dual-hosted git repository. aglinxinyuan pushed a commit to branch xiaozhen-sync-region-kill in repository https://gitbox.apache.org/repos/asf/texera.git
commit 937c6dfc64fc5d2318e2c8f1815f7f01042b9f25 Merge: 7a2038a542 ae68fc992a Author: Xinyuan Lin <[email protected]> AuthorDate: Tue Apr 28 20:50:00 2026 -0700 Merge branch 'main' into xiaozhen-sync-region-kill .github/workflows/build-and-push-images.yml | 6 +- .github/workflows/github-action-build.yml | 45 +- .gitignore | 7 + .licenserc.yaml | 3 + agent-service/.dockerignore | 2 + agent-service/.env.example | 15 + agent-service/.prettierrc | 13 + agent-service/bun.lock | 237 + agent-service/package.json | 37 + .../src/agent/index.ts | 9 +- agent-service/src/agent/prompts.ts | 296 + agent-service/src/agent/texera-agent.ts | 840 + .../src/agent/tools/index.ts | 9 +- agent-service/src/agent/tools/result-formatting.ts | 138 + agent-service/src/agent/tools/tools-utility.ts | 70 + .../src/agent/tools/workflow-crud-tools.ts | 346 + .../src/agent/tools/workflow-execution-tools.ts | 604 + agent-service/src/agent/util/auto-layout.ts | 70 + agent-service/src/agent/util/context-utils.ts | 288 + .../src/agent/util/workflow-system-metadata.ts | 268 + agent-service/src/agent/util/workflow-utils.ts | 209 + .../src/agent/workflow-result-state.test.ts | 95 + agent-service/src/agent/workflow-result-state.ts | 83 + agent-service/src/agent/workflow-state.test.ts | 176 + agent-service/src/agent/workflow-state.ts | 488 + agent-service/src/api/auth-api.ts | 65 + agent-service/src/api/backend-api.ts | 88 + agent-service/src/api/compile-api.ts | 74 + .../src/api/execution-api.ts | 24 +- .../src/api/index.ts | 11 +- agent-service/src/api/workflow-api.ts | 97 + .../src/config/env.ts | 27 +- .../src/index.ts | 14 +- .../src/logger.ts | 35 +- agent-service/src/server.test.ts | 223 + agent-service/src/server.ts | 663 + agent-service/src/types/agent.ts | 156 + .../src/types/execution.ts | 35 +- .../src/types/index.ts | 3 + agent-service/src/types/workflow.ts | 144 + agent-service/tsconfig.json | 23 + amber/requirements.txt | 2 - .../engine/architecture/rpc/controlcommands.proto | 22 +- .../architecture/rpc/controllerservice.proto | 1 + .../engine/architecture/rpc/workerservice.proto | 1 + .../handlers/control/debug_command_handler.py | 45 +- .../handlers/control/test_debug_command_handler.py | 195 + .../control/test_evaluate_expression_handler.py | 156 + .../control/test_replay_current_tuple_handler.py | 139 + .../handlers/control/update_executor_handler.py | 27 +- .../core/architecture/managers/pause_manager.py | 7 +- .../architecture/managers/test_debug_manager.py | 116 + .../rpc/async_rpc_handler_initializer.py | 4 + .../core/architecture/rpc/async_rpc_server.py | 21 +- amber/src/main/python/core/models/operator.py | 10 +- amber/src/main/python/core/models/state.py | 5 +- amber/src/main/python/core/runnables/main_loop.py | 21 +- .../main/python/core/runnables/network_receiver.py | 5 +- .../main/python/core/runnables/test_main_loop.py | 59 +- .../amber/engine/architecture/rpc/__init__.py | 99 +- .../architecture/common/AmberProcessor.scala | 5 +- .../ControllerAsyncRPCHandlerInitializer.scala | 3 +- .../controller/execution/WorkflowExecution.scala | 14 + .../promisehandlers/ReconfigurationHandler.scala | 141 + .../messaginglayer/NetworkInputGateway.scala | 6 + .../messaginglayer/NetworkOutputGateway.scala | 6 + .../scheduling/RegionExecutionCoordinator.scala | 17 +- .../scheduling/WorkflowExecutionCoordinator.scala | 8 +- .../engine/architecture/worker/DataProcessor.scala | 11 +- .../DataProcessorRPCHandlerInitializer.scala | 28 +- .../engine/architecture/worker/PauseType.scala | 2 - .../InitializeExecutorHandler.scala | 18 +- ...orHandler.scala => UpdateExecutorHandler.scala} | 29 +- .../common}/FriesReconfigurationAlgorithm.scala | 55 +- .../amber/engine/common/rpc/AsyncRPCClient.scala | 8 +- .../apache/texera/web/ComputingUnitMaster.scala | 7 +- .../web/resource/SyncExecutionResource.scala | 900 ++ .../service/ExecutionReconfigurationService.scala | 136 +- .../breakpoint/ExceptionBreakpointSpec.scala | 298 - .../architecture/control/TrivialControlSpec.scala | 45 +- .../control/utils/TrivialControlTester.scala | 14 +- .../messaginglayer/NetworkInputGatewaySpec.scala | 12 + .../apache/texera/amber/engine/e2e/PauseSpec.scala | 16 +- .../amber/engine/e2e/ReconfigurationSpec.scala | 327 + .../apache/texera/amber/engine/e2e/TestUtils.scala | 24 +- .../workflow/WorkflowExecutionsResourceSpec.scala | 40 + .../ExecutionReconfigurationServiceSpec.scala | 149 + .../amber/core/executor/OperatorExecutor.scala | 8 +- .../org/apache/texera/amber/core/state/State.scala | 5 +- .../texera/amber/operator/TestOperators.scala | 20 + frontend/.eslintrc.json | 12 +- frontend/angular.json | 42 +- frontend/karma.conf.js | 4 +- frontend/nx.json | 3 - frontend/package.json | 87 +- frontend/proxy.config.json | 13 +- frontend/src/app/app.component.ts | 1 + frontend/src/app/app.module.ts | 37 +- frontend/src/app/common/formly/array.type.ts | 1 + .../collab-wrapper/collab-wrapper.component.ts | 1 + frontend/src/app/common/formly/multischema.type.ts | 1 + frontend/src/app/common/formly/null.type.ts | 1 + frontend/src/app/common/formly/object.type.ts | 1 + .../preset-wrapper/preset-wrapper.component.ts | 1 + .../formly/repeat-dnd/repeat-dnd.component.ts | 1 + .../service/notification/notification.service.ts | 4 +- .../registration-request-modal.component.ts | 1 + .../workflow-persist.service.spec.ts | 38 +- .../src/app/common/util/computing-unit.util.ts | 1 + .../app/common/util/size-formatter.util.spec.ts | 2 +- .../admin/execution/admin-execution.component.ts | 1 + .../component/admin/gmail/admin-gmail.component.ts | 1 + .../settings/admin-settings.component.spec.ts | 3 +- .../admin/settings/admin-settings.component.ts | 1 + .../admin/user/admin-user.component.spec.ts | 5 +- .../component/admin/user/admin-user.component.ts | 1 + .../app/dashboard/component/dashboard.component.ts | 9 +- .../conflicting-file-modal-content.component.ts | 1 + .../files-uploader/files-uploader.component.ts | 1 + .../filters-instructions.component.ts | 1 + .../user/filters/filters.component.spec.ts | 2 + .../component/user/filters/filters.component.ts | 17 +- .../component/user/flarum/flarum.component.ts | 1 + .../user/list-item/list-item.component.ts | 3 +- .../markdown-description.component.ts | 1 + .../user/search-bar/search-bar.component.ts | 1 + .../search-results/search-results.component.ts | 1 + .../component/user/search/search.component.ts | 1 + .../user/share-access/share-access.component.ts | 1 + .../user/sort-button/sort-button.component.ts | 1 + .../user/user-avatar/user-avatar.component.ts | 1 + .../user-computing-unit-list-item.component.ts | 1 + .../user-computing-unit.component.spec.ts | 8 + .../user-computing-unit.component.ts | 1 + .../dataset-detail.component.ts | 1 + .../user-dataset-file-renderer.component.ts | 1 + .../user-dataset-staged-objects-list.component.ts | 1 + .../user-dataset-version-creator.component.ts | 1 + .../user-dataset-version-filetree.component.ts | 1 + .../user-dataset-list-item.component.html | 2 +- .../user-dataset-list-item.component.ts | 1 + .../user/user-dataset/user-dataset.component.ts | 1 + .../user/user-icon/user-icon.component.spec.ts | 2 + .../user/user-icon/user-icon.component.ts | 1 + .../public-project/public-project.component.ts | 1 + .../user-project-list-item.component.html | 6 +- .../user-project-list-item.component.ts | 1 + .../ngbd-modal-add-project-workflow.component.ts | 1 + ...ngbd-modal-remove-project-workflow.component.ts | 1 + .../user-project-section.component.ts | 1 + .../user/user-project/user-project.component.ts | 1 + .../user/user-quota/user-quota.component.html | 4 +- .../user/user-quota/user-quota.component.spec.ts | 10 +- .../user/user-quota/user-quota.component.ts | 10 +- .../workflow-execution-history.component.html | 4 +- .../workflow-execution-history.component.ts | 13 +- .../workflow-runtime-statistics.component.html | 4 +- .../workflow-runtime-statistics.component.ts | 22 +- .../highlight-search-terms.pipe.ts | 3 +- .../user-workflow-list-item.component.spec.ts | 4 +- .../user-workflow-list-item.component.ts | 1 + .../user-workflow/user-workflow.component.spec.ts | 4 +- .../user/user-workflow/user-workflow.component.ts | 1 + .../src/app/hub/component/about/about.component.ts | 1 + .../about/local-login/local-login.component.html | 4 +- .../about/local-login/local-login.component.ts | 3 +- .../browse-section/browse-section.component.ts | 1 + .../hub-search-result.component.ts | 1 + frontend/src/app/hub/component/hub.component.ts | 1 + .../landing-page/landing-page.component.ts | 1 + .../detail/hub-workflow-detail.component.ts | 1 + .../agent-chat/agent-chat.component.html | 424 - .../agent-chat/agent-chat.component.spec.ts | 65 - .../agent-panel/agent-chat/agent-chat.component.ts | 277 - .../component/agent-panel/agent-panel.component.ts | 200 - .../agent-interaction.component.html | 161 + .../agent-interaction.component.scss | 283 + .../agent-interaction.component.ts | 251 + .../agent-chat/agent-chat.component.html | 455 + .../agent-chat/agent-chat.component.scss | 109 +- .../agent-panel/agent-chat/agent-chat.component.ts | 712 + .../agent-panel/agent-panel.component.html | 60 +- .../agent-panel/agent-panel.component.scss | 15 + .../agent/agent-panel/agent-panel.component.ts | 364 + .../agent-registration.component.html | 13 +- .../agent-registration.component.scss | 0 .../agent-registration.component.ts | 51 +- .../react-step-detail-modal.component.html | 438 + .../react-step-detail-modal.component.scss} | 1 + .../react-step-detail-modal.component.ts | 222 + .../annotation-suggestion.component.ts | 1 + .../breakpoint-condition-input.component.spec.ts | 28 +- .../breakpoint-condition-input.component.ts | 5 +- .../code-debugger.component.spec.ts | 24 +- .../code-editor-dialog/code-debugger.component.ts | 20 +- .../code-editor-dialog/code-editor.component.ts | 8 +- .../codearea-custom-template.component.ts | 1 + .../dataset-file-selector.component.ts | 1 + .../dataset-selection-modal.component.ts | 1 + .../dataset-version-selector.component.ts | 1 + .../left-panel/left-panel.component.spec.ts | 23 +- .../component/left-panel/left-panel.component.ts | 3 +- .../operator-label/operator-label.component.ts | 1 + .../operator-menu/operator-menu.component.spec.ts | 33 +- .../operator-menu/operator-menu.component.ts | 1 + .../left-panel/settings/settings.component.ts | 1 + .../time-travel/time-travel.component.ts | 1 + .../versions-list/versions-list.component.html | 2 +- .../versions-list/versions-list.component.ts | 1 + .../coeditor-user-icon.component.css | 1 - .../coeditor-user-icon.component.ts | 1 + .../workspace/component/menu/menu.component.scss | 53 +- .../app/workspace/component/menu/menu.component.ts | 1 + .../computing-unit-selection.component.scss | 326 +- .../computing-unit-selection.component.ts | 1 + .../operator-property-edit-frame.component.spec.ts | 18 +- .../operator-property-edit-frame.component.ts | 25 +- .../port-property-edit-frame.component.ts | 1 + .../property-editor.component.spec.ts | 12 +- .../property-editor/property-editor.component.ts | 1 + .../type-casting-display.component.ts | 1 + .../result-exportation.component.ts | 1 + .../console-frame/console-frame.component.scss | 9 + .../console-frame/console-frame.component.ts | 1 + .../error-frame/error-frame.component.ts | 1 + .../result-panel/result-panel-modal.component.ts | 1 + .../result-panel/result-panel.component.html | 18 +- .../result-panel/result-panel.component.scss | 6 + .../result-panel/result-panel.component.ts | 1 + .../result-table-frame.component.ts | 1 + .../visualization-frame-content.component.scss | 6 +- .../visualization-frame-content.component.ts | 1 + .../nz-modal-comment-box.component.ts | 3 +- .../context-menu/context-menu.component.ts | 1 + .../workflow-editor/mini-map/mini-map.component.ts | 1 + .../workflow-editor/workflow-editor.component.html | 20 + .../workflow-editor/workflow-editor.component.scss | 65 +- .../workflow-editor.component.spec.ts | 9 +- .../workflow-editor/workflow-editor.component.ts | 266 +- .../workspace/component/workspace.component.html | 4 +- .../app/workspace/component/workspace.component.ts | 73 +- .../src/app/workspace/service/agent/agent-types.ts | 81 + .../app/workspace/service/agent/agent.service.ts | 1341 ++ .../copilot/texera-copilot-manager.service.spec.ts | 259 - .../copilot/texera-copilot-manager.service.ts | 248 - .../service/copilot/texera-copilot.spec.ts | 189 - .../workspace/service/copilot/texera-copilot.ts | 388 - .../current-workflow-editing-observing-tools.ts | 127 - .../copilot/tool/react-step-operator-parser.ts | 152 - .../service/copilot/tool/tools-utility.ts | 138 - .../copilot/tool/workflow-metadata-tools.ts | 140 - .../service/drag-drop/drag-drop.service.spec.ts | 16 +- .../dynamic-schema/dynamic-schema.service.spec.ts | 16 +- .../execute-workflow.service.spec.ts | 3 +- .../execute-workflow/execute-workflow.service.ts | 4 +- .../workspace/service/joint-ui/joint-ui.service.ts | 99 +- .../operator-metadata.service.spec.ts | 6 +- .../service/undo-redo/undo-redo.service.spec.ts | 4 +- .../validation/validation-workflow.service.spec.ts | 4 +- .../model/coeditor-presence.service.ts | 2 +- .../workflow-graph/model/joint-graph-wrapper.ts | 4 +- .../model/workflow-action.service.ts | 7 +- .../util/workflow-util.service.spec.ts | 2 +- frontend/src/main.ts | 6 +- frontend/src/tsconfig.app.json | 2 +- frontend/tsconfig.json | 17 +- frontend/yarn.lock | 15389 ++++++++----------- 267 files changed, 20566 insertions(+), 13020 deletions(-) diff --cc amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index a0c73b6506,db9c583a68..71991a6ab3 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@@ -181,7 -168,11 +182,11 @@@ class RegionExecutionCoordinator val actorRef = actorRefService.getActorRef(workerId) // Remove the actorRef so that no other actors can find the worker and send messages. actorRefService.removeActorRef(workerId) + // Restarted regions reuse actorId. Remove stale control channels so the + // controller does not reuse old control-message sequence numbers for new workers. + asyncRPCClient.inputGateway.removeControlChannel(workerId) + asyncRPCClient.outputGateway.removeControlChannel(workerId) - gracefulStop(actorRef, Duration(5, TimeUnit.SECONDS)).asTwitter() + gracefulStop(actorRef, ScalaDuration(5, TimeUnit.SECONDS)).asTwitter() } }.toSeq diff --cc amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala index 1c3ae89471,5a9c84c701..4b639fc241 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionCoordinator.scala @@@ -83,22 -79,21 +83,28 @@@ class WorkflowExecutionCoordinator } // All existing regions are completed. Start the next region (if any). + val nextRegions = getNextRegions() + if (nextRegions.isEmpty) { + if (workflowExecution.isCompleted && completionNotified.compareAndSet(false, true)) { + asyncRPCClient.sendToClient(ExecutionStateUpdate(workflowExecution.getState)) + } + return Future.Unit + } + + executedRegions.append(nextRegions) Future - .collect({ - val nextRegions = getNextRegions() - executedRegions.append(nextRegions) + .collect( nextRegions .map(region => { - workflowExecution.initRegionExecution(region) + val isRestart = workflowExecution.hasRegionExecution(region.id) + if (isRestart) { + workflowExecution.restartRegionExecution(region) + } else { + workflowExecution.initRegionExecution(region) + } regionExecutionCoordinators(region.id) = new RegionExecutionCoordinator( region, + isRestart, workflowExecution, asyncRPCClient, controllerConfig,
