This is an automated email from the ASF dual-hosted git repository. Xiao-zhen-Liu pushed a commit to branch xiaozhen-caching-prototype in repository https://gitbox.apache.org/repos/asf/texera.git
commit 76c03feae6a671c69bf0da27ef7e8046fb4959e9 Merge: 57a492f24d 95496cce27 Author: Xiaozhen Liu <[email protected]> AuthorDate: Tue Feb 24 14:22:11 2026 -0800 Merge branch 'refs/heads/main' into xiaozhen-caching-prototype # Conflicts: # amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/QueryWorkerStatisticsHandler.scala # amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/WorkerExecutionCompletedHandler.scala # amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala .github/release/vote-email-template.md | 24 +- .github/workflows/build-and-push-images.yml | 72 +++-- .github/workflows/create-release-candidate.yml | 313 +++++++++++++++------ .github/workflows/github-action-build.yml | 19 +- .../engine/architecture/rpc/controlcommands.proto | 7 + .../input_port_materialization_reader_runnable.py | 3 +- .../architecture/controller/ClientEvent.scala | 3 + .../architecture/controller/Controller.scala | 3 + .../controller/ControllerTimerService.scala | 54 +++- .../controller/promisehandlers/PauseHandler.scala | 14 +- .../promisehandlers/PortCompletedHandler.scala | 11 +- .../QueryWorkerStatisticsHandler.scala | 58 +++- .../controller/promisehandlers/ResumeHandler.scala | 15 +- .../promisehandlers/StartWorkflowHandler.scala | 1 + .../WorkerExecutionCompletedHandler.scala | 5 +- .../WorkerStateUpdatedHandler.scala | 11 +- .../scheduling/RegionExecutionCoordinator.scala | 15 +- .../texera/web/ServletAwareConfigurator.scala | 2 + .../apache/texera/web/auth/GuestAuthFilter.scala | 2 +- .../apache/texera/web/auth/UserAuthenticator.scala | 1 + .../texera/web/resource/auth/AuthResource.scala | 2 + .../dashboard/admin/user/AdminUserResource.scala | 6 +- .../web/resource/dashboard/user/UserResource.scala | 61 ++-- .../texera/web/service/ExecutionStatsService.scala | 17 +- bin/k8s/README.md | 26 +- .../access-control-service-deployment.yaml | 2 +- bin/k8s/templates/config-service-deployment.yaml | 2 +- bin/k8s/templates/example-data-loader-job.yaml | 2 +- bin/k8s/templates/file-service-deployment.yaml | 2 +- bin/k8s/templates/webserver-deployment.yaml | 2 +- .../workflow-compiling-service-deployment.yaml | 2 +- ...workflow-computing-unit-manager-deployment.yaml | 4 +- ...ow-computing-unit-master-prepull-daemonset.yaml | 2 +- bin/k8s/values.yaml | 20 +- bin/single-node/.env | 5 + bin/single-node/docker-compose.yml | 18 +- .../scala/org/apache/texera/auth/JwtParser.scala | 3 +- common/config/src/main/resources/application.conf | 3 + .../texera/amber/config/ApplicationConfig.scala | 2 + common/workflow-core/build.sbt | 2 +- .../apache/texera/amber/operator/LogicalOp.scala | 4 +- .../sklearn/testing/SklearnTestingOpDesc.scala | 115 ++++++++ file-service/build.sbt | 2 +- .../texera/service/resource/DatasetResource.scala | 25 +- .../service/resource/DatasetResourceSpec.scala | 4 + frontend/.yarnrc.yml | 1 + frontend/src/app/app.module.ts | 4 + .../src/app/common/service/gmail/gmail.service.ts | 2 +- .../src/app/common/service/user/auth.service.ts | 104 ++++++- .../registration-request-modal.component.html | 54 ++++ .../registration-request-modal.component.scss} | 18 +- .../registration-request-modal.component.ts | 51 ++++ .../app/common/service/user/stub-user.service.ts | 2 + .../src/app/common/service/user/user.service.ts | 32 --- frontend/src/app/common/type/user.ts | 1 + .../component/admin/user/admin-user.component.html | 6 + .../component/admin/user/admin-user.component.ts | 5 + .../dashboard/component/dashboard.component.html | 32 --- .../app/dashboard/component/dashboard.component.ts | 67 +---- .../user/list-item/list-item.component.html | 15 +- .../user/list-item/list-item.component.scss | 1 + .../user/list-item/list-item.component.ts | 49 +++- .../markdown-description.component.html | 97 +++++++ .../markdown-description.component.scss | 95 +++++++ .../markdown-description.component.ts | 143 ++++++++++ .../dataset-detail.component.html | 8 +- .../dataset-detail.component.ts | 21 ++ .../detail/hub-workflow-detail.component.html | 2 +- .../coeditor-user-icon.component.ts | 10 +- .../workspace/component/menu/menu.component.html | 8 + .../app/workspace/component/menu/menu.component.ts | 18 ++ .../workflow-editor/workflow-editor.component.scss | 4 + .../workflow-editor/workflow-editor.component.ts | 49 +++- .../workspace/service/joint-ui/joint-ui.service.ts | 19 ++ .../src/assets/operator_images/SklearnTesting.png | Bin 0 -> 843070 bytes sql/texera_ddl.sql | 5 +- .../updates/21.sql | 19 +- .../updates/22.sql | 23 +- 78 files changed, 1465 insertions(+), 466 deletions(-) diff --cc amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala index abb8b1357a,810c098c41..b64e11eeb0 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/controller/promisehandlers/PortCompletedHandler.scala @@@ -30,13 -29,12 +30,14 @@@ import org.apache.texera.amber.engine.a import org.apache.texera.amber.engine.architecture.rpc.controlcommands.{ AsyncRPCContext, PortCompletedRequest, - QueryStatisticsRequest + QueryStatisticsRequest, + StatisticsUpdateTarget } import org.apache.texera.amber.engine.architecture.rpc.controlreturns.EmptyReturn +import org.apache.texera.amber.engine.architecture.scheduling.config.OutputPortConfig import org.apache.texera.amber.engine.common.virtualidentity.util.CONTROLLER import org.apache.texera.amber.util.VirtualIdentityUtils +import org.apache.texera.web.resource.dashboard.user.workflow.WorkflowExecutionsResource /** Notify the completion of a port: * - For input port, it means the worker has finished consuming and processing all the data diff --cc amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala index fe34536c47,e490cde3d9..cb5d3606f3 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionCoordinator.scala @@@ -37,8 -37,8 +37,9 @@@ import org.apache.texera.amber.engine.a } import org.apache.texera.amber.engine.architecture.controller.{ ControllerConfig, + ExecutionStateUpdate, ExecutionStatsUpdate, + RuntimeStatisticsPersist, WorkerAssignmentUpdate } import org.apache.texera.amber.engine.architecture.rpc.controlcommands._ @@@ -598,14 -490,9 +599,12 @@@ class RegionExecutionCoordinator region: Region, isDependeePhase: Boolean ): Future[Seq[Unit]] = { + if (region.cached) { + return Future.value(Seq.empty) + } - asyncRPCClient.sendToClient( - ExecutionStatsUpdate( - workflowExecution.getAllRegionExecutionsStats - ) - ) + val stats = workflowExecution.getAllRegionExecutionsStats + asyncRPCClient.sendToClient(ExecutionStatsUpdate(stats)) + asyncRPCClient.sendToClient(RuntimeStatisticsPersist(stats)) val allStarterOperators = region.getStarterOperators val starterOpsForThisPhase = if (isDependeePhase) allStarterOperators.filter(_.dependeeInputs.nonEmpty) diff --cc frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts index eda67e5973,40a4129071..ba016ff700 --- a/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts +++ b/frontend/src/app/workspace/service/joint-ui/joint-ui.service.ts @@@ -344,13 -322,15 +346,18 @@@ export class JointUIService const inputMetrics = statistics.inputPortMetrics; const outputMetrics = statistics.outputPortMetrics; + // Cached operators show "-" for inputs and non-materialized outputs, and label workers as "from cache". + const isSkippedFromCache = statistics.operatorState === OperatorState.CompletedFromCache; const workerCount = statistics.numWorkers ?? 1; - element.attr(`.${operatorWorkerCountClass}/text`, "#workers: " + String(workerCount)); + const workerCountLabel = isSkippedFromCache ? "from cache" : "#workers: " + String(workerCount); + element.attr(`.${operatorWorkerCountClass}/text`, workerCountLabel); + element.attr( + `.${operatorStatusTextClass}/text`, + "status: " + JointUIService.getStatusDisplayText(statistics.operatorState) + ); + inPorts.forEach(portDef => { const portId = portDef.id; if (portId != null) {
