Xiao-zhen-Liu commented on code in PR #4158:
URL: https://github.com/apache/texera/pull/4158#discussion_r2715387174
##########
amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala:
##########
@@ -340,4 +352,205 @@ class DataProcessingSpec
)
executeWorkflow(workflow)
}
+
+ // MATERIALIZED ExecutionMode tests - each operator should be a region
+ "Engine" should "execute headerlessCsv workflow with MATERIALIZED mode" in {
Review Comment:
The first 4 test cases are all 1-operator workflows, and will execute with
exactly the same behavior under pipelined mode. Consider removing redundant
test cases.
##########
amber/src/main/scala/org/apache/texera/amber/engine/architecture/scheduling/CostBasedScheduleGenerator.scala:
##########
@@ -477,6 +482,29 @@ class CostBasedScheduleGenerator(
)
}
+ /** Constructs a baseline fully materialized region plan (one operator per
region) and evaluates its cost. */
+ def materializedSearch(): SearchResult = {
Review Comment:
This is not actually doing a search, so the name is misleading. You can call
it something like `getFullyMaterializedSearchState`
##########
frontend/src/app/workspace/component/left-panel/settings/settings.component.ts:
##########
@@ -33,43 +33,51 @@ import { GuiConfigService } from
"../../../../common/service/gui-config.service"
styleUrls: ["./settings.component.scss"],
})
export class SettingsComponent implements OnInit {
- settingsForm!: FormGroup;
- currentDataTransferBatchSize!: number;
- isSaving: boolean = false;
+ settingsForm: FormGroup;
constructor(
private fb: FormBuilder,
private workflowActionService: WorkflowActionService,
private workflowPersistService: WorkflowPersistService,
private userService: UserService,
- private notificationService: NotificationService,
- private config: GuiConfigService
- ) {}
-
- ngOnInit(): void {
- this.currentDataTransferBatchSize =
-
this.workflowActionService.getWorkflowContent().settings.dataTransferBatchSize
||
- this.config.env.defaultDataTransferBatchSize;
-
+ private notificationService: NotificationService
+ ) {
this.settingsForm = this.fb.group({
- dataTransferBatchSize: [this.currentDataTransferBatchSize,
[Validators.required, Validators.min(1)]],
+ dataTransferBatchSize: [
+
this.workflowActionService.getWorkflowContent().settings.dataTransferBatchSize,
+ [Validators.required, Validators.min(1)],
+ ],
+ executionMode:
[this.workflowActionService.getWorkflowContent().settings.executionMode],
});
+ }
- this.settingsForm.valueChanges.pipe(untilDestroyed(this)).subscribe(value
=> {
- if (this.settingsForm.valid) {
- this.confirmUpdateDataTransferBatchSize(value.dataTransferBatchSize);
- }
- });
+ ngOnInit(): void {
+ this.settingsForm
+ .get("dataTransferBatchSize")!
+ .valueChanges.pipe(untilDestroyed(this))
+ .subscribe((batchSize: number) => {
+ if (this.settingsForm.get("dataTransferBatchSize")!.valid) {
+ this.confirmUpdateDataTransferBatchSize(batchSize);
+ }
+ });
+
+ this.settingsForm
+ .get("executionMode")!
+ .valueChanges.pipe(untilDestroyed(this))
+ .subscribe((mode: ExecutionMode) => {
+ this.updateExecutionMode(mode);
+ });
this.workflowActionService
.workflowChanged()
.pipe(untilDestroyed(this))
.subscribe(() => {
- this.currentDataTransferBatchSize =
-
this.workflowActionService.getWorkflowContent().settings.dataTransferBatchSize
||
- this.config.env.defaultDataTransferBatchSize;
+ console.log("workflow changed",
this.workflowActionService.getWorkflowContent().settings.executionMode);
Review Comment:
Is this needed?
##########
common/workflow-core/src/main/scala/org/apache/texera/amber/core/workflow/WorkflowSettings.scala:
##########
@@ -21,5 +21,6 @@ package org.apache.texera.amber.core.workflow
case class WorkflowSettings(
dataTransferBatchSize: Int,
- outputPortsNeedingStorage: Set[GlobalPortIdentity] = Set.empty
+ outputPortsNeedingStorage: Set[GlobalPortIdentity] = Set.empty,
+ executionMode: ExecutionMode
Review Comment:
I think it would be safer if there is also a default value here, and you
would not have to modify `customWorkflowSettings`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
