[
https://issues.apache.org/jira/browse/BEAM-13560?focusedWorklogId=707466&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-707466
]
ASF GitHub Bot logged work on BEAM-13560:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 12/Jan/22 13:07
Start Date: 12/Jan/22 13:07
Worklog Time Spent: 10m
Work Description: ilya-kozyrev commented on a change in pull request
#16477:
URL: https://github.com/apache/beam/pull/16477#discussion_r783053929
##########
File path: playground/backend/internal/code_processing/code_processing.go
##########
@@ -210,30 +154,122 @@ func Process(ctx context.Context, cacheService
cache.Cache, lc *fs_tool.LifeCycl
_ = processRunSuccess(pipelineLifeCycleCtx, pipelineId, cacheService,
stopReadLogsChannel, finishReadLogsChannel)
}
+func compileStep(ctx context.Context, cacheService cache.Cache, lc
*fs_tool.LifeCycle, pipelineId uuid.UUID, sdkEnv *environment.BeamEnvs,
isUnitTest bool, pipelineLifeCycleCtx context.Context, cancelChannel chan bool)
*executors.Executor {
+ errorChannel, successChannel := createStatusChannels()
+ var executor = executors.Executor{}
+ // This condition is used for cases when the playground doesn't compile
source files. For the Python code and the Go Unit Tests
+ if sdkEnv.ApacheBeamSdk == pb.Sdk_SDK_PYTHON || (sdkEnv.ApacheBeamSdk
== pb.Sdk_SDK_GO && isUnitTest) {
+ if err := processCompileSuccess(pipelineLifeCycleCtx,
[]byte(""), pipelineId, cacheService); err != nil {
+ return nil
+ }
+ } else { // in case of Java, Go (not unit test), Scala - need compile
step
+ // Compile
+ executorBuilder := builder.SetupCompilerBuilder(lc, sdkEnv)
+ executor := executorBuilder.Build()
+ logger.Infof("%s: Compile() ...\n", pipelineId)
+ compileCmd := executor.Compile(pipelineLifeCycleCtx)
+ var compileError bytes.Buffer
+ var compileOutput bytes.Buffer
+ runCmdWithOutput(compileCmd, &compileOutput, &compileError,
successChannel, errorChannel)
+
+ // Start of the monitoring of background tasks (compile
step/cancellation/timeout)
+ ok, err := reconcileBackgroundTask(pipelineLifeCycleCtx, ctx,
pipelineId, cacheService, cancelChannel, successChannel)
+ if err != nil {
+ return nil
+ }
+ if !ok { // Compile step is finished, but code couldn't be
compiled (some typos for example)
+ err := <-errorChannel
+ _ = processErrorWithSavingOutput(pipelineLifeCycleCtx,
err, compileError.Bytes(), pipelineId, cache.CompileOutput, cacheService,
"Compile", pb.Status_STATUS_COMPILE_ERROR)
+ return nil
+ } // Compile step is finished and code is compiled
+ if err := processCompileSuccess(pipelineLifeCycleCtx,
compileOutput.Bytes(), pipelineId, cacheService); err != nil {
+ return nil
+ }
+ }
+ return &executor
+}
+
+func prepareStep(ctx context.Context, cacheService cache.Cache, lc
*fs_tool.LifeCycle, pipelineId uuid.UUID, sdkEnv *environment.BeamEnvs,
pipelineLifeCycleCtx context.Context, validationResults *sync.Map,
cancelChannel chan bool) *executors.Executor {
+ errorChannel, successChannel := createStatusChannels()
+ executorBuilder, err := builder.SetupPreparatorBuilder(lc, sdkEnv)
+ if err != nil {
+ _ = processSetupError(err, pipelineId, cacheService,
pipelineLifeCycleCtx)
+ return nil
+ }
+ executor := executorBuilder.Build()
+ // Prepare
Review comment:
do we need this comment?
##########
File path: playground/backend/internal/setup_tools/builder/setup_builder.go
##########
@@ -31,73 +31,134 @@ const (
javaLogConfigFilePlaceholder = "{logConfigFile}"
)
-// SetupExecutorBuilder return executor with set args for validator,
preparator, compiler and runner
-func SetupExecutorBuilder(lc *fs_tool.LifeCycle, pipelineOptions string,
sdkEnv *environment.BeamEnvs) (*executors.ExecutorBuilder, error) {
+// SetupValidatorBuilder return executor with set args for validator
+func SetupValidatorBuilder(lc *fs_tool.LifeCycle, sdkEnv
*environment.BeamEnvs) (*executors.ExecutorBuilder, error) {
sdk := sdkEnv.ApacheBeamSdk
-
- if sdk == pb.Sdk_SDK_JAVA {
- pipelineOptions = utils.ReplaceSpacesWithEquals(pipelineOptions)
- }
-
val, err := utils.GetValidators(sdk, lc.GetAbsoluteSourceFilePath())
if err != nil {
return nil, err
}
+ builder := executors.NewExecutorBuilder().
+ WithValidator().
+ WithSdkValidators(val).
+ ExecutorBuilder
+ return &builder, err
+}
+
+// SetupPreparatorBuilder return executor with set args for preparator
+func SetupPreparatorBuilder(lc *fs_tool.LifeCycle, sdkEnv
*environment.BeamEnvs) (*executors.ExecutorBuilder, error) {
+ sdk := sdkEnv.ApacheBeamSdk
prep, err := utils.GetPreparators(sdk, lc.GetAbsoluteSourceFilePath())
if err != nil {
return nil, err
}
- executorConfig := sdkEnv.ExecutorConfig
builder := executors.NewExecutorBuilder().
- WithExecutableFileName(lc.GetAbsoluteExecutableFilePath()).
- WithWorkingDir(lc.GetAbsoluteBaseFolderPath()).
- WithValidator().
- WithSdkValidators(val).
WithPreparator().
WithSdkPreparators(prep).
+ ExecutorBuilder
+ return &builder, err
+}
+
+// SetupCompilerBuilder return executor with set args for compiler
+func SetupCompilerBuilder(lc *fs_tool.LifeCycle, sdkEnv *environment.BeamEnvs)
*executors.ExecutorBuilder {
Review comment:
What about naming it just Compiler? Then we could use it like that:
`setup_tools.builder.Compiler()`
and the same for all other builders in setup_tools
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 707466)
Time Spent: 1h (was: 50m)
> [Playground] Split builder (refactoring)
> ----------------------------------------
>
> Key: BEAM-13560
> URL: https://issues.apache.org/jira/browse/BEAM-13560
> Project: Beam
> Issue Type: Improvement
> Components: beam-playground
> Reporter: Daria Malkova
> Assignee: Daria Malkova
> Priority: P3
> Labels: beam-playground-backend, beam-playground-sprint-7
> Time Spent: 1h
> Remaining Estimate: 0h
>
> Improve usage of executor builder by splitting it to parts.
> Now executor builder is created at once for all steps: validation,
> preparation, compiling, and run/test. To make a change at some step we have
> to change the created executor. To avoid it, we can create a builder for each
> step separately after the previous steps are executed.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
