pabloem commented on a change in pull request #15803:
URL: https://github.com/apache/beam/pull/15803#discussion_r736994177
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -54,3 +121,125 @@ func (controller *playgroundController)
GetCompileOutput(ctx context.Context, in
compileOutput := pb.GetCompileOutputResponse{Output: "test compile
output"}
return &compileOutput, nil
}
+
+// setupValidators returns validators based on sdk
+func setupValidators(sdk pb.Sdk, filepath string) *[]validators.Validator {
+ var val *[]validators.Validator
+ switch sdk {
+ case pb.Sdk_SDK_JAVA:
+ val = validators.GetJavaValidators(filepath)
+ }
+ return val
+}
+
+// processCode validates, compiles and runs code by pipelineId.
+// During each operation updates status of execution and saves it into cache.
+// In case of some step is failed saves output logs to cache.
+// After success code running saves output to cache.
+// At the end of this method deletes all created folders
+func processCode(ctx context.Context, cacheService cache.Cache, lc
*fs_tool.LifeCycle, execBuilder *executors.CompileBuilder, pipelineId
uuid.UUID, env *environment.Environment) {
+ defer cleanUp(pipelineId, lc)
+
+ exec := execBuilder.Build()
+
+ // validate
+ log.Printf("%s: Validate() ...\n", pipelineId)
+
+ validateFunc := exec.Validate()
+ if err := validateFunc(); err != nil {
+ // error during validation
+ // TODO move to processError when status for validation error
will be added
+ log.Printf("%s: Validate: %s\n", pipelineId, err.Error())
+ setToCache(ctx, cacheService, pipelineId, cache.Status,
pb.Status_STATUS_ERROR)
+ return
+ }
+ log.Printf("%s: Validate() finish\n", pipelineId)
+
+ // compile
+ log.Printf("%s: Compile() ...\n", pipelineId)
+ compileCmd := exec.Compile()
+ if data, err := compileCmd.CombinedOutput(); err != nil {
+ processError(ctx, err, data, pipelineId, cacheService,
pb.Status_STATUS_COMPILE_ERROR)
+ return
+ }
+ log.Printf("%s: Compile() finish\n", pipelineId)
+
+ // set empty value to pipelineId: cache.SubKey_CompileOutput
+ setToCache(ctx, cacheService, pipelineId, cache.CompileOutput, "")
+
+ className, err := lc.ExecutableName(pipelineId,
env.ApplicationEnvs.WorkingDir())
+ if err != nil {
+ log.Printf("%s: get executable file name: %s\n", pipelineId,
err.Error())
+ setToCache(ctx, cacheService, pipelineId, cache.Status,
pb.Status_STATUS_ERROR)
+ return
+ }
+
+ exec = execBuilder.
+ WithRunner().
+ WithCommand(env.BeamSdkEnvs.ExecutorConfig.RunCmd).
+ WithArgs(env.BeamSdkEnvs.ExecutorConfig.RunArgs).
+ WithClassName(className).
Review comment:
I suppose this is for Java, right? or do we also expect a class name for
go/python?
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -15,21 +15,88 @@
package main
import (
- "context"
-
pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/cache"
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/errors"
+ "beam.apache.org/playground/backend/internal/executors"
+ "beam.apache.org/playground/backend/internal/fs_tool"
+ "beam.apache.org/playground/backend/internal/validators"
+ "context"
"github.com/google/uuid"
+ "log"
)
type playgroundController struct {
+ env *environment.Environment
+ cacheService cache.Cache
+
pb.UnimplementedPlaygroundServiceServer
}
//RunCode is running code from requests using a particular SDK
func (controller *playgroundController) RunCode(ctx context.Context, info
*pb.RunCodeRequest) (*pb.RunCodeResponse, error) {
- // TODO implement this method
- pipelineInfo := pb.RunCodeResponse{PipelineUuid: uuid.NewString()}
+ // check for correct sdk
+ switch info.Sdk {
+ case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_GO, pb.Sdk_SDK_PYTHON,
pb.Sdk_SDK_SCIO:
+ log.Printf("RunCode(): unimplemented sdk: %s\n", info.Sdk)
+ return nil, errors.InvalidArgumentError("Run code()",
"unimplemented sdk: "+info.Sdk.String())
+ }
+
+ pipelineId := uuid.New()
+
+ defer func() {
+ log.Printf("RunCode() is completed for pipeline with id: %s\n",
pipelineId)
+ }()
+
+ cacheExpirationTime :=
controller.env.ApplicationEnvs.CacheEnvs().KeyExpirationTime()
Review comment:
Is it a default expiration time configured for the environment?
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -15,21 +15,88 @@
package main
import (
- "context"
-
pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/cache"
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/errors"
+ "beam.apache.org/playground/backend/internal/executors"
+ "beam.apache.org/playground/backend/internal/fs_tool"
+ "beam.apache.org/playground/backend/internal/validators"
+ "context"
"github.com/google/uuid"
+ "log"
Review comment:
should we be using our logger libnrary?
https://github.com/apache/beam/blob/master/playground/backend/internal/logger/logger.go
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -15,21 +15,88 @@
package main
import (
- "context"
-
pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/cache"
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/errors"
+ "beam.apache.org/playground/backend/internal/executors"
+ "beam.apache.org/playground/backend/internal/fs_tool"
+ "beam.apache.org/playground/backend/internal/validators"
+ "context"
"github.com/google/uuid"
+ "log"
)
type playgroundController struct {
+ env *environment.Environment
Review comment:
please document in detail what the playgroundController does so that
maintainers can easily read the code in the future
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -15,21 +15,88 @@
package main
import (
- "context"
-
pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/cache"
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/errors"
+ "beam.apache.org/playground/backend/internal/executors"
+ "beam.apache.org/playground/backend/internal/fs_tool"
+ "beam.apache.org/playground/backend/internal/validators"
+ "context"
"github.com/google/uuid"
+ "log"
)
type playgroundController struct {
+ env *environment.Environment
+ cacheService cache.Cache
+
pb.UnimplementedPlaygroundServiceServer
}
//RunCode is running code from requests using a particular SDK
func (controller *playgroundController) RunCode(ctx context.Context, info
*pb.RunCodeRequest) (*pb.RunCodeResponse, error) {
- // TODO implement this method
- pipelineInfo := pb.RunCodeResponse{PipelineUuid: uuid.NewString()}
+ // check for correct sdk
+ switch info.Sdk {
+ case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_GO, pb.Sdk_SDK_PYTHON,
pb.Sdk_SDK_SCIO:
+ log.Printf("RunCode(): unimplemented sdk: %s\n", info.Sdk)
+ return nil, errors.InvalidArgumentError("Run code()",
"unimplemented sdk: "+info.Sdk.String())
+ }
+
+ pipelineId := uuid.New()
Review comment:
if we generate the uuid here without inspecting the pipeline, when will
we ever have a cache hit for any given pipeline? what will be the job of the
cache?
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -54,3 +121,125 @@ func (controller *playgroundController)
GetCompileOutput(ctx context.Context, in
compileOutput := pb.GetCompileOutputResponse{Output: "test compile
output"}
return &compileOutput, nil
}
+
+// setupValidators returns validators based on sdk
+func setupValidators(sdk pb.Sdk, filepath string) *[]validators.Validator {
+ var val *[]validators.Validator
+ switch sdk {
+ case pb.Sdk_SDK_JAVA:
+ val = validators.GetJavaValidators(filepath)
+ }
+ return val
+}
+
+// processCode validates, compiles and runs code by pipelineId.
+// During each operation updates status of execution and saves it into cache.
+// In case of some step is failed saves output logs to cache.
+// After success code running saves output to cache.
+// At the end of this method deletes all created folders
+func processCode(ctx context.Context, cacheService cache.Cache, lc
*fs_tool.LifeCycle, execBuilder *executors.CompileBuilder, pipelineId
uuid.UUID, env *environment.Environment) {
+ defer cleanUp(pipelineId, lc)
+
+ exec := execBuilder.Build()
+
+ // validate
+ log.Printf("%s: Validate() ...\n", pipelineId)
+
+ validateFunc := exec.Validate()
+ if err := validateFunc(); err != nil {
+ // error during validation
+ // TODO move to processError when status for validation error
will be added
+ log.Printf("%s: Validate: %s\n", pipelineId, err.Error())
+ setToCache(ctx, cacheService, pipelineId, cache.Status,
pb.Status_STATUS_ERROR)
+ return
+ }
+ log.Printf("%s: Validate() finish\n", pipelineId)
+
+ // compile
+ log.Printf("%s: Compile() ...\n", pipelineId)
+ compileCmd := exec.Compile()
+ if data, err := compileCmd.CombinedOutput(); err != nil {
+ processError(ctx, err, data, pipelineId, cacheService,
pb.Status_STATUS_COMPILE_ERROR)
+ return
+ }
+ log.Printf("%s: Compile() finish\n", pipelineId)
+
+ // set empty value to pipelineId: cache.SubKey_CompileOutput
+ setToCache(ctx, cacheService, pipelineId, cache.CompileOutput, "")
+
+ className, err := lc.ExecutableName(pipelineId,
env.ApplicationEnvs.WorkingDir())
+ if err != nil {
+ log.Printf("%s: get executable file name: %s\n", pipelineId,
err.Error())
+ setToCache(ctx, cacheService, pipelineId, cache.Status,
pb.Status_STATUS_ERROR)
+ return
+ }
+
+ exec = execBuilder.
+ WithRunner().
+ WithCommand(env.BeamSdkEnvs.ExecutorConfig.RunCmd).
+ WithArgs(env.BeamSdkEnvs.ExecutorConfig.RunArgs).
+ WithClassName(className).
+ WithWorkingDir(lc.GetAbsoluteExecutableFilesFolderPath()).
+ Build()
+
+ log.Printf("%s: Run() ...\n", pipelineId)
+ runCmd := exec.Run()
+ data, err := runCmd.CombinedOutput()
+ if err != nil {
+ // error during run code
+ processError(ctx, err, data, pipelineId, cacheService,
pb.Status_STATUS_ERROR)
+ return
+ }
+ log.Printf("%s: Run() finish\n", pipelineId)
+ processSuccess(ctx, data, pipelineId, cacheService)
+}
+
+// cleanUp removes all prepared folders for received LifeCycle
+func cleanUp(pipelineId uuid.UUID, lc *fs_tool.LifeCycle) {
+ log.Printf("%s: DeleteFolders() ...\n", pipelineId)
+ err := lc.DeleteFolders()
+ if err != nil {
+ log.Printf("%s: DeleteFolders(): %s\n", pipelineId, err.Error())
+ }
+ log.Printf("%s: DeleteFolders() complete\n", pipelineId)
+ log.Printf("%s: complete\n", pipelineId)
+}
+
+// processError processes error received during processing code via setting a
corresponding status and output to cache
+func processError(ctx context.Context, err error, data []byte, pipelineId
uuid.UUID, cacheService cache.Cache, status pb.Status) {
+ switch status {
+ case pb.Status_STATUS_ERROR:
+ log.Printf("%s: Run: err: %s, output: %s\n", pipelineId,
err.Error(), data)
+
+ // set to cache pipelineId: cache.SubKey_RunOutput: err.Error()
+ setToCache(ctx, cacheService, pipelineId, cache.RunOutput,
"error: "+err.Error()+", output: "+string(data))
+
+ // set to cache pipelineId: cache.SubKey_Status:
pb.Status_STATUS_ERROR
+ setToCache(ctx, cacheService, pipelineId, cache.Status,
pb.Status_STATUS_ERROR)
+ case pb.Status_STATUS_COMPILE_ERROR:
+ log.Printf("%s: Compile: err: %s, output: %s\n", pipelineId,
err.Error(), data)
+
+ // set to cache pipelineId: cache.SubKey_CompileOutput:
err.Error()
+ setToCache(ctx, cacheService, pipelineId, cache.CompileOutput,
"error: "+err.Error()+", output: "+string(data))
+
+ // set to cache pipelineId: cache.SubKey_Status:
pb.Status_STATUS_ERROR
+ setToCache(ctx, cacheService, pipelineId, cache.Status,
pb.Status_STATUS_COMPILE_ERROR)
+ }
+}
+
+// processSuccess processes case after successful code processing via setting
a corresponding status and output to cache
+func processSuccess(ctx context.Context, output []byte, pipelineId uuid.UUID,
cacheService cache.Cache) {
+ // set to cache pipelineId: cache.SubKey_RunOutput: output
+ setToCache(ctx, cacheService, pipelineId, cache.RunOutput,
string(output))
Review comment:
does it make sense to also add compiler output to cache in case of
success? (maybe not - but if it doesn't, let's document)
##########
File path: playground/backend/cmd/server/controller.go
##########
@@ -15,21 +15,88 @@
package main
import (
- "context"
-
pb "beam.apache.org/playground/backend/internal/api/v1"
+ "beam.apache.org/playground/backend/internal/cache"
+ "beam.apache.org/playground/backend/internal/environment"
+ "beam.apache.org/playground/backend/internal/errors"
+ "beam.apache.org/playground/backend/internal/executors"
+ "beam.apache.org/playground/backend/internal/fs_tool"
+ "beam.apache.org/playground/backend/internal/validators"
+ "context"
"github.com/google/uuid"
+ "log"
)
type playgroundController struct {
+ env *environment.Environment
+ cacheService cache.Cache
+
pb.UnimplementedPlaygroundServiceServer
}
//RunCode is running code from requests using a particular SDK
func (controller *playgroundController) RunCode(ctx context.Context, info
*pb.RunCodeRequest) (*pb.RunCodeResponse, error) {
- // TODO implement this method
- pipelineInfo := pb.RunCodeResponse{PipelineUuid: uuid.NewString()}
+ // check for correct sdk
+ switch info.Sdk {
+ case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_GO, pb.Sdk_SDK_PYTHON,
pb.Sdk_SDK_SCIO:
+ log.Printf("RunCode(): unimplemented sdk: %s\n", info.Sdk)
+ return nil, errors.InvalidArgumentError("Run code()",
"unimplemented sdk: "+info.Sdk.String())
+ }
+
+ pipelineId := uuid.New()
+
+ defer func() {
+ log.Printf("RunCode() is completed for pipeline with id: %s\n",
pipelineId)
+ }()
+
+ cacheExpirationTime :=
controller.env.ApplicationEnvs.CacheEnvs().KeyExpirationTime()
Review comment:
please file a JIRA issue to document the `environment` package. For
example, I am not sure what `KeyExpirationTime` means in this case
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
