This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 858258c Support SCIO SDK via sbt projects
new 510417a Merge pull request #16563 from [BEAM-13701][Playground]
Support SCIO SDK via sbt projects
858258c is described below
commit 858258cafe0c900f0dd472b1aaa6afb0fd0446d0
Author: daria-malkova <[email protected]>
AuthorDate: Fri Jan 14 18:06:08 2022 +0300
Support SCIO SDK via sbt projects
---
playground/backend/cmd/server/controller.go | 2 +-
playground/backend/configs/SDK_SCIO.json | 10 +++
playground/backend/containers/scio/Dockerfile | 69 ++++++++++++++++++++
playground/backend/containers/scio/app.yaml | 22 +++++++
playground/backend/containers/scio/build.gradle | 74 ++++++++++++++++++++++
playground/backend/containers/scio/entrypoint.sh | 25 ++++++++
playground/backend/containers/scio/settings.gradle | 19 ++++++
.../cloud_bucket/precompiled_objects_test.go | 22 ++++---
.../internal/code_processing/code_processing.go | 2 +-
.../internal/environment/environment_service.go | 2 +-
playground/backend/internal/executors/executor.go | 2 +-
playground/backend/internal/fs_tool/fs.go | 3 +
playground/backend/internal/fs_tool/python_fs.go | 2 +-
.../internal/fs_tool/{python_fs.go => scio_fs.go} | 10 +--
.../backend/internal/preparers/java_preparers.go | 12 ++--
.../backend/internal/preparers/scio_preparers.go | 69 ++++++++++++++++++++
.../internal/setup_tools/builder/setup_builder.go | 17 ++++-
.../setup_tools/life_cycle/life_cycle_setuper.go | 50 +++++++++++++++
.../life_cycle/life_cycle_setuper_test.go | 1 -
.../backend/internal/utils/preparators_utils.go | 2 +
.../backend/internal/utils/validators_utils.go | 2 +
.../backend/internal/validators/scio_validators.go | 50 +++++++++++++++
.../backend/internal/validators/validator.go | 5 +-
playground/backend/new_scio_project.sh | 18 ++++++
settings.gradle.kts | 1 +
25 files changed, 461 insertions(+), 30 deletions(-)
diff --git a/playground/backend/cmd/server/controller.go
b/playground/backend/cmd/server/controller.go
index 1d9a334..d9dfac4 100644
--- a/playground/backend/cmd/server/controller.go
+++ b/playground/backend/cmd/server/controller.go
@@ -50,7 +50,7 @@ func (controller *playgroundController) RunCode(ctx
context.Context, info *pb.Ru
return nil, errors.InvalidArgumentError("Error during
preparing", "Incorrect sdk. Want to receive %s, but the request contains %s",
controller.env.BeamSdkEnvs.ApacheBeamSdk.String(), info.Sdk.String())
}
switch info.Sdk {
- case pb.Sdk_SDK_UNSPECIFIED, pb.Sdk_SDK_SCIO:
+ case pb.Sdk_SDK_UNSPECIFIED:
logger.Errorf("RunCode(): unimplemented sdk: %s\n", info.Sdk)
return nil, errors.InvalidArgumentError("Error during
preparing", "Sdk is not implemented yet: %s", info.Sdk.String())
}
diff --git a/playground/backend/configs/SDK_SCIO.json
b/playground/backend/configs/SDK_SCIO.json
new file mode 100644
index 0000000..0da4fb2
--- /dev/null
+++ b/playground/backend/configs/SDK_SCIO.json
@@ -0,0 +1,10 @@
+{
+ "compile_cmd": "",
+ "run_cmd": "sbt",
+ "test_cmd": "sbt",
+ "compile_args": [],
+ "run_args": [
+ "runMain"
+ ],
+ "test_args": []
+}
diff --git a/playground/backend/containers/scio/Dockerfile
b/playground/backend/containers/scio/Dockerfile
new file mode 100644
index 0000000..131cc3f
--- /dev/null
+++ b/playground/backend/containers/scio/Dockerfile
@@ -0,0 +1,69 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+ARG BASE_IMAGE=openjdk:8
+FROM golang:1.17-bullseye AS build
+
+# Setup Go Environment
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+RUN go install google.golang.org/protobuf/cmd/[email protected] &&\
+ go install google.golang.org/grpc/cmd/[email protected]
+
+# Prepare Application
+COPY src /go/src/playground/backend
+WORKDIR /go/src/playground/backend
+
+# Build Application
+RUN go mod download &&\
+ go mod tidy &&\
+ cd cmd/server &&\
+ go build -o /go/bin/server_scio_backend
+
+FROM $BASE_IMAGE
+ENV SERVER_IP=0.0.0.0
+ENV SERVER_PORT=8080
+ENV APP_WORK_DIR=/opt/playground/backend/
+ENV BEAM_SDK="SDK_SCIO"
+
+# Copy build result
+COPY --from=build /go/bin/server_scio_backend /opt/playground/backend/
+COPY --from=build /go/src/playground/backend/configs
/opt/playground/backend/configs/
+COPY --from=build /go/src/playground/backend/logging.properties
/opt/playground/backend/
+COPY --from=build /go/src/playground/backend/new_scio_project.sh
/opt/playground/backend/
+
+# Install sbt
+RUN echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | tee
/etc/apt/sources.list.d/sbt.list &&\
+echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee
/etc/apt/sources.list.d/sbt_old.list &&\
+curl -sL
"https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823";
| apt-key add
+RUN apt-get update && apt-get install -y sbt
+
+
+## Install mitmpoxy
+RUN mkdir /opt/mitmproxy &&\
+ cd /opt/mitmproxy &&\
+ wget https://snapshots.mitmproxy.org/7.0.4/mitmproxy-7.0.4-linux.tar.gz &&\
+ tar -zxvf mitmproxy-7.0.4-linux.tar.gz &&\
+ mkdir /usr/local/share/ca-certificates/extra
+COPY entrypoint.sh /
+COPY allow_list_proxy.py /opt/mitmproxy/
+COPY allow_list.py /opt/mitmproxy/
+ENV HTTP_PROXY="http://127.0.0.1:8081";
+ENV HTTPS_PROXY="http://127.0.0.1:8081";
+
+ENTRYPOINT ["/opt/playground/backend/server_scio_backend"]
diff --git a/playground/backend/containers/scio/app.yaml
b/playground/backend/containers/scio/app.yaml
new file mode 100644
index 0000000..76e513c
--- /dev/null
+++ b/playground/backend/containers/scio/app.yaml
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+service: default
+runtime: custom
+env: flex
+manual_scaling:
+ instances: 1
diff --git a/playground/backend/containers/scio/build.gradle
b/playground/backend/containers/scio/build.gradle
new file mode 100644
index 0000000..66e9f8c
--- /dev/null
+++ b/playground/backend/containers/scio/build.gradle
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+apply plugin: 'org.apache.beam.module'
+apply plugin: 'base'
+applyDockerNature()
+
+def playgroundJobServerProject = "${project.path.replace('-container', '')}"
+
+description = project(playgroundJobServerProject).description + " :: Container"
+
+configurations {
+ dockerDependency
+}
+
+dependencies {
+ dockerDependency project(path: playgroundJobServerProject, configuration:
"shadow")
+}
+
+task copyDockerfileDependencies(type: Copy) {
+ copy {
+ from '../../../backend/'
+ into 'build/src'
+ exclude 'containers'
+ }
+ copy {
+ from 'entrypoint.sh'
+ into 'build/'
+ }
+ copy {
+ from '../../../infrastructure/proxy/allow_list.py'
+ into 'build/'
+ }
+ copy {
+ from '../../../infrastructure/proxy/allow_list_proxy.py'
+ into 'build/'
+ }
+ copy {
+ from '../../../playground'
+ into 'build/playground'
+ }
+}
+
+docker {
+ name containerImageName(
+ name: project.docker_image_default_repo_prefix +
"playground-backend",
+ root: project.rootProject.hasProperty(["docker-repository-root"]) ?
+ project.rootProject["docker-repository-root"] :
+ project.docker_image_default_repo_root)
+ files "./build/"
+ tags containerImageTags()
+ buildArgs(['BASE_IMAGE': project.rootProject.hasProperty(["base-image"]) ?
+ project.rootProject["base-image"] :
+ "openjdk:8" ])
+}
+
+// Ensure that we build the required resources and copy and file dependencies
from related projects
+dockerPrepare.dependsOn copyDockerfileDependencies
diff --git a/playground/backend/containers/scio/entrypoint.sh
b/playground/backend/containers/scio/entrypoint.sh
new file mode 100755
index 0000000..59df631
--- /dev/null
+++ b/playground/backend/containers/scio/entrypoint.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nohup /opt/mitmproxy/mitmdump -s /opt/mitmproxy/allow_list_proxy.py -p 8081 &
+while [ ! -f /root/.mitmproxy/mitmproxy-ca.pem ] ;
+do
+ sleep 2
+done
+openssl x509 -in /root/.mitmproxy/mitmproxy-ca.pem -inform PEM -out
/root/.mitmproxy/mitmproxy-ca.crt
+cp /root/.mitmproxy/mitmproxy-ca.crt /usr/local/share/ca-certificates/extra/
+update-ca-certificates
+/opt/playground/backend/server_scio_backend
diff --git a/playground/backend/containers/scio/settings.gradle
b/playground/backend/containers/scio/settings.gradle
new file mode 100644
index 0000000..1540be7
--- /dev/null
+++ b/playground/backend/containers/scio/settings.gradle
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+rootProject.name = 'apache-beam-playground-backend'
diff --git
a/playground/backend/internal/cloud_bucket/precompiled_objects_test.go
b/playground/backend/internal/cloud_bucket/precompiled_objects_test.go
index a7114cd..9656bb8 100644
--- a/playground/backend/internal/cloud_bucket/precompiled_objects_test.go
+++ b/playground/backend/internal/cloud_bucket/precompiled_objects_test.go
@@ -165,22 +165,24 @@ func Test_appendPrecompiledObject(t *testing.T) {
name: "Test append new objects",
args: args{
objectInfo: ObjectInfo{
- Name: "",
- CloudPath: "",
- Description: "",
- Type: 0,
- Categories: []string{"Common"},
+ Name: "",
+ CloudPath: "",
+ Description: "",
+ Type: 0,
+ Categories: []string{"Common"},
+ PipelineOptions: "",
},
sdkToCategories: &SdkToCategories{},
pathToObject: "SDK_JAVA/HelloWorld",
categoryName: "Common",
},
want: &SdkToCategories{"SDK_JAVA":
CategoryToPrecompiledObjects{"Common": PrecompiledObjects{ObjectInfo{
- Name: "HelloWorld",
- CloudPath: "SDK_JAVA/HelloWorld",
- Description: "",
- Type: 0,
- Categories: []string{"Common"},
+ Name: "HelloWorld",
+ CloudPath: "SDK_JAVA/HelloWorld",
+ Description: "",
+ Type: 0,
+ Categories: []string{"Common"},
+ PipelineOptions: "",
}}}},
},
}
diff --git a/playground/backend/internal/code_processing/code_processing.go
b/playground/backend/internal/code_processing/code_processing.go
index f02171a..219c4ef 100644
--- a/playground/backend/internal/code_processing/code_processing.go
+++ b/playground/backend/internal/code_processing/code_processing.go
@@ -163,7 +163,7 @@ func compileStep(ctx context.Context, cacheService
cache.Cache, paths *fs_tool.L
errorChannel, successChannel := createStatusChannels()
var executor = executors.Executor{}
// This condition is used for cases when the playground doesn't compile
source files. For the Python code and the Go Unit Tests
- if sdkEnv.ApacheBeamSdk == pb.Sdk_SDK_PYTHON || (sdkEnv.ApacheBeamSdk
== pb.Sdk_SDK_GO && isUnitTest) {
+ if sdkEnv.ApacheBeamSdk == pb.Sdk_SDK_PYTHON || sdkEnv.ApacheBeamSdk ==
pb.Sdk_SDK_SCIO || (sdkEnv.ApacheBeamSdk == pb.Sdk_SDK_GO && isUnitTest) {
if err := processCompileSuccess(pipelineLifeCycleCtx,
[]byte(""), pipelineId, cacheService); err != nil {
return nil
}
diff --git a/playground/backend/internal/environment/environment_service.go
b/playground/backend/internal/environment/environment_service.go
index 81f733e..c940aef 100644
--- a/playground/backend/internal/environment/environment_service.go
+++ b/playground/backend/internal/environment/environment_service.go
@@ -212,7 +212,7 @@ func createExecutorConfig(apacheBeamSdk pb.Sdk, configPath
string) (*ExecutorCon
case pb.Sdk_SDK_PYTHON:
// Python sdk doesn't need any additional arguments from the
config file
case pb.Sdk_SDK_SCIO:
- return nil, errors.New("not yet supported")
+ // Scala sdk doesn't need any additional arguments from the
config file
}
return executorConfig, nil
}
diff --git a/playground/backend/internal/executors/executor.go
b/playground/backend/internal/executors/executor.go
index 15c9594..75e0fdc 100644
--- a/playground/backend/internal/executors/executor.go
+++ b/playground/backend/internal/executors/executor.go
@@ -107,7 +107,7 @@ func (ex *Executor) Run(ctx context.Context) *exec.Cmd {
if ex.runArgs.fileName != "" {
args = append(args, ex.runArgs.fileName)
}
- if ex.runArgs.pipelineOptions[0] != "" {
+ if ex.runArgs.pipelineOptions != nil && ex.runArgs.pipelineOptions[0]
!= "" {
args = append(args, ex.runArgs.pipelineOptions...)
}
cmd := exec.CommandContext(ctx, ex.runArgs.commandName, args...)
diff --git a/playground/backend/internal/fs_tool/fs.go
b/playground/backend/internal/fs_tool/fs.go
index 6a897cb..0cea7c5 100644
--- a/playground/backend/internal/fs_tool/fs.go
+++ b/playground/backend/internal/fs_tool/fs.go
@@ -40,6 +40,7 @@ type LifeCyclePaths struct {
AbsoluteExecutableFilePath string //
/path/to/workingDir/pipelinesFolder/{pipelineId}/bin/{pipelineId}.{executableFileExtension}
AbsoluteBaseFolderPath string //
/path/to/workingDir/pipelinesFolder/{pipelineId}
AbsoluteLogFilePath string //
/path/to/workingDir/pipelinesFolder/{pipelineId}/logs.log
+ ProjectDir string // /path/to/workingDir/
ExecutableName func(string) (string, error)
}
@@ -58,6 +59,8 @@ func NewLifeCycle(sdk pb.Sdk, pipelineId uuid.UUID,
pipelinesFolder string) (*Li
return newGoLifeCycle(pipelineId, pipelinesFolder), nil
case pb.Sdk_SDK_PYTHON:
return newPythonLifeCycle(pipelineId, pipelinesFolder), nil
+ case pb.Sdk_SDK_SCIO:
+ return newScioLifeCycle(pipelineId, pipelinesFolder), nil
default:
return nil, fmt.Errorf("%s isn't supported now", sdk)
}
diff --git a/playground/backend/internal/fs_tool/python_fs.go
b/playground/backend/internal/fs_tool/python_fs.go
index ba01acc..54c70ad 100644
--- a/playground/backend/internal/fs_tool/python_fs.go
+++ b/playground/backend/internal/fs_tool/python_fs.go
@@ -23,7 +23,7 @@ const (
pythonExecutableFileExtension = ".py"
)
-// newPythonLifeCycle creates LifeCycle with go SDK environment.
+// newPythonLifeCycle creates LifeCycle with python SDK environment.
func newPythonLifeCycle(pipelineId uuid.UUID, pipelinesFolder string)
*LifeCycle {
return newInterpretedLifeCycle(pipelineId, pipelinesFolder,
pythonExecutableFileExtension)
}
diff --git a/playground/backend/internal/fs_tool/python_fs.go
b/playground/backend/internal/fs_tool/scio_fs.go
similarity index 72%
copy from playground/backend/internal/fs_tool/python_fs.go
copy to playground/backend/internal/fs_tool/scio_fs.go
index ba01acc..ea8d834 100644
--- a/playground/backend/internal/fs_tool/python_fs.go
+++ b/playground/backend/internal/fs_tool/scio_fs.go
@@ -20,10 +20,12 @@ import (
)
const (
- pythonExecutableFileExtension = ".py"
+ scioExecutableFileExtension = ".scala"
)
-// newPythonLifeCycle creates LifeCycle with go SDK environment.
-func newPythonLifeCycle(pipelineId uuid.UUID, pipelinesFolder string)
*LifeCycle {
- return newInterpretedLifeCycle(pipelineId, pipelinesFolder,
pythonExecutableFileExtension)
+// newScioLifeCycle creates LifeCycle with scala SDK environment.
+func newScioLifeCycle(pipelineId uuid.UUID, pipelinesFolder string) *LifeCycle
{
+ lc := newInterpretedLifeCycle(pipelineId, pipelinesFolder,
scioExecutableFileExtension)
+ lc.Paths.ExecutableName = executableName
+ return lc
}
diff --git a/playground/backend/internal/preparers/java_preparers.go
b/playground/backend/internal/preparers/java_preparers.go
index bf604c2..14e5e4b 100644
--- a/playground/backend/internal/preparers/java_preparers.go
+++ b/playground/backend/internal/preparers/java_preparers.go
@@ -35,7 +35,7 @@ const (
newLinePattern = "\n"
pathSeparatorPattern = os.PathSeparator
tmpFileSuffix = "tmp"
- publicClassNamePattern = "public class (.*?)
[{|implements(.*)]"
+ javaPublicClassNamePattern = "public class (.*?)
[{|implements(.*)]"
)
//JavaPreparersBuilder facet of PreparersBuilder
@@ -204,31 +204,31 @@ func addNewLine(newLine bool, file *os.File) error {
func changeJavaTestFileName(args ...interface{}) error {
filePath := args[0].(string)
- className, err := getPublicClassName(filePath)
+ className, err := getPublicClassName(filePath,
javaPublicClassNamePattern)
if err != nil {
return err
}
- err = renameJavaFile(filePath, className)
+ err = renameSourceCodeFile(filePath, className)
if err != nil {
return err
}
return nil
}
-func renameJavaFile(filePath string, className string) error {
+func renameSourceCodeFile(filePath string, className string) error {
currentFileName := filepath.Base(filePath)
newFilePath := strings.Replace(filePath, currentFileName,
fmt.Sprintf("%s%s", className, filepath.Ext(currentFileName)), 1)
err := os.Rename(filePath, newFilePath)
return err
}
-func getPublicClassName(filePath string) (string, error) {
+func getPublicClassName(filePath, pattern string) (string, error) {
code, err := ioutil.ReadFile(filePath)
if err != nil {
logger.Errorf("Preparer: Error during open file: %s, err:
%s\n", filePath, err.Error())
return "", err
}
- re := regexp.MustCompile(publicClassNamePattern)
+ re := regexp.MustCompile(pattern)
className := re.FindStringSubmatch(string(code))[1]
return className, err
}
diff --git a/playground/backend/internal/preparers/scio_preparers.go
b/playground/backend/internal/preparers/scio_preparers.go
new file mode 100644
index 0000000..70cb0b1
--- /dev/null
+++ b/playground/backend/internal/preparers/scio_preparers.go
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package preparers
+
+const (
+ scioPublicClassNamePattern = "object (.*?) [{]"
+ scioPackagePattern = `^package [\w]+`
+)
+
+//ScioPreparersBuilder facet of PreparersBuilder
+type ScioPreparersBuilder struct {
+ PreparersBuilder
+}
+
+//ScioPreparers chains to type *PreparersBuilder and returns a
*ScioPreparersBuilder
+func (builder *PreparersBuilder) ScioPreparers() *ScioPreparersBuilder {
+ return &ScioPreparersBuilder{*builder}
+}
+
+//WithFileNameChanger adds preparer to change source code file name
+func (builder *ScioPreparersBuilder) WithFileNameChanger()
*ScioPreparersBuilder {
+ changeNamePreparer := Preparer{
+ Prepare: changeScioFileName,
+ Args: []interface{}{builder.filePath},
+ }
+ builder.AddPreparer(changeNamePreparer)
+ return builder
+}
+
+//WithPackageRemover adds preparer to remove package from the code
+func (builder *ScioPreparersBuilder) WithPackageRemover()
*ScioPreparersBuilder {
+ removePackagePreparer := Preparer{
+ Prepare: replace,
+ Args: []interface{}{builder.filePath, scioPackagePattern,
""},
+ }
+ builder.AddPreparer(removePackagePreparer)
+ return builder
+}
+
+// GetScioPreparers returns preparation methods that should be applied to Scio
code
+func GetScioPreparers(builder *PreparersBuilder) {
+ builder.ScioPreparers().WithPackageRemover().WithFileNameChanger()
+}
+
+func changeScioFileName(args ...interface{}) error {
+ filePath := args[0].(string)
+ className, err := getPublicClassName(filePath,
scioPublicClassNamePattern)
+ if err != nil {
+ return err
+ }
+ err = renameSourceCodeFile(filePath, className)
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/playground/backend/internal/setup_tools/builder/setup_builder.go
b/playground/backend/internal/setup_tools/builder/setup_builder.go
index 5209ff0..e439370 100644
--- a/playground/backend/internal/setup_tools/builder/setup_builder.go
+++ b/playground/backend/internal/setup_tools/builder/setup_builder.go
@@ -85,7 +85,7 @@ func Compiler(paths *fs_tool.LifeCyclePaths, sdkEnv
*environment.BeamEnvs) *exec
func Runner(paths *fs_tool.LifeCyclePaths, pipelineOptions string, sdkEnv
*environment.BeamEnvs) (*executors.ExecutorBuilder, error) {
sdk := sdkEnv.ApacheBeamSdk
- if sdk == pb.Sdk_SDK_JAVA {
+ if sdk == pb.Sdk_SDK_JAVA || sdk == pb.Sdk_SDK_SCIO {
pipelineOptions = utils.ReplaceSpacesWithEquals(pipelineOptions)
}
executorConfig := sdkEnv.ExecutorConfig
@@ -94,7 +94,6 @@ func Runner(paths *fs_tool.LifeCyclePaths, pipelineOptions
string, sdkEnv *envir
WithWorkingDir(paths.AbsoluteBaseFolderPath).
WithCommand(executorConfig.RunCmd).
WithArgs(executorConfig.RunArgs).
- WithPipelineOptions(strings.Split(pipelineOptions, " ")).
ExecutorBuilder
switch sdk {
@@ -108,17 +107,31 @@ func Runner(paths *fs_tool.LifeCyclePaths,
pipelineOptions string, sdkEnv *envir
WithRunner().
WithArgs(args).
WithExecutableFileName(className).
+ WithPipelineOptions(strings.Split(pipelineOptions, "
")).
ExecutorBuilder
case pb.Sdk_SDK_GO: //go run command is executable file itself
builder = builder.
WithRunner().
WithExecutableFileName("").
WithCommand(paths.AbsoluteExecutableFilePath).
+ WithPipelineOptions(strings.Split(pipelineOptions, "
")).
ExecutorBuilder
case pb.Sdk_SDK_PYTHON:
builder = builder.
WithRunner().
WithExecutableFileName(paths.AbsoluteExecutableFilePath).
+ WithPipelineOptions(strings.Split(pipelineOptions, "
")).
+ ExecutorBuilder
+ case pb.Sdk_SDK_SCIO:
+ className, err :=
paths.ExecutableName(paths.AbsoluteBaseFolderPath)
+ if err != nil {
+ return nil, fmt.Errorf("no executable file name found
for SCIO pipeline at %s", paths.AbsoluteBaseFolderPath)
+ }
+ stringArg := fmt.Sprintf("%s %s %s", executorConfig.RunArgs[0],
className, pipelineOptions)
+ builder = builder.
+ WithRunner().
+ WithWorkingDir(paths.ProjectDir).
+ WithArgs([]string{stringArg}).
ExecutorBuilder
}
return &builder, nil
diff --git
a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
index b9fd155..cf7a0d2 100644
--- a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
+++ b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
@@ -24,6 +24,7 @@ import (
"github.com/google/uuid"
"io"
"os"
+ "os/exec"
"path/filepath"
"strings"
)
@@ -34,6 +35,12 @@ const (
javaLogFilePlaceholder = "{logFilePath}"
goModFileName = "go.mod"
goSumFileName = "go.sum"
+ scioProjectName = "scioproject"
+ projectPath = "scioproject/src/main/scala/scioproject"
+ logFileName = "logs.log"
+ defaultExampleInSbt = "WordCount.scala"
+ shCmd = "sh"
+ scioProject = "new_scio_project.sh"
)
// Setup returns fs_tool.LifeCycle.
@@ -65,6 +72,11 @@ func Setup(sdk pb.Sdk, code string, pipelineId uuid.UUID,
workingDir, pipelinesF
lc.DeleteFolders()
return nil, errors.New("error during create necessary
files for the Java sdk")
}
+ case pb.Sdk_SDK_SCIO:
+ if lc, err = prepareSbtFiles(lc,
lc.Paths.AbsoluteBaseFolderPath, workingDir); err != nil {
+ lc.DeleteFolders()
+ return nil, errors.New("error during create necessary
files for the SCIO sdk")
+ }
}
// create file with code
@@ -143,3 +155,41 @@ func updateJavaLogConfigFile(paths fs_tool.LifeCyclePaths)
error {
}
return nil
}
+
+func prepareSbtFiles(lc *fs_tool.LifeCycle, pipelineFolder string, workingDir
string) (*fs_tool.LifeCycle, error) {
+ cmd := exec.Command(shCmd, filepath.Join(workingDir, scioProject))
+ cmd.Dir = pipelineFolder
+ _, err := cmd.Output()
+ if err != nil {
+ return nil, err
+ }
+
+ sourceFileFolder := filepath.Join(pipelineFolder, projectPath)
+ fileName := lc.Paths.SourceFileName
+ absFileFolderPath, _ := filepath.Abs(sourceFileFolder)
+ absFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
fileName))
+ absLogFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
logFileName))
+ projectFolder, _ := filepath.Abs(filepath.Join(pipelineFolder,
scioProjectName))
+ executableName := lc.Paths.ExecutableName
+
+ _, err = exec.Command("rm", filepath.Join(absFileFolderPath,
defaultExampleInSbt)).Output()
+ if err != nil {
+ return nil, err
+ }
+
+ lc = &fs_tool.LifeCycle{
+ Paths: fs_tool.LifeCyclePaths{
+ SourceFileName: fileName,
+ AbsoluteSourceFileFolderPath: absFileFolderPath,
+ AbsoluteSourceFilePath: absFilePath,
+ ExecutableFileName: fileName,
+ AbsoluteExecutableFileFolderPath: absFileFolderPath,
+ AbsoluteExecutableFilePath: absFilePath,
+ AbsoluteBaseFolderPath: absFileFolderPath,
+ AbsoluteLogFilePath: absLogFilePath,
+ ProjectDir: projectFolder,
+ },
+ }
+ lc.Paths.ExecutableName = executableName
+ return lc, nil
+}
diff --git
a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper_test.go
b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper_test.go
index a698991..2138168 100644
---
a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper_test.go
+++
b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper_test.go
@@ -33,7 +33,6 @@ const (
javaSourceFileExtension = ".java"
javaCompiledFileExtension = ".class"
pipelinesFolder = "executable_files"
- logFileName = "logs.log"
)
func TestSetup(t *testing.T) {
diff --git a/playground/backend/internal/utils/preparators_utils.go
b/playground/backend/internal/utils/preparators_utils.go
index 876225e..eb2e61e 100644
--- a/playground/backend/internal/utils/preparators_utils.go
+++ b/playground/backend/internal/utils/preparators_utils.go
@@ -42,6 +42,8 @@ func GetPreparers(sdk pb.Sdk, filepath string, valResults
*sync.Map) (*[]prepare
preparers.GetGoPreparers(builder, isUnitTest.(bool))
case pb.Sdk_SDK_PYTHON:
preparers.GetPythonPreparers(builder)
+ case pb.Sdk_SDK_SCIO:
+ preparers.GetScioPreparers(builder)
default:
return nil, fmt.Errorf("incorrect sdk: %s", sdk)
}
diff --git a/playground/backend/internal/utils/validators_utils.go
b/playground/backend/internal/utils/validators_utils.go
index 05e402c..5d9406e 100644
--- a/playground/backend/internal/utils/validators_utils.go
+++ b/playground/backend/internal/utils/validators_utils.go
@@ -31,6 +31,8 @@ func GetValidators(sdk pb.Sdk, filepath string)
(*[]validators.Validator, error)
val = validators.GetGoValidators(filepath)
case pb.Sdk_SDK_PYTHON:
val = validators.GetPyValidators(filepath)
+ case pb.Sdk_SDK_SCIO:
+ val = validators.GetScioValidators(filepath)
default:
return nil, fmt.Errorf("incorrect sdk: %s", sdk)
}
diff --git a/playground/backend/internal/validators/scio_validators.go
b/playground/backend/internal/validators/scio_validators.go
new file mode 100644
index 0000000..d43dfe1
--- /dev/null
+++ b/playground/backend/internal/validators/scio_validators.go
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validators
+
+import (
+ "beam.apache.org/playground/backend/internal/fs_tool"
+)
+
+const (
+ scalaExtension = ".scala"
+)
+
+// GetScioValidators return validators methods that should be applied to scio
code
+// The last validator should check that the code is unit tests or not
+func GetScioValidators(filePath string) *[]Validator {
+ validatorArgs := make([]interface{}, 2)
+ validatorArgs[0] = filePath
+ validatorArgs[1] = scalaExtension
+ pathCheckerValidator := Validator{
+ Validator: fs_tool.CheckPathIsValid,
+ Args: validatorArgs,
+ Name: "Valid path",
+ }
+ unitTestValidator := Validator{
+ Validator: checkIsUnitTestScio,
+ Args: validatorArgs,
+ Name: UnitTestValidatorName,
+ }
+ validators := []Validator{pathCheckerValidator, unitTestValidator}
+ return &validators
+}
+
+//checkIsUnitTestScio checks if the pipeline is a UnitTest
+func checkIsUnitTestScio(args ...interface{}) (bool, error) {
+ return false, nil
+ //TODO BEAM-13702
+}
diff --git a/playground/backend/internal/validators/validator.go
b/playground/backend/internal/validators/validator.go
index 63c9e1c..cf504f7 100644
--- a/playground/backend/internal/validators/validator.go
+++ b/playground/backend/internal/validators/validator.go
@@ -16,8 +16,9 @@
package validators
const (
- UnitTestValidatorName = "UnitTest"
- KatasValidatorName = "Katas"
+ UnitTestValidatorName = "UnitTest"
+ KatasValidatorName = "Katas"
+ PublicClassValidatorName = "ClassName"
)
type Validator struct {
diff --git a/playground/backend/new_scio_project.sh
b/playground/backend/new_scio_project.sh
new file mode 100644
index 0000000..d274c78
--- /dev/null
+++ b/playground/backend/new_scio_project.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+yes scioproject | sbt new spotify/scio-template.g8
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 50f37c2..9b71524 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -63,6 +63,7 @@ include(":playground:frontend")
include(":playground:backend:containers:java")
include(":playground:backend:containers:go")
include(":playground:backend:containers:python")
+include(":playground:backend:containers:scio")
include(":runners:core-construction-java")
include(":runners:core-java")
include(":runners:direct-java")
