This is an automated email from the ASF dual-hosted git repository.
damccorm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 08ae6c7acff [Playground] Implement Java & Go multifile examples
execution (#24874)
08ae6c7acff is described below
commit 08ae6c7acffe1a85571f44baa05db811ddea359b
Author: Timur Sultanov <[email protected]>
AuthorDate: Tue Mar 28 20:42:47 2023 +0400
[Playground] Implement Java & Go multifile examples execution (#24874)
* Find class which contains Java's main() method using javap
* Support compiling multifile Java examples
* Support compiling multifile Go examples
* Improve error handling in java_fs_test.go
* Remove redundant function
* Add error handling to GetFilesFromFolder()
* Improve error handling in java_fs_test.go
* Treat inability to find class with main() method as an error
* Fix failing tests in setup_builder_test.go
* Revert changes to run_output_writer.go
* Look up class with unit test for Java examples using javap
* Remove unused functions
* Pass context into methods for finding executable file names to support
canceling search process due to timeout
* Use more descriptive names for executable file searching functions
* Improve error handling
* Clarify purposes of tests
---
.../internal/code_processing/code_processing.go | 10 +-
.../code_processing/code_processing_test.go | 2 +-
playground/backend/internal/executors/executor.go | 10 +-
.../backend/internal/executors/executor_builder.go | 30 +--
.../internal/executors/executor_builder_test.go | 2 +-
.../backend/internal/executors/executor_test.go | 8 +-
playground/backend/internal/fs_tool/fs.go | 6 +-
playground/backend/internal/fs_tool/fs_test.go | 4 +-
playground/backend/internal/fs_tool/go_fs.go | 4 +-
playground/backend/internal/fs_tool/go_fs_test.go | 4 +-
playground/backend/internal/fs_tool/java_fs.go | 130 +++++++++++--
.../backend/internal/fs_tool/java_fs_test.go | 215 ++++++++++++++++++---
.../fs_tool/java_testdata/HasIncorrectMain.java | 28 +++
.../fs_tool/java_testdata/HasMainTest1.java | 28 +++
.../fs_tool/java_testdata/HasMainTest2.java | 29 +++
.../internal/fs_tool/java_testdata/HasNoMain.java | 28 +++
playground/backend/internal/fs_tool/scio_fs.go | 2 +-
.../internal/setup_tools/builder/setup_builder.go | 51 +++--
.../setup_tools/builder/setup_builder_test.go | 149 +++++++++++---
.../setup_tools/life_cycle/life_cycle_setuper.go | 4 +-
playground/backend/internal/utils/file_utils.go | 4 +
21 files changed, 609 insertions(+), 139 deletions(-)
diff --git a/playground/backend/internal/code_processing/code_processing.go
b/playground/backend/internal/code_processing/code_processing.go
index 23e34240eaa..32cd9c38027 100644
--- a/playground/backend/internal/code_processing/code_processing.go
+++ b/playground/backend/internal/code_processing/code_processing.go
@@ -99,9 +99,9 @@ func runStep(ctx context.Context, cacheService cache.Cache,
paths *fs_tool.LifeC
var executorBuilder *executors.ExecutorBuilder
err := error(nil)
if isUnitTest {
- executorBuilder, err = builder.TestRunner(paths, sdkEnv)
+ executorBuilder, err = builder.TestRunner(pipelineLifeCycleCtx,
paths, sdkEnv)
} else {
- executorBuilder, err = builder.Runner(paths,
utils.ReduceWhiteSpacesToSinge(pipelineOptions), sdkEnv)
+ executorBuilder, err = builder.Runner(pipelineLifeCycleCtx,
paths, utils.ReduceWhiteSpacesToSinge(pipelineOptions), sdkEnv)
}
if err != nil {
_ = processSetupError(err, pipelineId, cacheService,
pipelineLifeCycleCtx)
@@ -171,7 +171,11 @@ func compileStep(ctx context.Context, cacheService
cache.Cache, paths *fs_tool.L
return nil
}
} else { // in case of Java, Go (not unit test), Scala - need compile
step
- executorBuilder := builder.Compiler(paths, sdkEnv)
+ executorBuilder, err := builder.Compiler(paths, sdkEnv)
+ if err != nil {
+ logger.Errorf("compileStep(): failed creating
ExecutorBuilder = %v", executorBuilder)
+ return nil
+ }
executor := executorBuilder.Build()
logger.Infof("%s: Compile() ...\n", pipelineId)
compileCmd := executor.Compile(pipelineLifeCycleCtx)
diff --git
a/playground/backend/internal/code_processing/code_processing_test.go
b/playground/backend/internal/code_processing/code_processing_test.go
index f44cdb68d90..b9f4f7131c4 100644
--- a/playground/backend/internal/code_processing/code_processing_test.go
+++ b/playground/backend/internal/code_processing/code_processing_test.go
@@ -593,7 +593,7 @@ func Test_getRunOrTestCmd(t *testing.T) {
Build()
wantRunExec := exec.CommandContext(context.Background(), "runCommand",
"arg1")
- wantTestExec := exec.CommandContext(context.Background(),
"testCommand", "arg1", "")
+ wantTestExec := exec.CommandContext(context.Background(),
"testCommand", "arg1")
type args struct {
isUnitTest bool
diff --git a/playground/backend/internal/executors/executor.go
b/playground/backend/internal/executors/executor.go
index 4bba3b0d5eb..a2a6afec8e5 100644
--- a/playground/backend/internal/executors/executor.go
+++ b/playground/backend/internal/executors/executor.go
@@ -32,7 +32,7 @@ const (
// CmdConfiguration for base cmd code execution
type CmdConfiguration struct {
- fileName string
+ fileNames []string
workingDir string
commandName string
commandArgs []string
@@ -94,7 +94,7 @@ func (ex *Executor) Prepare() func(chan bool, chan error,
*sync.Map) {
// Compile prepares the Cmd for code compilation
// Returns Cmd instance
func (ex *Executor) Compile(ctx context.Context) *exec.Cmd {
- args := append(ex.compileArgs.commandArgs, ex.compileArgs.fileName)
+ args := append(ex.compileArgs.commandArgs, ex.compileArgs.fileNames...)
cmd := exec.CommandContext(ctx, ex.compileArgs.commandName, args...)
cmd.Dir = ex.compileArgs.workingDir
return cmd
@@ -104,8 +104,8 @@ func (ex *Executor) Compile(ctx context.Context) *exec.Cmd {
// Returns Cmd instance
func (ex *Executor) Run(ctx context.Context) *exec.Cmd {
args := ex.runArgs.commandArgs
- if ex.runArgs.fileName != "" {
- args = append(args, ex.runArgs.fileName)
+ if len(ex.runArgs.fileNames) > 0 {
+ args = append(args, ex.runArgs.fileNames...)
}
if ex.runArgs.pipelineOptions != nil && ex.runArgs.pipelineOptions[0]
!= "" {
args = append(args, ex.runArgs.pipelineOptions...)
@@ -118,7 +118,7 @@ func (ex *Executor) Run(ctx context.Context) *exec.Cmd {
// RunTest prepares the Cmd for execution of the unit test
// Returns Cmd instance
func (ex *Executor) RunTest(ctx context.Context) *exec.Cmd {
- args := append(ex.testArgs.commandArgs, ex.testArgs.fileName)
+ args := append(ex.testArgs.commandArgs, ex.testArgs.fileNames...)
cmd := exec.CommandContext(ctx, ex.testArgs.commandName, args...)
cmd.Dir = ex.testArgs.workingDir
return cmd
diff --git a/playground/backend/internal/executors/executor_builder.go
b/playground/backend/internal/executors/executor_builder.go
index c810f96e89c..371f02e3418 100644
--- a/playground/backend/internal/executors/executor_builder.go
+++ b/playground/backend/internal/executors/executor_builder.go
@@ -106,18 +106,18 @@ func (b *CompileBuilder) WithArgs(compileArgs []string)
*CompileBuilder {
return b
}
-// WithFileName adds file name to executor
-func (b *CompileBuilder) WithFileName(fileName string) *CompileBuilder {
+// WithFileNames adds file names to executor
+func (b *CompileBuilder) WithFileNames(fileNames ...string) *CompileBuilder {
b.actions = append(b.actions, func(e *Executor) {
- e.compileArgs.fileName = fileName
+ e.compileArgs.fileNames = fileNames
})
return b
}
-// WithExecutableFileName adds file name to executor
-func (b *RunBuilder) WithExecutableFileName(name string) *RunBuilder {
+// WithExecutableFileNames adds file name to executor
+func (b *RunBuilder) WithExecutableFileNames(names ...string) *RunBuilder {
b.actions = append(b.actions, func(e *Executor) {
- e.runArgs.fileName = name
+ e.runArgs.fileNames = names
})
return b
}
@@ -146,14 +146,6 @@ func (b *RunBuilder) WithArgs(runArgs []string)
*RunBuilder {
return b
}
-// WithGraphOutput adds the need of graph output to executor
-func (b *RunBuilder) WithGraphOutput() *RunBuilder {
- b.actions = append(b.actions, func(e *Executor) {
- //todo
- })
- return b
-}
-
// WithCommand adds test command to executor
func (b *UnitTestExecutorBuilder) WithCommand(testCmd string)
*UnitTestExecutorBuilder {
b.actions = append(b.actions, func(e *Executor) {
@@ -178,18 +170,10 @@ func (b *UnitTestExecutorBuilder) WithWorkingDir(dir
string) *UnitTestExecutorBu
return b
}
-// WithGraphOutput adds the need of graph output to executor
-func (b *UnitTestExecutorBuilder) WithGraphOutput() *UnitTestExecutorBuilder {
- b.actions = append(b.actions, func(e *Executor) {
- //todo
- })
- return b
-}
-
// WithExecutableFileName adds file name to executor
func (b *UnitTestExecutorBuilder) WithExecutableFileName(name string)
*UnitTestExecutorBuilder {
b.actions = append(b.actions, func(e *Executor) {
- e.testArgs.fileName = name
+ e.testArgs.fileNames = append(e.testArgs.fileNames, name)
})
return b
}
diff --git a/playground/backend/internal/executors/executor_builder_test.go
b/playground/backend/internal/executors/executor_builder_test.go
index 9c61a875369..8778b3b2f72 100644
--- a/playground/backend/internal/executors/executor_builder_test.go
+++ b/playground/backend/internal/executors/executor_builder_test.go
@@ -25,7 +25,7 @@ var handlers []handler
func TestMain(m *testing.M) {
handlers = []handler{
func(e *Executor) {
- e.testArgs.fileName = "file name"
+ e.testArgs.fileNames = append(e.testArgs.fileNames,
"file name")
},
func(e *Executor) {
e.runArgs.pipelineOptions = []string{"--opt val"}
diff --git a/playground/backend/internal/executors/executor_test.go
b/playground/backend/internal/executors/executor_test.go
index 603b748211f..947832d4fca 100644
--- a/playground/backend/internal/executors/executor_test.go
+++ b/playground/backend/internal/executors/executor_test.go
@@ -44,7 +44,7 @@ func TestExecutor_Compile(t *testing.T) {
name: "TestCompile",
fields: fields{
compileArgs: CmdConfiguration{
- fileName: "filePath",
+ fileNames: []string{"filePath"},
workingDir: "./",
commandName: "testCommand",
commandArgs: []string{"-d", "bin",
"-parameters", "-classpath",
"/opt/apache/beam/jars/beam-sdks-java-harness.jar"},
@@ -97,7 +97,7 @@ func TestExecutor_Run(t *testing.T) {
name: "TestRun",
fields: fields{
runArgs: CmdConfiguration{
- fileName: "HelloWorld",
+ fileNames: []string{"HelloWorld"},
workingDir: "./",
commandName: "runCommand",
commandArgs: []string{"-cp",
"bin:/opt/apache/beam/jars/beam-sdks-java-harness.jar:" +
@@ -124,7 +124,7 @@ func TestExecutor_Run(t *testing.T) {
name: "TestRun with pipelineOptions",
fields: fields{
runArgs: CmdConfiguration{
- fileName: "HelloWorld",
+ fileNames: []string{"HelloWorld"},
workingDir: "./",
commandName: "runCommand",
commandArgs: []string{"-cp",
"bin:/opt/apache/beam/jars/beam-sdks-java-harness.jar:" +
@@ -185,7 +185,7 @@ func TestExecutor_RunTest(t *testing.T) {
name: "TestRunTest",
fields: fields{
testArgs: CmdConfiguration{
- fileName: "HelloWorld",
+ fileNames: []string{"HelloWorld"},
workingDir: "./",
commandName: "testCommand",
commandArgs: []string{"-cp",
"option1:option2"},
diff --git a/playground/backend/internal/fs_tool/fs.go
b/playground/backend/internal/fs_tool/fs.go
index f132e71c21d..9c49fca53a0 100644
--- a/playground/backend/internal/fs_tool/fs.go
+++ b/playground/backend/internal/fs_tool/fs.go
@@ -16,7 +16,7 @@
package fs_tool
import (
- "beam.apache.org/playground/backend/internal/logger"
+ "context"
"fmt"
"github.com/google/uuid"
"io/fs"
@@ -25,6 +25,7 @@ import (
pb "beam.apache.org/playground/backend/internal/api/v1"
"beam.apache.org/playground/backend/internal/db/entity"
"beam.apache.org/playground/backend/internal/emulators"
+ "beam.apache.org/playground/backend/internal/logger"
)
const (
@@ -44,7 +45,8 @@ type LifeCyclePaths struct {
AbsoluteLogFilePath string //
/path/to/workingDir/pipelinesFolder/{pipelineId}/logs.log
AbsoluteGraphFilePath string //
/path/to/workingDir/pipelinesFolder/{pipelineId}/graph.dot
ProjectDir string // /path/to/workingDir/
- ExecutableName func(string) (string, error)
+ FindExecutableName func(context.Context, string) (string,
error)
+ FindTestExecutableName func(context.Context, string) (string,
error)
}
// LifeCycle is used for preparing folders and files to process code for one
code processing request.
diff --git a/playground/backend/internal/fs_tool/fs_test.go
b/playground/backend/internal/fs_tool/fs_test.go
index 4241fd9a486..db029635961 100644
--- a/playground/backend/internal/fs_tool/fs_test.go
+++ b/playground/backend/internal/fs_tool/fs_test.go
@@ -327,9 +327,9 @@ func TestNewLifeCycle(t *testing.T) {
want: &LifeCycle{
folderGlobs: []string{baseFileFolder,
srcFileFolder, execFileFolder},
Paths: LifeCyclePaths{
- SourceFileName:
fmt.Sprintf("%s%s", pipelineId.String(), goSourceFileExtension),
+ SourceFileName:
fmt.Sprintf("%s%s", pipelineId.String(), GoSourceFileExtension),
AbsoluteSourceFileFolderPath:
srcFileFolder,
- AbsoluteSourceFilePath:
filepath.Join(srcFileFolder, fmt.Sprintf("%s%s", pipelineId.String(),
goSourceFileExtension)),
+ AbsoluteSourceFilePath:
filepath.Join(srcFileFolder, fmt.Sprintf("%s%s", pipelineId.String(),
GoSourceFileExtension)),
ExecutableFileName:
fmt.Sprintf("%s%s", pipelineId.String(), goExecutableFileExtension),
AbsoluteExecutableFileFolderPath:
execFileFolder,
AbsoluteExecutableFilePath:
filepath.Join(execFileFolder, fmt.Sprintf("%s%s", pipelineId.String(),
goExecutableFileExtension)),
diff --git a/playground/backend/internal/fs_tool/go_fs.go
b/playground/backend/internal/fs_tool/go_fs.go
index 883f1803f68..f624152d3c9 100644
--- a/playground/backend/internal/fs_tool/go_fs.go
+++ b/playground/backend/internal/fs_tool/go_fs.go
@@ -20,11 +20,11 @@ import (
)
const (
- goSourceFileExtension = ".go"
+ GoSourceFileExtension = ".go"
goExecutableFileExtension = ""
)
// newGoLifeCycle creates LifeCycle with go SDK environment.
func newGoLifeCycle(pipelineId uuid.UUID, pipelinesFolder string) *LifeCycle {
- return newCompilingLifeCycle(pipelineId, pipelinesFolder,
goSourceFileExtension, goExecutableFileExtension)
+ return newCompilingLifeCycle(pipelineId, pipelinesFolder,
GoSourceFileExtension, goExecutableFileExtension)
}
diff --git a/playground/backend/internal/fs_tool/go_fs_test.go
b/playground/backend/internal/fs_tool/go_fs_test.go
index 8021e1c906b..e5e513fd253 100644
--- a/playground/backend/internal/fs_tool/go_fs_test.go
+++ b/playground/backend/internal/fs_tool/go_fs_test.go
@@ -50,9 +50,9 @@ func Test_newGoLifeCycle(t *testing.T) {
want: &LifeCycle{
folderGlobs: []string{baseFileFolder,
srcFileFolder, binFileFolder},
Paths: LifeCyclePaths{
- SourceFileName:
pipelineId.String() + goSourceFileExtension,
+ SourceFileName:
pipelineId.String() + GoSourceFileExtension,
AbsoluteSourceFileFolderPath:
srcFileFolder,
- AbsoluteSourceFilePath:
filepath.Join(srcFileFolder, pipelineId.String()+goSourceFileExtension),
+ AbsoluteSourceFilePath:
filepath.Join(srcFileFolder, pipelineId.String()+GoSourceFileExtension),
ExecutableFileName:
pipelineId.String() + goExecutableFileExtension,
AbsoluteExecutableFileFolderPath:
binFileFolder,
AbsoluteExecutableFilePath:
filepath.Join(binFileFolder, pipelineId.String()+goExecutableFileExtension),
diff --git a/playground/backend/internal/fs_tool/java_fs.go
b/playground/backend/internal/fs_tool/java_fs.go
index aa0ee6f1282..3666ee37244 100644
--- a/playground/backend/internal/fs_tool/java_fs.go
+++ b/playground/backend/internal/fs_tool/java_fs.go
@@ -16,10 +16,13 @@
package fs_tool
import (
+ "bytes"
+ "context"
"errors"
"fmt"
- "io/ioutil"
"os"
+ "os/exec"
+ "path/filepath"
"strings"
"github.com/google/uuid"
@@ -30,19 +33,23 @@ import (
)
const (
- JavaSourceFileExtension = ".java"
- javaCompiledFileExtension = ".class"
+ JavaSourceFileExtension = ".java"
+ javaCompiledFileExtension = ".class"
+ javaEntryPointFullName = "public static void
main(java.lang.String[])"
+ javaDecompilerCommand = "javap"
+ juintRunWithTestAnnotationConstant = "Lorg/junit/runner/RunWith;"
)
// newJavaLifeCycle creates LifeCycle with java SDK environment.
func newJavaLifeCycle(pipelineId uuid.UUID, pipelinesFolder string) *LifeCycle
{
javaLifeCycle := newCompilingLifeCycle(pipelineId, pipelinesFolder,
JavaSourceFileExtension, javaCompiledFileExtension)
- javaLifeCycle.Paths.ExecutableName = executableName
+ javaLifeCycle.Paths.FindExecutableName = findExecutableName
+ javaLifeCycle.Paths.FindTestExecutableName = findTestExecutableName
return javaLifeCycle
}
-// executableName returns name that should be executed (HelloWorld for
HelloWorld.class for java SDK)
-func executableName(executableFileFolderPath string) (string, error) {
+// findExecutableName returns name of the .class file which has main() method
+func findExecutableName(ctx context.Context, executableFileFolderPath string)
(string, error) {
dirEntries, err := os.ReadDir(executableFileFolderPath)
if err != nil {
return "", err
@@ -52,27 +59,110 @@ func executableName(executableFileFolderPath string)
(string, error) {
}
if len(dirEntries) == 1 {
- return strings.Split(dirEntries[0].Name(), ".")[0], nil
+ return utils.TrimExtension(dirEntries[0].Name()), nil
}
for _, entry := range dirEntries {
- content, err := ioutil.ReadFile(fmt.Sprintf("%s/%s",
executableFileFolderPath, entry.Name()))
- if err != nil {
- logger.Error(fmt.Sprintf("error during file reading:
%s", err.Error()))
- break
- }
- ext := strings.Split(entry.Name(), ".")[1]
- sdk := utils.ToSDKFromExt("." + ext)
+ select {
+ case <-ctx.Done():
+ return "", ctx.Err()
+ default:
+ filePath := fmt.Sprintf("%s/%s",
executableFileFolderPath, entry.Name())
+ content, err := os.ReadFile(filePath)
+ if err != nil {
+ logger.Errorf("findExecutableName(): error when
reading file %s: %s", entry.Name(), err.Error())
+ break
+ }
+ ext := filepath.Ext(entry.Name())
+ filename := strings.TrimSuffix(entry.Name(), ext)
+ sdk := utils.ToSDKFromExt(ext)
+
+ if sdk == pb.Sdk_SDK_UNSPECIFIED {
+ logger.Errorf("findExecutableName(): file %s:
unknown file extension: %s, skipping", entry.Name(), ext)
+ continue
+ }
- if sdk == pb.Sdk_SDK_UNSPECIFIED {
- logger.Error("invalid a file extension")
- break
+ switch ext {
+ case javaCompiledFileExtension:
+ isMain, err := isMainClass(ctx,
executableFileFolderPath, filename)
+ if err != nil {
+ logger.Errorf("findExecutableName():
file %s: error during checking main class: %s", entry.Name(), err.Error())
+ break
+ }
+ if isMain {
+ logger.Infof("findExecutableName():
main file is %s", filename)
+ return filename, nil
+ }
+ default:
+ if utils.IsFileMain(string(content), sdk) {
+ return filename, nil
+ }
+ }
}
+ }
+
+ return "", errors.New("cannot find file with main() method")
+}
+
+// findTestExecutableName returns name of the .class file which has JUnit tests
+func findTestExecutableName(ctx context.Context, executableFileFolderPath
string) (string, error) {
+ dirEntries, err := os.ReadDir(executableFileFolderPath)
+ if err != nil {
+ return "", err
+ }
+ if len(dirEntries) < 1 {
+ return "", errors.New("number of executable files should be at
least one")
+ }
+
+ if len(dirEntries) == 1 {
+ return utils.TrimExtension(dirEntries[0].Name()), nil
+ }
+
+ for _, entry := range dirEntries {
+ select {
+ case <-ctx.Done():
+ return "", ctx.Err()
+ default:
+ ext := filepath.Ext(entry.Name())
+ filename := strings.TrimSuffix(entry.Name(), ext)
- if utils.IsFileMain(string(content), sdk) {
- return strings.Split(entry.Name(), ".")[0], nil
+ if ext == javaCompiledFileExtension {
+ isTest, err := isTestClass(ctx,
executableFileFolderPath, filename)
+ if err != nil {
+
logger.Errorf("findTestExecutableName(): file %s: error during checking main
class: %s", entry.Name(), err.Error())
+ break
+ }
+ if isTest {
+ logger.Infof("findTestExecutableName():
main file is %s", filename)
+ return filename, nil
+ }
+ }
}
}
- return strings.Split(dirEntries[len(dirEntries)-1].Name(), ".")[0], nil
+ return "", errors.New("cannot find file with unit tests")
+}
+
+func isMainClass(ctx context.Context, classPath string, className string)
(bool, error) {
+ cmd := exec.CommandContext(ctx, javaDecompilerCommand, "-public",
"-classpath", classPath, className)
+ var out bytes.Buffer
+ cmd.Stdout = &out
+ err := cmd.Run()
+ if err != nil {
+ return false, err
+ }
+
+ return strings.Contains(out.String(), javaEntryPointFullName), nil
+}
+
+func isTestClass(ctx context.Context, classPath string, className string)
(bool, error) {
+ cmd := exec.CommandContext(ctx, javaDecompilerCommand, "-verbose",
"-classpath", classPath, className)
+ var out bytes.Buffer
+ cmd.Stdout = &out
+ err := cmd.Run()
+ if err != nil {
+ return false, err
+ }
+
+ return strings.Contains(out.String(),
juintRunWithTestAnnotationConstant), nil
}
diff --git a/playground/backend/internal/fs_tool/java_fs_test.go
b/playground/backend/internal/fs_tool/java_fs_test.go
index b8144f816a4..9ac452b91e1 100644
--- a/playground/backend/internal/fs_tool/java_fs_test.go
+++ b/playground/backend/internal/fs_tool/java_fs_test.go
@@ -16,14 +16,15 @@
package fs_tool
import (
+ "beam.apache.org/playground/backend/internal/utils"
+ "context"
"os"
+ "os/exec"
"path/filepath"
"reflect"
"testing"
"github.com/google/uuid"
-
- "beam.apache.org/playground/backend/internal/utils"
)
func Test_newJavaLifeCycle(t *testing.T) {
@@ -62,7 +63,7 @@ func Test_newJavaLifeCycle(t *testing.T) {
AbsoluteBaseFolderPath:
baseFileFolder,
AbsoluteLogFilePath:
filepath.Join(baseFileFolder, logFileName),
AbsoluteGraphFilePath:
filepath.Join(baseFileFolder, utils.GraphFileName),
- ExecutableName:
executableName,
+ FindExecutableName:
findExecutableName,
},
},
},
@@ -85,24 +86,62 @@ func Test_executableName(t *testing.T) {
workDir := "workingDir"
preparedPipelinesFolder := filepath.Join(workDir, pipelinesFolder)
lc := newJavaLifeCycle(pipelineId, preparedPipelinesFolder)
- lc.CreateFolders()
- defer os.RemoveAll(workDir)
+ err := lc.CreateFolders()
+ if err != nil {
+ t.Errorf("Failed to create folders %s, error = %v", workDir,
err)
+ }
+ defer func() {
+ err := os.RemoveAll(workDir)
+ if err != nil {
+ t.Errorf("Failed to cleanup %s, error = %v", workDir,
err)
+ }
+ }()
+
+ compileJavaFiles := func(sourceFiles ...string) error {
+ compiledDir := filepath.Join(workDir, pipelinesFolder,
pipelineId.String(), compiledFolderName)
+
+ args := append([]string{"-d", compiledDir}, sourceFiles...)
+ err := exec.Command("javac", args...).Run()
+ if err != nil {
+ return err
+ }
+ return nil
+ }
+
+ cleanupFunc := func() error {
+ compiled := filepath.Join(workDir, pipelinesFolder,
pipelineId.String(), compiledFolderName)
+ dirEntries, err := os.ReadDir(compiled)
+ if err != nil {
+ return err
+ }
+
+ for _, entry := range dirEntries {
+ err := os.Remove(filepath.Join(compiled, entry.Name()))
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
type args struct {
executableFolder string
}
tests := []struct {
name string
- prepare func()
+ prepare func() error
+ cleanup func() error
args args
want string
wantErr bool
}{
{
- // Test case with calling sourceFileName method with
empty directory.
+ // Test case with calling findExecutableName() function
with empty directory.
// As a result, want to receive an error.
name: "Directory is empty",
- prepare: func() {},
+ prepare: func() error { return nil },
+ cleanup: func() error { return nil },
args: args{
executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
},
@@ -110,17 +149,19 @@ func Test_executableName(t *testing.T) {
wantErr: true,
},
{
- // Test case with calling sourceFileName method with
correct pipelineId and workingDir.
+ // Test case with calling findExecutableName() function
with correct pipelineId and workingDir.
// As a result, want to receive a name that should be
executed
name: "Get executable name",
- prepare: func() {
+ prepare: func() error {
compiled := filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), compiledFolderName)
filePath := filepath.Join(compiled,
"temp.class")
err := os.WriteFile(filePath,
[]byte("TEMP_DATA"), 0600)
if err != nil {
- panic(err)
+ return err
}
+ return nil
},
+ cleanup: cleanupFunc,
args: args{
executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
},
@@ -128,10 +169,11 @@ func Test_executableName(t *testing.T) {
wantErr: false,
},
{
- // Test case with calling sourceFileName method with
wrong directory.
+ // Test case with calling findExecutableName() function
with wrong directory.
// As a result, want to receive an error.
name: "Directory doesn't exist",
- prepare: func() {},
+ prepare: func() error { return nil },
+ cleanup: func() error { return nil },
args: args{
executableFolder: filepath.Join(workDir,
pipelineId.String()),
},
@@ -139,39 +181,162 @@ func Test_executableName(t *testing.T) {
wantErr: true,
},
{
- // Test case with calling sourceFileName method with
multiple files where one of them is main
+ // Test case with calling findExecutableName() function
with multiple files where one of them is main
// As a result, want to receive a name that should be
executed
name: "Multiple files where one of them is main",
- prepare: func() {
+ prepare: func() error {
compiled := filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), compiledFolderName)
- secondaryFilePath := filepath.Join(compiled,
"temp.scala")
- err := os.WriteFile(secondaryFilePath,
[]byte("TEMP_DATA"), 0600)
+ primaryFilePath := filepath.Join(compiled,
"main.scala")
+ err := os.WriteFile(primaryFilePath,
[]byte("object MinimalWordCount {def main(cmdlineArgs: Array[String]): Unit =
{}}"), 0600)
if err != nil {
- panic(err)
+ return err
}
- primaryFilePath := filepath.Join(compiled,
"main.scala")
- err = os.WriteFile(primaryFilePath,
[]byte("object MinimalWordCount {def main(cmdlineArgs: Array[String]): Unit =
{}}"), 0600)
+ secondaryFilePath := filepath.Join(compiled,
"temp.scala")
+ err = os.WriteFile(secondaryFilePath,
[]byte("TEMP_DATA"), 0600)
if err != nil {
- panic(err)
+ return err
}
+ return nil
},
+ cleanup: cleanupFunc,
args: args{
executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
},
want: "main",
wantErr: false,
},
+ {
+ // Test case with calling findExecutableName() function
with multiple files where one of them is a .class file
+ // with main() method. The executable class name is the
same as the source file name.
+ // As a result, want to receive a name that should be
executed
+ name: "Multiple Java class files where one of them
contains main",
+ prepare: func() error {
+ testdataPath := "java_testdata"
+ sourceFile := filepath.Join(testdataPath,
"HasMainTest1.java")
+
+ err := compileJavaFiles(sourceFile)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ },
+ cleanup: cleanupFunc,
+ args: args{
+ executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
+ },
+ want: "HasMainTest1",
+ wantErr: false,
+ },
+ {
+ // Test case with calling findExecutableName() function
with multiple files where one of them is a .class file
+ // with main() method. The executable class name is
different from the source file name.
+ // As a result, want to receive a name that should be
executed
+ name: "Multiple Java class files where one of them
contains main",
+ prepare: func() error {
+ testdataPath := "java_testdata"
+ // The source file contains two classes, one of
them has main() method (class name is different from the source file name)
+ sourceFile := filepath.Join(testdataPath,
"HasMainTest2.java")
+
+ err := compileJavaFiles(sourceFile)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ },
+ cleanup: cleanupFunc,
+ args: args{
+ executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
+ },
+ want: "Bar",
+ wantErr: false,
+ },
+ {
+ // Test case with calling findExecutableName() function
with multiple files where one of them is a .class file
+ // with main() method with incorrect signature.
+ // As a result, want to receive a name that should be
executed
+ name: "Multiple Java class files where one of them
contains main() with incorrect signature",
+ prepare: func() error {
+ testdataPath := "java_testdata"
+ sourceFile := filepath.Join(testdataPath,
"HasIncorrectMain.java")
+
+ err := compileJavaFiles(sourceFile)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ },
+ cleanup: cleanupFunc,
+ args: args{
+ executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
+ },
+ want: "",
+ wantErr: true,
+ },
+ {
+ // Test case with calling findExecutableName() function
with multiple files where none of them has main().
+ // As a result, want to receive a name that should be
executed
+ name: "Multiple Java class files where none of them
contain main()",
+ prepare: func() error {
+ testdataPath := "java_testdata"
+ sourceFile := filepath.Join(testdataPath,
"HasNoMain.java")
+
+ err := compileJavaFiles(sourceFile)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ },
+ cleanup: cleanupFunc,
+ args: args{
+ executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
+ },
+ want: "",
+ wantErr: true,
+ },
+ {
+ // Test case with calling findExecutableName() function
with file which has multiple dots in its name
+ // As a result, want to receive a name that should be
executed
+ name: "File with multiple dots in the name",
+ prepare: func() error {
+ compiled := filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), compiledFolderName)
+ primaryFilePath := filepath.Join(compiled,
"main.function.scala")
+ err := os.WriteFile(primaryFilePath,
[]byte("object MinimalWordCount {def main(cmdlineArgs: Array[String]): Unit =
{}}"), 0600)
+ if err != nil {
+ return err
+ }
+ return nil
+ },
+ cleanup: cleanupFunc,
+ args: args{
+ executableFolder: filepath.Join(workDir,
pipelinesFolder, pipelineId.String(), "bin"),
+ },
+ want: "main.function",
+ wantErr: false,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- tt.prepare()
- got, err := executableName(tt.args.executableFolder)
+ err := tt.prepare()
+ if err != nil {
+ t.Errorf("java_fs_test cleanup error = %v", err)
+ }
+ defer func() {
+ err = tt.cleanup()
+ if err != nil {
+ t.Errorf("java_fs_test cleanup error =
%v", err)
+ }
+ }()
+ got, err := findExecutableName(context.Background(),
tt.args.executableFolder)
if (err != nil) != tt.wantErr {
- t.Errorf("sourceFileName() error = %v, wantErr
%v", err, tt.wantErr)
+ t.Errorf("java_fs_test error = %v, wantErr %v",
err, tt.wantErr)
return
}
if got != tt.want {
- t.Errorf("sourceFileName() got = %v, want %v",
got, tt.want)
+ t.Errorf("java_fs_test got = %v, want %v", got,
tt.want)
}
})
}
diff --git
a/playground/backend/internal/fs_tool/java_testdata/HasIncorrectMain.java
b/playground/backend/internal/fs_tool/java_testdata/HasIncorrectMain.java
new file mode 100644
index 00000000000..c9309e9e89f
--- /dev/null
+++ b/playground/backend/internal/fs_tool/java_testdata/HasIncorrectMain.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+class Bar {
+ public static void bar() {
+ System.out.println("Hello");
+ }
+}
+
+class HasIncorrectMain {
+ public static void main() {
+ Bar.bar();
+ }
+}
diff --git
a/playground/backend/internal/fs_tool/java_testdata/HasMainTest1.java
b/playground/backend/internal/fs_tool/java_testdata/HasMainTest1.java
new file mode 100644
index 00000000000..718ea320581
--- /dev/null
+++ b/playground/backend/internal/fs_tool/java_testdata/HasMainTest1.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+class Bar {
+ public static void bar() {
+ System.out.println("Hello");
+ }
+}
+
+class HasMainTest1 {
+ public static void main(java.lang.String[] args) {
+ Bar.bar();
+ }
+}
diff --git
a/playground/backend/internal/fs_tool/java_testdata/HasMainTest2.java
b/playground/backend/internal/fs_tool/java_testdata/HasMainTest2.java
new file mode 100644
index 00000000000..299d88a18c9
--- /dev/null
+++ b/playground/backend/internal/fs_tool/java_testdata/HasMainTest2.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+class Bar {
+ public static void bar() {
+ System.out.println("Hello");
+ }
+ public static void main(java.lang.String[] args) {
+ Bar.bar();
+ }
+}
+
+class HasMainTest1 {
+
+}
diff --git a/playground/backend/internal/fs_tool/java_testdata/HasNoMain.java
b/playground/backend/internal/fs_tool/java_testdata/HasNoMain.java
new file mode 100644
index 00000000000..c9e8c0e3d1b
--- /dev/null
+++ b/playground/backend/internal/fs_tool/java_testdata/HasNoMain.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+class Bar {
+ public static void bar() {
+ System.out.println("Hello");
+ }
+}
+
+class HasIncorrectMain {
+ public static void foo() {
+ Bar.bar();
+ }
+}
diff --git a/playground/backend/internal/fs_tool/scio_fs.go
b/playground/backend/internal/fs_tool/scio_fs.go
index ea8d834ca8b..491c158d2b7 100644
--- a/playground/backend/internal/fs_tool/scio_fs.go
+++ b/playground/backend/internal/fs_tool/scio_fs.go
@@ -26,6 +26,6 @@ const (
// newScioLifeCycle creates LifeCycle with scala SDK environment.
func newScioLifeCycle(pipelineId uuid.UUID, pipelinesFolder string) *LifeCycle
{
lc := newInterpretedLifeCycle(pipelineId, pipelinesFolder,
scioExecutableFileExtension)
- lc.Paths.ExecutableName = executableName
+ lc.Paths.FindExecutableName = findExecutableName
return lc
}
diff --git a/playground/backend/internal/setup_tools/builder/setup_builder.go
b/playground/backend/internal/setup_tools/builder/setup_builder.go
index 772cb6aebcf..f7d68582b06 100644
--- a/playground/backend/internal/setup_tools/builder/setup_builder.go
+++ b/playground/backend/internal/setup_tools/builder/setup_builder.go
@@ -16,6 +16,7 @@
package builder
import (
+ "context"
"fmt"
"path/filepath"
"strings"
@@ -64,7 +65,7 @@ func Preparer(paths *fs_tool.LifeCyclePaths, sdkEnv
*environment.BeamEnvs, valRe
}
// Compiler return executor with set args for compiler
-func Compiler(paths *fs_tool.LifeCyclePaths, sdkEnv *environment.BeamEnvs)
*executors.ExecutorBuilder {
+func Compiler(paths *fs_tool.LifeCyclePaths, sdkEnv *environment.BeamEnvs)
(*executors.ExecutorBuilder, error) {
sdk := sdkEnv.ApacheBeamSdk
executorConfig := sdkEnv.ExecutorConfig
builder := executors.NewExecutorBuilder().
@@ -72,21 +73,38 @@ func Compiler(paths *fs_tool.LifeCyclePaths, sdkEnv
*environment.BeamEnvs) *exec
WithCommand(executorConfig.CompileCmd).
WithWorkingDir(paths.AbsoluteBaseFolderPath).
WithArgs(executorConfig.CompileArgs).
- WithFileName(paths.AbsoluteSourceFilePath).
ExecutorBuilder
switch sdk {
case pb.Sdk_SDK_JAVA:
+ javaSources, err :=
GetFilesFromFolder(paths.AbsoluteSourceFileFolderPath,
fs_tool.JavaSourceFileExtension)
+ if err != nil {
+ return nil, err
+ }
builder = builder.
WithCompiler().
-
WithFileName(GetFirstFileFromFolder(paths.AbsoluteSourceFileFolderPath)).
+ WithFileNames(javaSources...).
+ ExecutorBuilder
+ case pb.Sdk_SDK_GO:
+ goSources, err :=
GetFilesFromFolder(paths.AbsoluteSourceFileFolderPath,
fs_tool.GoSourceFileExtension)
+ if err != nil {
+ return nil, err
+ }
+ builder = builder.
+ WithCompiler().
+ WithFileNames(goSources...).
+ ExecutorBuilder
+ default:
+ builder = builder.
+ WithCompiler().
+ WithFileNames(paths.AbsoluteSourceFilePath).
ExecutorBuilder
}
- return &builder
+ return &builder, nil
}
// Runner return executor with set args for runner
-func Runner(paths *fs_tool.LifeCyclePaths, pipelineOptions string, sdkEnv
*environment.BeamEnvs) (*executors.ExecutorBuilder, error) {
+func Runner(ctx context.Context, paths *fs_tool.LifeCyclePaths,
pipelineOptions string, sdkEnv *environment.BeamEnvs)
(*executors.ExecutorBuilder, error) {
sdk := sdkEnv.ApacheBeamSdk
if sdk == pb.Sdk_SDK_JAVA || sdk == pb.Sdk_SDK_SCIO {
@@ -103,31 +121,30 @@ func Runner(paths *fs_tool.LifeCyclePaths,
pipelineOptions string, sdkEnv *envir
switch sdk {
case pb.Sdk_SDK_JAVA: // Executable name for java class is known after
compilation
args := replaceLogPlaceholder(paths, executorConfig)
- className, err :=
paths.ExecutableName(paths.AbsoluteExecutableFileFolderPath)
+ className, err := paths.FindExecutableName(ctx,
paths.AbsoluteExecutableFileFolderPath)
if err != nil {
return nil, fmt.Errorf("no executable file name found
for JAVA pipeline at %s", paths.AbsoluteExecutableFileFolderPath)
}
builder = builder.
WithRunner().
WithArgs(args).
- WithExecutableFileName(className).
+ WithExecutableFileNames(className).
WithPipelineOptions(strings.Split(pipelineOptions, "
")).
ExecutorBuilder
case pb.Sdk_SDK_GO: //go run command is executable file itself
builder = builder.
WithRunner().
- WithExecutableFileName("").
WithCommand(paths.AbsoluteExecutableFilePath).
WithPipelineOptions(strings.Split(pipelineOptions, "
")).
ExecutorBuilder
case pb.Sdk_SDK_PYTHON:
builder = builder.
WithRunner().
-
WithExecutableFileName(paths.AbsoluteExecutableFilePath).
+
WithExecutableFileNames(paths.AbsoluteExecutableFilePath).
WithPipelineOptions(strings.Split(pipelineOptions, "
")).
ExecutorBuilder
case pb.Sdk_SDK_SCIO:
- className, err :=
paths.ExecutableName(paths.AbsoluteBaseFolderPath)
+ className, err := paths.FindExecutableName(ctx,
paths.AbsoluteBaseFolderPath)
if err != nil {
return nil, fmt.Errorf("no executable file name found
for SCIO pipeline at %s", paths.AbsoluteBaseFolderPath)
}
@@ -142,12 +159,11 @@ func Runner(paths *fs_tool.LifeCyclePaths,
pipelineOptions string, sdkEnv *envir
}
// TestRunner return executor with set args for runner
-func TestRunner(paths *fs_tool.LifeCyclePaths, sdkEnv *environment.BeamEnvs)
(*executors.ExecutorBuilder, error) {
+func TestRunner(ctx context.Context, paths *fs_tool.LifeCyclePaths, sdkEnv
*environment.BeamEnvs) (*executors.ExecutorBuilder, error) {
sdk := sdkEnv.ApacheBeamSdk
executorConfig := sdkEnv.ExecutorConfig
builder := executors.NewExecutorBuilder().
WithTestRunner().
- WithExecutableFileName(paths.AbsoluteExecutableFilePath).
WithCommand(executorConfig.TestCmd).
WithArgs(executorConfig.TestArgs).
WithWorkingDir(paths.AbsoluteSourceFileFolderPath).
@@ -155,7 +171,7 @@ func TestRunner(paths *fs_tool.LifeCyclePaths, sdkEnv
*environment.BeamEnvs) (*e
switch sdk {
case pb.Sdk_SDK_JAVA: // Executable name for java class is known after
compilation
- className, err :=
paths.ExecutableName(paths.AbsoluteExecutableFileFolderPath)
+ className, err := paths.FindTestExecutableName(ctx,
paths.AbsoluteExecutableFileFolderPath)
if err != nil {
return nil, fmt.Errorf("no executable file name found
for JAVA pipeline at %s", paths.AbsoluteExecutableFileFolderPath)
}
@@ -167,6 +183,10 @@ func TestRunner(paths *fs_tool.LifeCyclePaths, sdkEnv
*environment.BeamEnvs) (*e
builder = builder.WithTestRunner().
WithExecutableFileName(paths.AbsoluteSourceFileFolderPath). // run all tests in
folder
ExecutorBuilder
+ default:
+ builder = builder.WithTestRunner().
+
WithExecutableFileName(paths.AbsoluteExecutableFilePath).
+ ExecutorBuilder
}
return &builder, nil
}
@@ -185,7 +205,6 @@ func replaceLogPlaceholder(paths *fs_tool.LifeCyclePaths,
executorConfig *enviro
}
// GetFirstFileFromFolder return a name of the first file in a specified folder
-func GetFirstFileFromFolder(folderAbsolutePath string) string {
- files, _ := filepath.Glob(fmt.Sprintf("%s/*%s", folderAbsolutePath,
fs_tool.JavaSourceFileExtension))
- return files[0]
+func GetFilesFromFolder(folderAbsolutePath string, extension string)
([]string, error) {
+ return filepath.Glob(fmt.Sprintf("%s/*%s", folderAbsolutePath,
extension))
}
diff --git
a/playground/backend/internal/setup_tools/builder/setup_builder_test.go
b/playground/backend/internal/setup_tools/builder/setup_builder_test.go
index 08eaab3d542..2387a0c9164 100644
--- a/playground/backend/internal/setup_tools/builder/setup_builder_test.go
+++ b/playground/backend/internal/setup_tools/builder/setup_builder_test.go
@@ -16,7 +16,9 @@
package builder
import (
+ "context"
"fmt"
+ "log"
"os"
"path/filepath"
"reflect"
@@ -38,7 +40,10 @@ const emptyFolder = "emptyFolder"
var pythonPaths *fs_tool.LifeCyclePaths
var pythonSdkEnv *environment.BeamEnvs
+var pythonLC *fs_tool.LifeCycle
var javaLC *fs_tool.LifeCycle
+var goLC *fs_tool.LifeCycle
+var scioLC *fs_tool.LifeCycle
var javaPaths *fs_tool.LifeCyclePaths
var javaSdkEnv *environment.BeamEnvs
var goPaths *fs_tool.LifeCyclePaths
@@ -47,30 +52,80 @@ var scioPaths *fs_tool.LifeCyclePaths
var scioSdkEnv *environment.BeamEnvs
func TestMain(m *testing.M) {
- setup()
- defer teardown()
+ err := setup()
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer func() {
+ err = teardown()
+ if err != nil {
+ log.Fatal(err)
+ }
+ }()
m.Run()
}
-func setup() {
- os.Mkdir(emptyFolder, 0666)
-
- pipelineId := uuid.New()
+func setup() error {
+ err := os.Mkdir(emptyFolder, 0666)
+ if err != nil {
+ return err
+ }
- pythonLC, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_PYTHON, pipelineId, "")
+ pythonPipelineId := uuid.New()
+ pythonLC, err = fs_tool.NewLifeCycle(pb.Sdk_SDK_PYTHON,
pythonPipelineId, "")
+ if err != nil {
+ return err
+ }
+ err = pythonLC.CreateFolders()
+ if err != nil {
+ return err
+ }
pythonPaths = &pythonLC.Paths
- javaLC, _ = fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, pipelineId, "")
+ javaPipelineId := uuid.New()
+ javaLC, err = fs_tool.NewLifeCycle(pb.Sdk_SDK_JAVA, javaPipelineId, "")
+ if err != nil {
+ return err
+ }
javaPaths = &javaLC.Paths
- javaLC.CreateFolders()
- os.Create(filepath.Join(javaPaths.AbsoluteExecutableFilePath))
- os.Create(filepath.Join(javaPaths.AbsoluteSourceFilePath))
+ err = javaLC.CreateFolders()
+ if err != nil {
+ return err
+ }
+ _, err = os.Create(filepath.Join(javaPaths.AbsoluteExecutableFilePath))
+ if err != nil {
+ return err
+ }
+ _, err = os.Create(filepath.Join(javaPaths.AbsoluteSourceFilePath))
+ if err != nil {
+ return err
+ }
- goLC, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_GO, pipelineId, "")
+ goPipelineId := uuid.New()
+ goLC, err = fs_tool.NewLifeCycle(pb.Sdk_SDK_GO, goPipelineId, "")
+ if err != nil {
+ return err
+ }
+ err = goLC.CreateFolders()
+ if err != nil {
+ return err
+ }
goPaths = &goLC.Paths
- scioLC, _ := fs_tool.NewLifeCycle(pb.Sdk_SDK_SCIO, pipelineId, "")
+ scioPipelineId := uuid.New()
+ scioLC, err = fs_tool.NewLifeCycle(pb.Sdk_SDK_SCIO, scioPipelineId, "")
+ if err != nil {
+ return err
+ }
+ err = scioLC.CreateFolders()
+ if err != nil {
+ return err
+ }
scioPaths = &scioLC.Paths
+ _, err = os.Create(filepath.Join(scioPaths.AbsoluteSourceFilePath))
+ if err != nil {
+ return err
+ }
executorConfig := &environment.ExecutorConfig{
CompileCmd: "MOCK_COMPILE_CMD",
@@ -81,11 +136,32 @@ func setup() {
javaSdkEnv = environment.NewBeamEnvs(pb.Sdk_SDK_JAVA, "",
executorConfig, "", 0)
goSdkEnv = environment.NewBeamEnvs(pb.Sdk_SDK_GO, "", executorConfig,
"", 0)
scioSdkEnv = environment.NewBeamEnvs(pb.Sdk_SDK_SCIO, "",
executorConfig, "", 0)
+
+ return nil
}
-func teardown() {
- os.Remove(emptyFolder)
- javaLC.DeleteFolders()
+func teardown() error {
+ err := os.Remove(emptyFolder)
+ if err != nil {
+ return err
+ }
+ err = pythonLC.DeleteFolders()
+ if err != nil {
+ return err
+ }
+ err = javaLC.DeleteFolders()
+ if err != nil {
+ return err
+ }
+ err = goLC.DeleteFolders()
+ if err != nil {
+ return err
+ }
+ err = scioLC.DeleteFolders()
+ if err != nil {
+ return err
+ }
+ return nil
}
func TestValidator(t *testing.T) {
@@ -307,26 +383,36 @@ func TestPreparer(t *testing.T) {
}
func TestCompiler(t *testing.T) {
+ javaSources, err :=
GetFilesFromFolder(javaPaths.AbsoluteSourceFileFolderPath,
fs_tool.JavaSourceFileExtension)
+ if err != nil {
+ t.Errorf("Failed to get Java source files, error = %v", err)
+ }
+
wantJavaExecutor := executors.NewExecutorBuilder().
WithCompiler().
WithCommand(javaSdkEnv.ExecutorConfig.CompileCmd).
WithWorkingDir(javaPaths.AbsoluteBaseFolderPath).
WithArgs(javaSdkEnv.ExecutorConfig.CompileArgs).
-
WithFileName(GetFirstFileFromFolder(javaPaths.AbsoluteSourceFileFolderPath))
+ WithFileNames(javaSources...)
+
+ goSources, err :=
GetFilesFromFolder(goPaths.AbsoluteSourceFileFolderPath,
fs_tool.GoSourceFileExtension)
+ if err != nil {
+ t.Errorf("Failed to get Go source files, error = %v", err)
+ }
wantGoExecutor := executors.NewExecutorBuilder().
WithCompiler().
WithCommand(goSdkEnv.ExecutorConfig.CompileCmd).
WithWorkingDir(goPaths.AbsoluteBaseFolderPath).
WithArgs(goSdkEnv.ExecutorConfig.CompileArgs).
- WithFileName(goPaths.AbsoluteSourceFilePath)
+ WithFileNames(goSources...)
wantScioExecutor := executors.NewExecutorBuilder().
WithCompiler().
WithCommand(scioSdkEnv.ExecutorConfig.CompileCmd).
WithWorkingDir(scioPaths.AbsoluteBaseFolderPath).
WithArgs(scioSdkEnv.ExecutorConfig.CompileArgs).
- WithFileName(scioPaths.AbsoluteSourceFilePath)
+ WithFileNames(scioPaths.AbsoluteSourceFilePath)
type args struct {
paths *fs_tool.LifeCyclePaths
@@ -366,7 +452,10 @@ func TestCompiler(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := Compiler(tt.args.paths, tt.args.sdkEnv)
+ got, err := Compiler(tt.args.paths, tt.args.sdkEnv)
+ if err != nil {
+ t.Errorf("Compiler() error = %v", err)
+ }
if !reflect.DeepEqual(fmt.Sprint(got.Build()),
fmt.Sprint(tt.want.Build())) {
t.Errorf("Compiler() = %v, want %v",
got.Build(), tt.want.Build())
}
@@ -381,20 +470,20 @@ func TestRunnerBuilder(t *testing.T) {
wantPythonExecutor := executors.NewExecutorBuilder().
WithRunner().
- WithExecutableFileName(pythonPaths.AbsoluteExecutableFilePath).
+ WithExecutableFileNames(pythonPaths.AbsoluteExecutableFilePath).
WithWorkingDir(pythonPaths.AbsoluteBaseFolderPath).
WithCommand(pythonSdkEnv.ExecutorConfig.RunCmd).
WithArgs(pythonSdkEnv.ExecutorConfig.RunArgs).
WithPipelineOptions(strings.Split("", " "))
arg := replaceLogPlaceholder(javaPaths, javaSdkEnv.ExecutorConfig)
- javaClassName, err :=
javaPaths.ExecutableName(javaPaths.AbsoluteExecutableFileFolderPath)
+ javaClassName, err :=
javaPaths.FindExecutableName(context.Background(),
javaPaths.AbsoluteExecutableFileFolderPath)
if err != nil {
- panic(err)
+ t.Errorf("Cannot get executable name for Java, error = %v", err)
}
wantJavaExecutor := executors.NewExecutorBuilder().
WithRunner().
- WithExecutableFileName(javaClassName).
+ WithExecutableFileNames(javaClassName).
WithWorkingDir(javaPaths.AbsoluteBaseFolderPath).
WithCommand(javaSdkEnv.ExecutorConfig.RunCmd).
WithArgs(arg).
@@ -404,13 +493,13 @@ func TestRunnerBuilder(t *testing.T) {
WithRunner().
WithWorkingDir(goPaths.AbsoluteBaseFolderPath).
WithCommand(goPaths.AbsoluteExecutableFilePath).
- WithExecutableFileName("").
+ WithExecutableFileNames("").
WithArgs(goSdkEnv.ExecutorConfig.RunArgs).
WithPipelineOptions(strings.Split("", " "))
- scioClassName, err :=
scioPaths.ExecutableName(scioPaths.AbsoluteBaseFolderPath)
+ scioClassName, err :=
scioPaths.FindExecutableName(context.Background(),
scioPaths.AbsoluteSourceFileFolderPath)
if err != nil {
- panic(err)
+ t.Errorf("Cannot get executable name for SCIO, error = %v", err)
}
stringArg := fmt.Sprintf("%s %s %s",
scioSdkEnv.ExecutorConfig.RunArgs[0], scioClassName, "")
wantScioExecutor := executors.NewExecutorBuilder().
@@ -482,7 +571,7 @@ func TestRunnerBuilder(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got, _ := Runner(tt.args.paths,
tt.args.pipelineOptions, tt.args.sdkEnv)
+ got, _ := Runner(context.Background(), tt.args.paths,
tt.args.pipelineOptions, tt.args.sdkEnv)
if tt.want != nil {
if !reflect.DeepEqual(fmt.Sprint(got.Build()),
fmt.Sprint(tt.want.Build())) {
t.Errorf("Runner() got = %v, want %v",
got.Build(), tt.want.Build())
@@ -500,7 +589,7 @@ func TestTestRunner(t *testing.T) {
incorrectJavaPaths := *javaPaths
incorrectJavaPaths.AbsoluteExecutableFileFolderPath = emptyFolder
- className, err :=
javaPaths.ExecutableName(javaPaths.AbsoluteExecutableFileFolderPath)
+ className, err := javaPaths.FindExecutableName(context.Background(),
javaPaths.AbsoluteExecutableFileFolderPath)
if err != nil {
panic(err)
}
@@ -569,7 +658,7 @@ func TestTestRunner(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got, _ := TestRunner(tt.args.paths, tt.args.sdkEnv)
+ got, _ := TestRunner(context.Background(),
tt.args.paths, tt.args.sdkEnv)
if tt.want != nil {
if !reflect.DeepEqual(fmt.Sprint(got.Build()),
fmt.Sprint(tt.want.Build())) {
t.Errorf("TestRunner() got = %v, want
%v", got.Build(), tt.want.Build())
diff --git
a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
index 878d73efb1c..ee1d92dcca6 100644
--- a/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
+++ b/playground/backend/internal/setup_tools/life_cycle/life_cycle_setuper.go
@@ -206,7 +206,7 @@ func prepareSbtFiles(lc *fs_tool.LifeCycle, pipelineFolder
string, workingDir st
absLogFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
logFileName))
absGraphFilePath, _ := filepath.Abs(filepath.Join(absFileFolderPath,
utils.GraphFileName))
projectFolder, _ := filepath.Abs(filepath.Join(pipelineFolder,
scioProjectName))
- executableName := lc.Paths.ExecutableName
+ executableName := lc.Paths.FindExecutableName
err = os.Remove(filepath.Join(absFileFolderPath, defaultExampleInSbt))
if err != nil {
@@ -229,7 +229,7 @@ func prepareSbtFiles(lc *fs_tool.LifeCycle, pipelineFolder
string, workingDir st
AbsoluteLogFilePath: absLogFilePath,
AbsoluteGraphFilePath: absGraphFilePath,
ProjectDir: projectFolder,
- ExecutableName: executableName,
+ FindExecutableName: executableName,
}
return lc, nil
diff --git a/playground/backend/internal/utils/file_utils.go
b/playground/backend/internal/utils/file_utils.go
index d303fe75649..5e0d524363c 100644
--- a/playground/backend/internal/utils/file_utils.go
+++ b/playground/backend/internal/utils/file_utils.go
@@ -171,3 +171,7 @@ func ToSDKFromExt(ext string) pb.Sdk {
return pb.Sdk_SDK_UNSPECIFIED
}
}
+
+func TrimExtension(filename string) string {
+ return strings.TrimSuffix(filename, filepath.Ext(filename))
+}
