angoenka commented on a change in pull request #12042:
URL: https://github.com/apache/beam/pull/12042#discussion_r446830218
##########
File path:
beam-ci/src/main/resources/io/jenkins/plugins/ExecuteBeamPipelineOnDataflowBuilder/config.jelly
##########
@@ -0,0 +1,36 @@
+<?jelly escape-by-default='true'?>
Review comment:
Can we move these files to 'plugins' directory if possible.
##########
File path:
beam-ci/src/main/java/io/jenkins/plugins/ExecuteBeamPipelineOnDataflowBuilder.java
##########
@@ -0,0 +1,173 @@
+package io.jenkins.plugins;
+
+import hudson.Launcher;
+import hudson.Extension;
+import hudson.FilePath;
+import hudson.util.FormValidation;
+import hudson.model.AbstractProject;
+import hudson.model.Run;
+import hudson.model.TaskListener;
+import hudson.tasks.Builder;
+import hudson.tasks.BuildStepDescriptor;
+import jenkins.util.SystemProperties;
+import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.QueryParameter;
+
+import javax.servlet.ServletException;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.*;
+
+import jenkins.tasks.SimpleBuildStep;
+
+public class ExecuteBeamPipelineOnDataflowBuilder extends Builder implements
SimpleBuildStep {
+
+ private final String pathToCreds;
+ private final String pathToMainClass;
+ private final String pipelineOptions;
+ private final String buildReleaseOptions;
+ private boolean useJava; // if false, use Python
+ private boolean useGradle; // if false, use Maven
+
+ @DataBoundConstructor
+ public ExecuteBeamPipelineOnDataflowBuilder(String pathToCreds, String
pathToMainClass, String pipelineOptions, String buildReleaseOptions, boolean
useJava, boolean useGradle) {
+ this.pathToCreds = pathToCreds;
+ this.pathToMainClass = pathToMainClass;
+ this.pipelineOptions = pipelineOptions;
+ this.buildReleaseOptions = buildReleaseOptions;
+ this.useJava = useJava;
+ this.useGradle = useGradle;
+ }
+
+ public String getPathToCreds() { return pathToCreds; }
+
+ public String getPathToMainClass() {
+ return pathToMainClass;
+ }
+
+ public String getPipelineOptions() {
+ return pipelineOptions;
+ }
+
+ public String getBuildReleaseOptions() {
+ return buildReleaseOptions;
+ }
+
+ public boolean getUseJava() {
+ return useJava;
+ }
+
+ public boolean getUseGradle() {
+ return useGradle;
+ }
+
+ /**
+ * Builds and sets the command on the ProcessBuilder depending on
configurations set by the user
+ * */
+ private void buildCommand(Run<?, ?> run, String workspace, ProcessBuilder
processBuilder) {
+ ArrayList<String> command;
+ if (this.useJava) {
+ String pipelineOptions =
this.pipelineOptions.replaceAll("[\\t\\n]+"," ");
+ if (this.useGradle) { // gradle
+ command = new ArrayList<>(Arrays.asList("gradle", "clean",
"execute", "-DmainClass=" + this.pathToMainClass, "-Dexec.args=" +
pipelineOptions));
+ } else { // maven
+ command = new ArrayList<>(Arrays.asList("mvn", "compile",
"exec:java", "-Dexec.mainClass=" + this.pathToMainClass, "-Dexec.args=" +
pipelineOptions));
+ }
+
+ // add pipeline and build release options if included
+ if (!this.buildReleaseOptions.equals("")) {
+ String[] buildReleaseOptions =
this.buildReleaseOptions.split("\\s+"); // split build release options by
whitespace
+ command.addAll(Arrays.asList(buildReleaseOptions)); // add
build release options as separate list elements
+ }
+ // System.out.println(Arrays.toString(command.toArray()));
+ processBuilder.command(command);
+ } else { // python
+ // Get Path to the Bash Script
+ String dir = System.getProperty("user.dir"); // Get the directory
the plugin is located
+ String pathToScript = dir +
"/src/main/java/io/jenkins/plugins/executePythonBeamPipeline.sh";
+
+ // Get Path to the current build directory to create Virtual
Environment in
+ String jobBuildPathDirectory = getJobBuildDirectory(run);
+
+ // Execute Bash Script
+ processBuilder.command(pathToScript, workspace,
jobBuildPathDirectory, this.pathToMainClass, this.pipelineOptions,
this.buildReleaseOptions);
+ }
+ }
+
+ /**
+ * @return absolute path to current build folder
+ * */
+ private String getJobBuildDirectory(Run<?, ?> run) {
+ String jenkinsHome = System.getProperty("JENKINS_HOME");
+ String jobName = run.getFullDisplayName().split(" ")[0];
+ int buildNumber = run.getNumber();
+ return jenkinsHome + "/jobs/" + jobName + "/builds/" + buildNumber;
+ }
+
+ @Override
+ public void perform(Run<?, ?> run, FilePath workspace, Launcher launcher,
TaskListener listener) throws InterruptedException, IOException {
+ ProcessBuilder processBuilder = new ProcessBuilder();
+
+ // see that all configurations are received correctly
+ listener.getLogger().println("path to google app creds : " +
this.pathToCreds);
Review comment:
google app creds -> google cloud credentials
##########
File path:
beam-ci/src/main/resources/io/jenkins/plugins/ExecuteBeamPipelineOnDataflowBuilder/help-buildReleaseOptions.html
##########
@@ -0,0 +1,4 @@
+<div>
+ Include optional build release options. <br/>
+ <b>Maven and Gradle example</b>: -Pdataflow-runner
+</div>
Review comment:
Let's add new lines to all the files.
##########
File path: beam-ci/pom.xml
##########
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.jenkins-ci.plugins</groupId>
+ <artifactId>plugin</artifactId>
+ <version>3.50</version>
+ <relativePath />
+ </parent>
+ <groupId>io.jenkins.plugins</groupId>
+ <artifactId>beam-ci</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>hpi</packaging>
+ <properties>
+ <jenkins.version>2.176.4</jenkins.version>
+ <java.level>8</java.level>
+ </properties>
+ <name>Beam CI</name>
+ <licenses>
+ <license>
+ <name>MIT License</name>
+ <url>https://opensource.org/licenses/MIT</url>
Review comment:
We will need apache as Beam is under the same license
https://www.apache.org/licenses/LICENSE-2.0
##########
File path:
beam-ci/src/main/java/io/jenkins/plugins/ExecuteBeamPipelineOnDataflowBuilder.java
##########
@@ -0,0 +1,173 @@
+package io.jenkins.plugins;
Review comment:
Add the license header to all the new files
https://www.apache.org/legal/src-headers.html#headers
Example
https://github.com/apache/beam/blob/e68397c8cc834bdd62ef2de5f5c1476d9c456b9d/sdks/java/core/src/main/java/org/apache/beam/sdk/function/ThrowingConsumer.java#L1
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
