This is an automated email from the ASF dual-hosted git repository.

chamikara pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new af28351  [BEAM-3217] Jenkins job for HadoopInputFormatIOIT (#4758)
af28351 is described below

commit af28351e896b71579bac3f530544fd5c9a7e9a44
Author: Ɓukasz Gajowy <lukasz.gaj...@gmail.com>
AuthorDate: Mon Mar 12 03:56:57 2018 +0100

    [BEAM-3217] Jenkins job for HadoopInputFormatIOIT (#4758)
---
 .test-infra/jenkins/common_job_properties.groovy   | 48 +++++++++++++++++++++
 ...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 12 ++----
 ...beam_PerformanceTests_HadoopInputFormat.groovy} | 49 ++++++++--------------
 .../jenkins/job_beam_PerformanceTests_JDBC.groovy  | 33 +++++----------
 .../postgres/postgres-service-for-local-dev.yml    | 27 ++++++++++++
 sdks/java/io/hadoop-input-format/pom.xml           |  2 +-
 sdks/java/io/jdbc/pom.xml                          |  2 +-
 7 files changed, 108 insertions(+), 65 deletions(-)

diff --git a/.test-infra/jenkins/common_job_properties.groovy 
b/.test-infra/jenkins/common_job_properties.groovy
index c370dc2..118fe50 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -262,6 +262,31 @@ class common_job_properties {
     return mapToArgString(joinedArgs)
   }
 
+  static def setupKubernetes(def context, def namespace, def 
kubeconfigLocation) {
+    context.steps {
+      shell('gcloud container clusters get-credentials io-datastores 
--zone=us-central1-a --verbosity=debug')
+      shell("cp /home/jenkins/.kube/config ${kubeconfigLocation}")
+
+      shell("kubectl --kubeconfig=${kubeconfigLocation} create namespace 
${namespace}")
+      shell("kubectl --kubeconfig=${kubeconfigLocation} config set-context 
\$(kubectl config current-context) --namespace=${namespace}")
+    }
+  }
+
+  static def cleanupKubernetes(def context, def namespace, def 
kubeconfigLocation) {
+    context.steps {
+      shell("kubectl --kubeconfig=${kubeconfigLocation} delete namespace 
${namespace}")
+      shell("rm ${kubeconfigLocation}")
+    }
+  }
+
+  static String getKubernetesNamespace(def testName) {
+    return "${testName}-${new Date().getTime()}"
+  }
+
+  static String getKubeconfigLocationForNamespace(def namespace) {
+    return '"$WORKSPACE/' + "config-${namespace}" + '"'
+  }
+
   // Adds the standard performance test job steps.
   static def buildPerformanceTest(def context, def argMap) {
     def pkbArgs = genPerformanceArgs(argMap)
@@ -388,4 +413,27 @@ class common_job_properties {
       }
     }
   }
+
+  /**
+   * Transforms pipeline options to a string of format like below:
+   * ["--pipelineOption=123", "--pipelineOption2=abc", ...]
+   *
+   * @param pipelineOptions A map of pipeline options.
+   */
+  static String joinPipelineOptions(Map pipelineOptions) {
+    List<String> pipelineArgList = []
+    pipelineOptions.each({
+      key, value -> pipelineArgList.add("\"--$key=$value\"")
+    })
+    return "[" + pipelineArgList.join(',') + "]"
+  }
+
+
+  /**
+   * Returns absolute path to beam project's files.
+   * @param path A relative path to project resource.
+   */
+  static String makePathAbsolute(String path) {
+    return '"$WORKSPACE/' + path + '"'
+  }
 }
diff --git 
a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy 
b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
index 117ff40..b5558c0 100644
--- a/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_FileBasedIO_IT.groovy
@@ -108,21 +108,15 @@ private void 
create_filebasedio_performance_test_job(testConfiguration) {
                 'commits@beam.apache.org',
                 false)
 
-        def pipelineArgs = [
+        def pipelineOptions = [
                 project        : 'apache-beam-testing',
                 tempRoot       : 'gs://temp-storage-for-perf-tests',
                 filenamePrefix : 
"gs://temp-storage-for-perf-tests/${testConfiguration.jobName}/\${BUILD_ID}/",
         ]
         if (testConfiguration.containsKey('extraPipelineArgs')) {
-            pipelineArgs << testConfiguration.extraPipelineArgs
+            pipelineOptions << testConfiguration.extraPipelineArgs
         }
 
-        def pipelineArgList = []
-        pipelineArgs.each({
-            key, value -> pipelineArgList.add("\"--$key=$value\"")
-        })
-        def pipelineArgsJoined = "[" + pipelineArgList.join(',') + "]"
-
         def argMap = [
                 benchmarks               : 'beam_integration_benchmark',
                 beam_it_timeout          : '1200',
@@ -131,7 +125,7 @@ private void 
create_filebasedio_performance_test_job(testConfiguration) {
                 beam_sdk                 : 'java',
                 beam_it_module           : 'sdks/java/io/file-based-io-tests',
                 beam_it_class            : testConfiguration.itClass,
-                beam_it_options          : pipelineArgsJoined,
+                beam_it_options          : 
common_job_properties.joinPipelineOptions(pipelineOptions),
                 beam_extra_mvn_properties: '["filesystem=gcs"]',
                 bigquery_table           : testConfiguration.bqTable,
         ]
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy 
b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy
similarity index 54%
copy from .test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
copy to .test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy
index 1e5131f..093390d 100644
--- a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_HadoopInputFormat.groovy
@@ -18,8 +18,7 @@
 
 import common_job_properties
 
-// This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_JDBC') {
+job('beam_PerformanceTests_HadoopInputFormat') {
     // Set default Beam job properties.
     common_job_properties.setTopLevelMainJobProperties(delegate)
 
@@ -34,48 +33,36 @@ job('beam_PerformanceTests_JDBC') {
 
     common_job_properties.enablePhraseTriggeringFromPullRequest(
             delegate,
-            'Java JdbcIO Performance Test',
-            'Run Java JdbcIO Performance Test')
+            'Java HadoopInputFormatIO Performance Test',
+            'Run Java HadoopInputFormatIO Performance Test')
 
-    def pipelineArgs = [
+    def pipelineOptions = [
             tempRoot       : 'gs://temp-storage-for-perf-tests',
             project        : 'apache-beam-testing',
             postgresPort   : '5432',
-            numberOfRecords: '5000000'
+            numberOfRecords: '600000'
     ]
 
+    String namespace = 
common_job_properties.getKubernetesNamespace('hadoopinputformatioit')
+    String kubeconfig = 
common_job_properties.getKubeconfigLocationForNamespace(namespace)
+
     def testArgs = [
-            kubeconfig              : '"$HOME/.kube/config"',
-            beam_it_timeout         : '1800',
+            kubeconfig              : kubeconfig,
+            beam_it_timeout         : '1200',
             benchmarks              : 'beam_integration_benchmark',
             beam_it_profile         : 'io-it',
             beam_prebuilt           : 'true',
             beam_sdk                : 'java',
-            beam_it_module          : 'sdks/java/io/jdbc',
-            beam_it_class           : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
-            beam_it_options         : joinPipelineOptions(pipelineArgs),
-            beam_kubernetes_scripts : 
makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml')
-                    + ',' + 
makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
-            beam_options_config_file: 
makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
-            bigquery_table          : 'beam_performance.jdbcioit_pkb_results'
+            beam_it_module          : 'sdks/java/io/hadoop-input-format',
+            beam_it_class           : 
'org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT',
+            beam_it_options         : 
common_job_properties.joinPipelineOptions(pipelineOptions),
+            beam_kubernetes_scripts : 
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
+            beam_options_config_file: 
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
+            bigquery_table          : 
'beam_performance.hadoopinputformatioit_pkb_results'
     ]
 
-    steps {
-        // create .kube/config file for perfkit (if not exists)
-        shell('gcloud container clusters get-credentials io-datastores 
--zone=us-central1-a --verbosity=debug')
-    }
-
+    common_job_properties.setupKubernetes(delegate, namespace, kubeconfig)
     common_job_properties.buildPerformanceTest(delegate, testArgs)
+    common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig)
 }
 
-static String joinPipelineOptions(Map pipelineArgs) {
-    List<String> pipelineArgList = []
-    pipelineArgs.each({
-        key, value -> pipelineArgList.add("\"--$key=$value\"")
-    })
-    return "[" + pipelineArgList.join(',') + "]"
-}
-
-static String makePathAbsolute(String path) {
-    return '"$WORKSPACE/' + path + '"'
-}
\ No newline at end of file
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy 
b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
index 1e5131f..401a910 100644
--- a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
@@ -18,7 +18,6 @@
 
 import common_job_properties
 
-// This job runs the Beam performance tests on PerfKit Benchmarker.
 job('beam_PerformanceTests_JDBC') {
     // Set default Beam job properties.
     common_job_properties.setTopLevelMainJobProperties(delegate)
@@ -37,15 +36,18 @@ job('beam_PerformanceTests_JDBC') {
             'Java JdbcIO Performance Test',
             'Run Java JdbcIO Performance Test')
 
-    def pipelineArgs = [
+    def pipelineOptions = [
             tempRoot       : 'gs://temp-storage-for-perf-tests',
             project        : 'apache-beam-testing',
             postgresPort   : '5432',
             numberOfRecords: '5000000'
     ]
 
+    String namespace = common_job_properties.getKubernetesNamespace('jdbcioit')
+    String kubeconfig = 
common_job_properties.getKubeconfigLocationForNamespace(namespace)
+
     def testArgs = [
-            kubeconfig              : '"$HOME/.kube/config"',
+            kubeconfig              : kubeconfig,
             beam_it_timeout         : '1800',
             benchmarks              : 'beam_integration_benchmark',
             beam_it_profile         : 'io-it',
@@ -53,29 +55,14 @@ job('beam_PerformanceTests_JDBC') {
             beam_sdk                : 'java',
             beam_it_module          : 'sdks/java/io/jdbc',
             beam_it_class           : 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
-            beam_it_options         : joinPipelineOptions(pipelineArgs),
-            beam_kubernetes_scripts : 
makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres.yml')
-                    + ',' + 
makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
-            beam_options_config_file: 
makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
+            beam_it_options         : 
common_job_properties.joinPipelineOptions(pipelineOptions),
+            beam_kubernetes_scripts : 
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml'),
+            beam_options_config_file: 
common_job_properties.makePathAbsolute('src/.test-infra/kubernetes/postgres/pkb-config-local.yml'),
             bigquery_table          : 'beam_performance.jdbcioit_pkb_results'
     ]
 
-    steps {
-        // create .kube/config file for perfkit (if not exists)
-        shell('gcloud container clusters get-credentials io-datastores 
--zone=us-central1-a --verbosity=debug')
-    }
-
+    common_job_properties.setupKubernetes(delegate, namespace, kubeconfig)
     common_job_properties.buildPerformanceTest(delegate, testArgs)
+    common_job_properties.cleanupKubernetes(delegate, namespace, kubeconfig)
 }
 
-static String joinPipelineOptions(Map pipelineArgs) {
-    List<String> pipelineArgList = []
-    pipelineArgs.each({
-        key, value -> pipelineArgList.add("\"--$key=$value\"")
-    })
-    return "[" + pipelineArgList.join(',') + "]"
-}
-
-static String makePathAbsolute(String path) {
-    return '"$WORKSPACE/' + path + '"'
-}
\ No newline at end of file
diff --git a/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml 
b/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml
index 5d2c664..7ba106a 100644
--- a/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml
+++ b/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml
@@ -26,3 +26,30 @@ spec:
   selector:
     name: postgres
   type: LoadBalancer
+
+---
+
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: postgres
+spec:
+  replicas: 1
+  selector:
+    name: postgres
+  template:
+    metadata:
+      name: postgres
+      labels:
+        name: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: postgres
+          env:
+            - name: POSTGRES_PASSWORD
+              value: uuinkks
+            - name: PGDATA
+              value: /var/lib/postgresql/data/pgdata
+          ports:
+            - containerPort: 5432
diff --git a/sdks/java/io/hadoop-input-format/pom.xml 
b/sdks/java/io/hadoop-input-format/pom.xml
index e8c3565..1907deb 100644
--- a/sdks/java/io/hadoop-input-format/pom.xml
+++ b/sdks/java/io/hadoop-input-format/pom.xml
@@ -253,7 +253,7 @@
                 
<argument>-beam_it_module=sdks/java/io/hadoop-input-format</argument>
                 
<argument>-beam_it_class=org.apache.beam.sdk.io.hadoop.inputformat.HadoopInputFormatIOIT</argument>
                 
<argument>-beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml</argument>
-                
<argument>-beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml</argument>
+                
<argument>-beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml</argument>
                 <!-- arguments typically defined by user -->
                 
<argument>-beam_it_options=${integrationTestPipelineOptions}</argument>
               </arguments>
diff --git a/sdks/java/io/jdbc/pom.xml b/sdks/java/io/jdbc/pom.xml
index 4d52746..9151cfb 100644
--- a/sdks/java/io/jdbc/pom.xml
+++ b/sdks/java/io/jdbc/pom.xml
@@ -220,7 +220,7 @@
                 <argument>${pkbBeamRunnerOption}</argument>
                 <!-- specific to this IO -->
                 
<argument>-beam_options_config_file=${beamRootProjectDir}/.test-infra/kubernetes/postgres/pkb-config-local.yml</argument>
-                
<argument>-beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres.yml,${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml</argument>
+                
<argument>-beam_kubernetes_scripts=${beamRootProjectDir}/.test-infra/kubernetes/postgres/postgres-service-for-local-dev.yml</argument>
                 <argument>-beam_it_module=sdks/java/io/jdbc</argument>
                 
<argument>-beam_it_class=org.apache.beam.sdk.io.jdbc.JdbcIOIT</argument>
                 <!-- arguments typically defined by user -->

-- 
To stop receiving notification emails like this one, please contact
chamik...@apache.org.

Reply via email to