[beam] branch master updated: Merge pull request #14019 from [BEAM-11791] Fixing FnApiRunner Microbenchmarks to export to influxDB

[pabloem]

This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new edae900  Merge pull request #14019 from [BEAM-11791] Fixing 
FnApiRunner Microbenchmarks to export to influxDB
edae900 is described below

commit edae900d0e3b68e80f0a4ff7724f8f081be0af6e
Author: Pablo <pabl...@users.noreply.github.com>
AuthorDate: Fri Feb 19 09:54:08 2021 -0800

    Merge pull request #14019 from [BEAM-11791] Fixing FnApiRunner 
Microbenchmarks to export to influxDB
    
    * [BEAM-11791] Fixing FnApiRunner Microbenchmarks to export to influxDB
    
    * fixup
---
 .../job_LoadTests_FnApiRunner_Python.groovy        | 24 +++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/.test-infra/jenkins/job_LoadTests_FnApiRunner_Python.groovy 
b/.test-infra/jenkins/job_LoadTests_FnApiRunner_Python.groovy
index 69c42f8..befc04a 100644
--- a/.test-infra/jenkins/job_LoadTests_FnApiRunner_Python.groovy
+++ b/.test-infra/jenkins/job_LoadTests_FnApiRunner_Python.groovy
@@ -16,8 +16,10 @@
  * limitations under the License.
  */
 
+import CommonJobProperties as commonJobProperties
 import LoadTestsBuilder as loadTestsBuilder
 import PhraseTriggeringPostCommitBuilder
+import InfluxDBCredentialsHelper
 
 def now = new Date().format("MMddHHmmss", TimeZone.getTimeZone('UTC'))
 
@@ -29,7 +31,7 @@ def loadTestConfigurations = { datasetName ->
       runner         : CommonTestProperties.Runner.DIRECT,
       pipelineOptions: [
         publish_to_big_query: true,
-        influx_measurement  : 'python_microbenchmarks',
+        influx_measurement  : 'python_direct_microbenchmarks',
         project             : 'apache-beam-testing',
         metrics_dataset     : datasetName,
         metrics_table       : 'python_direct_microbenchmarks',
@@ -37,6 +39,17 @@ def loadTestConfigurations = { datasetName ->
       ]
     ],
   ]
+  .each { test -> test.pipelineOptions.putAll(additionalPipelineArgs) }
+}
+
+def loadTestJob = { scope, triggeringContext ->
+  scope.description("Runs Python FnApiRunner Microbenchmark")
+  commonJobProperties.setTopLevelMainJobProperties(scope, 'master', 120)
+
+  def datasetName = loadTestsBuilder.getBigQueryDataset('load_test', 
triggeringContext)
+  for (testConfiguration in loadTestConfigurations(datasetName)) {
+    loadTestsBuilder.loadTest(scope, testConfiguration.title, 
testConfiguration.runner, CommonTestProperties.SDK.PYTHON, 
testConfiguration.pipelineOptions, testConfiguration.test)
+  }
 }
 
 PhraseTriggeringPostCommitBuilder.postCommitJob(
@@ -45,9 +58,8 @@ PhraseTriggeringPostCommitBuilder.postCommitJob(
     'Python Load Tests FnApiRunner Microbenchmark',
     this
     ) {
-      def datasetName = loadTestsBuilder.getBigQueryDataset('load_test', 
CommonTestProperties.TriggeringContext.PR)
-      loadTestsBuilder.loadTests(delegate, CommonTestProperties.SDK.PYTHON,
-          loadTestConfigurations(datasetName), "MicroBenchmarks", "batch")
+      additionalPipelineArgs = [:]
+      loadTestJob(delegate, CommonTestProperties.TriggeringContext.PR)
     }
 
 
@@ -57,8 +69,6 @@ 
CronJobBuilder.cronJob('beam_Python_LoadTests_FnApiRunner_Microbenchmark', 'H */
     influx_db_name: InfluxDBCredentialsHelper.InfluxDBDatabaseName,
     influx_hostname: InfluxDBCredentialsHelper.InfluxDBHostUrl,
   ]
-  def datasetName = loadTestsBuilder.getBigQueryDataset('load_test', 
CommonTestProperties.TriggeringContext.POST_COMMIT)
-  loadTestsBuilder.loadTests(delegate, CommonTestProperties.SDK.PYTHON,
-      loadTestConfigurations(datasetName), "MicroBenchmarks", "batch")
+  loadTestJob(delegate, CommonTestProperties.TriggeringContext.POST_COMMIT)
 }