[
https://issues.apache.org/jira/browse/BEAM-4774?focusedWorklogId=124021&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124021
]
ASF GitHub Bot logged work on BEAM-4774:
----------------------------------------
Author: ASF GitHub Bot
Created on: 17/Jul/18 09:01
Start Date: 17/Jul/18 09:01
Worklog Time Spent: 10m
Work Description: echauchot closed pull request #5947: [BEAM-4774]
Nexmark SQL Postcommits
URL: https://github.com/apache/beam/pull/5947
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
index 70e5bc6b47c..ccf101fc1b2 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
@@ -65,5 +65,39 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--enforceEncodability=true',
'--enforceImmutability=true"'].join(' '))
}
+ shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING DIRECT RUNNER ***')
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-sdks-java-nexmark:run')
+ commonJobProperties.setGradleSwitches(delegate)
+ switches('-Pnexmark.runner=":beam-runners-direct-java"' +
+ ' -Pnexmark.args="' +
+ [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
+ '--runner=DirectRunner',
+ '--queryLanguage=sql',
+ '--streaming=false',
+ '--suite=SMOKE',
+ '--manageResources=false',
+ '--monitorJobs=true',
+ '--enforceEncodability=true',
+ '--enforceImmutability=true"'].join(' '))
+ }
+ shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING DIRECT RUNNER ***')
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-sdks-java-nexmark:run')
+ commonJobProperties.setGradleSwitches(delegate)
+ switches('-Pnexmark.runner=":beam-runners-direct-java"' +
+ ' -Pnexmark.args="' +
+ [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
+ '--runner=DirectRunner',
+ '--queryLanguage=sql',
+ '--streaming=true',
+ '--suite=SMOKE',
+ '--manageResources=false',
+ '--monitorJobs=true',
+ '--enforceEncodability=true',
+ '--enforceImmutability=true"'].join(' '))
+ }
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
index 40c3f0e8f68..736c7d8b11f 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
@@ -65,5 +65,39 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--monitorJobs=true',
'--flinkMaster=local"'].join(' '))
}
+ shell('echo *** RUN NEXMARK IN SQL BATCH MODE USING FLINK RUNNER ***')
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-sdks-java-nexmark:run')
+ commonJobProperties.setGradleSwitches(delegate)
+ switches('-Pnexmark.runner=":beam-runners-flink_2.11"' +
+ ' -Pnexmark.args="' +
+ [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
+ '--runner=FlinkRunner',
+ '--queryLanguage=sql',
+ '--streaming=false',
+ '--suite=SMOKE',
+ '--streamTimeout=60' ,
+ '--manageResources=false',
+ '--monitorJobs=true',
+ '--flinkMaster=local"'].join(' '))
+ }
+ shell('echo *** RUN NEXMARK IN SQL STREAMING MODE USING FLINK RUNNER ***')
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-sdks-java-nexmark:run')
+ commonJobProperties.setGradleSwitches(delegate)
+ switches('-Pnexmark.runner=":beam-runners-flink_2.11"' +
+ ' -Pnexmark.args="' +
+ [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
+ '--runner=FlinkRunner',
+ '--queryLanguage=sql',
+ '--streaming=true',
+ '--suite=SMOKE',
+ '--streamTimeout=60' ,
+ '--manageResources=false',
+ '--monitorJobs=true',
+ '--flinkMaster=local"'].join(' '))
+ }
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
index b27e00a3fc4..e4c5363994f 100644
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
@@ -48,5 +48,21 @@
NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_
'--manageResources=false',
'--monitorJobs=true'].join(' '))
}
+ shell('echo *** RUN NEXMARK SQL IN BATCH MODE USING SPARK RUNNER ***')
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-sdks-java-nexmark:run')
+ commonJobProperties.setGradleSwitches(delegate)
+ switches('-Pnexmark.runner=":beam-runners-spark"' +
+ ' -Pnexmark.args="' +
+ [NexmarkBigqueryProperties.nexmarkBigQueryArgs,
+ '--runner=SparkRunner',
+ '--queryLanguage=sql',
+ '--streaming=false',
+ '--suite=SMOKE',
+ '--streamTimeout=60' ,
+ '--manageResources=false',
+ '--monitorJobs=true'].join(' '))
+ }
}
}
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
index 350152798ea..537ddd22985 100644
--- a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
+++ b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/Main.java
@@ -151,7 +151,11 @@ public NexmarkPerf decode(InputStream inStream)
new
TableFieldSchema().setName("eventsPerSec").setType("FLOAT"),
new
TableFieldSchema().setName("numResults").setType("INTEGER")));
- String tableSpec = NexmarkUtils.tableSpec(options, "{query}", 0L, null);
+ String queryName = "{query}";
+ if (options.getQueryLanguage() != null) {
+ queryName = queryName + "_" + options.getQueryLanguage();
+ }
+ final String tableSpec = NexmarkUtils.tableSpec(options, queryName, 0L,
null);
SerializableFunction<
ValueInSingleWindow<KV<NexmarkConfiguration, NexmarkPerf>>,
TableDestination>
tableFunction =
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
index bbb7496bf8c..99d79c1bb5c 100644
---
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
+++
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
@@ -1210,6 +1210,11 @@ public NexmarkPerf run(NexmarkConfiguration
runConfiguration) throws IOException
}
NexmarkQuery query = getNexmarkQuery();
+ if (query == null) {
+ NexmarkUtils.console("skipping since configuration is not
implemented");
+ return null;
+ }
+
queryName = query.getName();
NexmarkQueryModel model = getNexmarkQueryModel();
@@ -1297,8 +1302,7 @@ private NexmarkQuery getNexmarkQuery() {
List<NexmarkQuery> queries = createQueries();
if (configuration.query >= queries.size()) {
- throw new UnsupportedOperationException(
- "Query " + options.getQuery() + " is not implemented yet");
+ return null;
}
return queries.get(configuration.query);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 124021)
Time Spent: 1h 50m (was: 1h 40m)
> Intergrate Nexmark SQL with Perfkit
> -----------------------------------
>
> Key: BEAM-4774
> URL: https://issues.apache.org/jira/browse/BEAM-4774
> Project: Beam
> Issue Type: New Feature
> Components: dsl-sql
> Reporter: Andrew Pilloud
> Assignee: Andrew Pilloud
> Priority: Major
> Time Spent: 1h 50m
> Remaining Estimate: 0h
>
> We should publish to a dashboard like pure JAVA does. See BEAM-4283.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
