[
https://issues.apache.org/jira/browse/BEAM-5419?focusedWorklogId=173152&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-173152
]
ASF GitHub Bot logged work on BEAM-5419:
----------------------------------------
Author: ASF GitHub Bot
Created on: 08/Dec/18 02:24
Start Date: 08/Dec/18 02:24
Worklog Time Spent: 10m
Work Description: angoenka commented on a change in pull request #7229:
[BEAM-5419] Add Flink multi-version builds
URL: https://github.com/apache/beam/pull/7229#discussion_r239989799
##########
File path: runners/flink/job-server/build.gradle
##########
@@ -15,100 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import groovy.json.JsonOutput
-apply plugin: org.apache.beam.gradle.BeamModulePlugin
-applyJavaNature(
- validateShadowJar: false,
- shadowClosure: {
- append "reference.conf"
- },
-)
+// Set the version of all Flink-related dependencies here.
+ext.flink_version = '1.5.5'
-description = "Apache Beam :: Runners :: Flink :: Job Server"
-
-apply plugin: "application"
-
-mainClassName = "org.apache.beam.runners.flink.FlinkJobServerDriver"
-
-configurations {
- validatesPortableRunner
-}
-
-configurations.all {
- // replace commons logging with the jcl-over-slf4j bridge
- exclude group: "commons-logging", module: "commons-logging"
-}
-
-dependencies {
- compile project(path: ":beam-runners-flink_2.11", configuration: "shadow")
- compile group: "org.slf4j", name: "jcl-over-slf4j", version:
dependencies.create(project.library.java.slf4j_api).getVersion()
- validatesPortableRunner project(path: ":beam-runners-flink_2.11",
configuration: "shadowTest")
- validatesPortableRunner project(path: ":beam-sdks-java-core", configuration:
"shadowTest")
- validatesPortableRunner project(path: ":beam-runners-core-java",
configuration: "shadowTest")
- validatesPortableRunner project(path: ":beam-runners-reference-java",
configuration: "shadowTest")
- compile project(path:
":beam-sdks-java-extensions-google-cloud-platform-core", configuration:
"shadow")
- compile library.java.slf4j_simple
-// TODO: Enable AWS and HDFS file system.
-}
-
-// NOTE: runShadow must be used in order to run the job server. The standard
run
-// task will not work because the flink runner classes only exist in the shadow
-// jar.
-runShadow {
- args = []
- if (project.hasProperty('jobHost'))
- args += ["--job-host=${project.property('jobHost')}"]
- if (project.hasProperty('artifactsDir'))
- args += ["--artifacts-dir=${project.property('artifactsDir')}"]
- if (project.hasProperty('cleanArtifactsPerJob'))
- args += ["--clean-artifacts-per-job"]
- if (project.hasProperty('flinkMasterUrl'))
- args += ["--flink-master-url=${project.property('flinkMasterUrl')}"]
- if (project.hasProperty('flinkConfDir'))
- args += ["--flink-conf-dir=${project.property('flinkConfDir')}"]
-
- // Enable remote debugging.
- jvmArgs = ["-Xdebug",
"-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"]
- if (project.hasProperty("logLevel"))
- jvmArgs +=
["-Dorg.slf4j.simpleLogger.defaultLogLevel=${project.property('logLevel')}"]
-}
-
-def portableValidatesRunnerTask(String name, Boolean streaming) {
- createPortableValidatesRunnerTask(
- name: "validatesPortableRunner${name}",
- jobServerDriver:
"org.apache.beam.runners.flink.FlinkJobServerDriver",
- jobServerConfig:
"--clean-artifacts-per-job,--job-host=localhost,--job-port=0,--artifact-port=0",
- testClasspathConfiguration: configurations.validatesPortableRunner,
- parallelism: 2,
- pipelineOpts: streaming ? ["--streaming"] : [],
- testCategories: {
- includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
- excludeCategories
'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders'
- excludeCategories
'org.apache.beam.sdk.testing.LargeKeys$Above100MB'
- excludeCategories
'org.apache.beam.sdk.testing.UsesAttemptedMetrics'
- excludeCategories
'org.apache.beam.sdk.testing.UsesCommittedMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesCounterMetrics'
- excludeCategories
'org.apache.beam.sdk.testing.UsesCustomWindowMerging'
- excludeCategories
'org.apache.beam.sdk.testing.UsesDistributionMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage'
- excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
- excludeCategories 'org.apache.beam.sdk.testing.UsesMapState'
- excludeCategories 'org.apache.beam.sdk.testing.UsesSetState'
- excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
- //SplitableDoFnTests
- excludeCategories
'org.apache.beam.sdk.testing.UsesBoundedSplittableParDo'
- excludeCategories
'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs'
- excludeCategories
'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
- },
- )
-}
-
-project.ext.validatesPortableRunnerBatch =
portableValidatesRunnerTask("Batch", false)
-project.ext.validatesPortableRunnerStreaming =
portableValidatesRunnerTask("Streaming", true)
-
-task validatesPortableRunner() {
- dependsOn validatesPortableRunnerBatch
- dependsOn validatesPortableRunnerStreaming
-}
+// Load the main build script which contains all build logic.
+// It is only parameterized by the $flink_version property.
+apply from: 'flink_job_server.gradle'
Review comment:
nit: new line
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 173152)
Time Spent: 1.5h (was: 1h 20m)
> Build multiple versions of the Flink Runner against different Flink versions
> ----------------------------------------------------------------------------
>
> Key: BEAM-5419
> URL: https://issues.apache.org/jira/browse/BEAM-5419
> Project: Beam
> Issue Type: New Feature
> Components: build-system, runner-flink
> Reporter: Maximilian Michels
> Assignee: Maximilian Michels
> Priority: Major
> Fix For: 2.10.0
>
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> Following up on a discussion on the mailing list.
> We want to keep the Flink version stable across different versions to avoid
> upgrade pain for long-term users. At the same time, there are users out there
> with newer Flink clusters and developers also want to utilize new Flink
> features.
> It would be great to build multiple versions of the Flink Runner against
> different Flink versions.
> When the upgrade is as simple as changing the version property in the build
> script, this should be pretty straight-forward. If not, having a "base
> version" and applying a patch during the build could be an option. We should
> avoid duplicating any Runner code.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
