This is an automated email from the ASF dual-hosted git repository. mxm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 102a972 [BEAM-4130] Build Docker image for Flink's JobServer new d44a54d Merge pull request #6238 from mxm/job-server-startup 102a972 is described below commit 102a97293af08db5d639406879c87c4f3a6ddc40 Author: Maximilian Michels <m...@apache.org> AuthorDate: Thu Aug 16 14:55:56 2018 +0200 [BEAM-4130] Build Docker image for Flink's JobServer This adds a new Gradle module flink-job-server-container which builds a docker image during the `docker` task. The image contains the FlinkJobServerDriver which is the entry point for submitting Beam pipelines to the cluster. The image can then be used to spawn a JobServer container when executing a Beam pipeline, i.e. `p.run()`. The SDKs (Java/Python/Go) need to be updated to either spawn up a container or use the address of a remote JobServer. --- runners/flink/job-server-container/Dockerfile | 26 +++++++++++ runners/flink/job-server-container/build.gradle | 54 ++++++++++++++++++++++ .../flink/job-server-container/flink-job-server.sh | 29 ++++++++++++ settings.gradle | 2 + 4 files changed, 111 insertions(+) diff --git a/runners/flink/job-server-container/Dockerfile b/runners/flink/job-server-container/Dockerfile new file mode 100644 index 0000000..a9aff21 --- /dev/null +++ b/runners/flink/job-server-container/Dockerfile @@ -0,0 +1,26 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM openjdk:8 +MAINTAINER "Apache Beam <d...@beam.apache.org>" + +ADD target/beam-runners-flink_2.11-job-server.jar /opt/apache/beam/jars/ +ADD target/flink-job-server.sh /opt/apache/beam/ + +WORKDIR /opt/apache/beam +ENTRYPOINT ["./flink-job-server.sh"] diff --git a/runners/flink/job-server-container/build.gradle b/runners/flink/job-server-container/build.gradle new file mode 100644 index 0000000..4d5f533 --- /dev/null +++ b/runners/flink/job-server-container/build.gradle @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Build a Docker image to bootstrap FlinkJobServerDriver which requires a Java environment. + * Alternatively, it can also be bootstrapped through :beam-runners-flink_2.11-job-server:runShadow + * or by directly running the generated JAR file. + */ + +apply plugin: org.apache.beam.gradle.BeamModulePlugin +applyDockerNature() + +description = "Apache Beam :: Runners :: Flink :: Job Server :: Container" + +configurations { + dockerDependency +} + +dependencies { + dockerDependency project(path: ":beam-runners-flink_2.11-job-server", configuration: "shadow") +} + +task copyDockerfileDependencies(type: Copy) { + // Required Jars + from configurations.dockerDependency + rename 'beam-runners-flink_2.11-job-server.*.jar', 'beam-runners-flink_2.11-job-server.jar' + into "build/target" + // Entry script + from file("./flink-job-server.sh") + into "build/target" +} + +docker { + name containerImageName(name: "flink-job-server") + files "./build/" +} + +// Ensure that we build the required resources and copy and file dependencies from related projects +dockerPrepare.dependsOn copyDockerfileDependencies diff --git a/runners/flink/job-server-container/flink-job-server.sh b/runners/flink/job-server-container/flink-job-server.sh new file mode 100755 index 0000000..399e5e4 --- /dev/null +++ b/runners/flink/job-server-container/flink-job-server.sh @@ -0,0 +1,29 @@ +#!/bin/sh +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +### Just a simple script to bootstrap the FlinkJobServerDriver +### For the environment, see the Dockerfile + +# The following (forking to the background, then waiting) enables to use CTRL+C to kill the container. +# We're PID 1 which doesn't handle signals. By forking the Java process to the background, +# a PID > 1 is created which handles signals. After the command shuts down, the script and +# thus the container will also exit. + +java -cp "jars/*" org.apache.beam.runners.flink.FlinkJobServerDriver "$@" & +wait diff --git a/settings.gradle b/settings.gradle index 01ea889..a606b17 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,6 +42,8 @@ include "beam-runners-flink_2.11" project(":beam-runners-flink_2.11").dir = file("runners/flink") include "beam-runners-flink_2.11-job-server" project(":beam-runners-flink_2.11-job-server").dir = file("runners/flink/job-server") +include "beam-runners-flink_2.11-job-server-container" +project(":beam-runners-flink_2.11-job-server-container").dir = file("runners/flink/job-server-container") include "beam-runners-gcp-gcemd" project(":beam-runners-gcp-gcemd").dir = file("runners/gcp/gcemd") include "beam-runners-gcp-gcsproxy"