This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 2a2385b BIGTOP-3382. Upgrade Flink to 1.11.3 (#730)
2a2385b is described below
commit 2a2385b0042e5ee668481f466df1b7b6e58e1e40
Author: okumurays <[email protected]>
AuthorDate: Tue Feb 9 12:57:12 2021 +0900
BIGTOP-3382. Upgrade Flink to 1.11.3 (#730)
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 9 ++++++---
.../puppet/modules/flink/manifests/init.pp | 4 ++--
.../puppet/modules/flink/templates/flink-conf.yaml | 23 ++++++----------------
.../src/common/flink/flink-jobmanager.svc | 12 ++++++-----
.../src/common/flink/flink-taskmanager.svc | 12 ++++++-----
bigtop-packages/src/common/flink/install_flink.sh | 2 +-
.../flink/patch0-fix-ApplicationReport-api.diff | 12 -----------
bigtop-tests/smoke-tests/flink/TestFlink.groovy | 2 +-
bigtop.bom | 2 +-
9 files changed, 31 insertions(+), 47 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 0d6229d..4956414 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -203,9 +203,12 @@ zeppelin::server::hiveserver2_password:
"%{hiera('bigtop::hiveserver2_password')
# Flink
flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}"
flink::common::jobmanager_port: "6123"
-
-flink::common::ui_port: "8081"
-flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}"
+flink::common::jobmanager_memory: "1600m"
+flink::common::taskmanager_memory: "1728m"
+flink::common::taskmanager_number_of_taskslots: "1"
+flink::common::parallelism_default: "1"
+flink::common::jobmanager_failover_strategy: "region"
+flink::common::rest_port: "8081"
# GPDB
# The first element is FQDN for master node and the succeeding ones are for
segment nodes.
diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp
b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
index e31d182..3538bca 100644
--- a/bigtop-deploy/puppet/modules/flink/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
@@ -21,8 +21,8 @@ class flink {
}
}
- class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) {
- # make sure flink is installed
+ class common($jobmanager_host, $jobmanager_port, $jobmanager_memory,
$taskmanager_memory, $taskmanager_number_of_taskslots, $parallelism_default,
$jobmanager_failover_strategy, $rest_port) {
+ # make sure flink is installed
package { "flink":
ensure => latest
}
diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
index cb9c1f4..ead8d08 100644
--- a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
+++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
@@ -20,20 +20,9 @@
# Configuration values managed by puppet:
jobmanager.rpc.address: <%= @jobmanager_host %>
jobmanager.rpc.port: <%= @jobmanager_port %>
-jobmanager.web.port: <%= @ui_port %>
-
-<% if defined?(storage_dirs) %>
-taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %>
-<% end %>
-
-# Starting from Flink v1.5, there is a rewrite of Flink’s deployment and
process
-# model (internally known as FLIP-6). A new option 'mode' was introduced to
-# indicate which executions to use. However, the way Bigtop puppet deploys
-# 'flink-jobmanager' and 'flink-taskmanager', ie. calling 'flink-daemon.sh
-# jobmanager' and 'flink-daemon.sh taskmanager', is legacy. Without setting
-# mode to 'legacy', flink's built-in exmaples fail with error
-# "JobSubmissionException: Failed to submit JobGraph"
-mode: legacy
-
-# For performance reasons its highly recommended to allocate as much memory to
the
-# Flink TaskManager as possible by setting 'taskmanager.heap.mb'.
+jobmanager.memory.process.size: <%= @jobmanager_memory %>
+taskmanager.memory.process.size: <%= @taskmanager_memory %>
+taskmanager.numberOfTaskSlots: <%= @taskmanager_number_of_taskslots %>
+parallelism.default: <%= @parallelism_default %>
+jobmanager.execution.failover-strategy: <%= @jobmanager_failover_strategy %>
+rest.port: <%=@rest_port %>
diff --git a/bigtop-packages/src/common/flink/flink-jobmanager.svc
b/bigtop-packages/src/common/flink/flink-jobmanager.svc
index ac7106e..7e74e80 100644
--- a/bigtop-packages/src/common/flink/flink-jobmanager.svc
+++ b/bigtop-packages/src/common/flink/flink-jobmanager.svc
@@ -16,12 +16,12 @@
TYPE="jobmanager"
DAEMON="flink-${TYPE}"
DESC="Flink ${TYPE}"
-EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
+EXEC_PATH="/usr/lib/flink/bin/jobmanager.sh"
WORKING_DIR="/var/lib/flink"
DAEMON_FLAGS=jobmanager
CONF_DIR="/etc/flink/conf"
SVC_USER="flink"
-PIDFILE="/var/run/flink/flink-$SVC_USER-jobmanager.pid"
+PIDFILE="/var/run/flink/flink-$SVC_USER-standalonesession.pid"
EXEC_MODE="cluster"
generate_start() {
@@ -31,6 +31,10 @@ start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "
+ if command -v hadoop > /dev/null; then
+ export HADOOP_CLASSPATH=$(hadoop classpath)
+ fi
+ export FLINK_CONF_DIR="${CONF_DIR}"
export FLINK_LOG_DIR="/var/log/flink"
export FLINK_PID_DIR="/var/run/flink"
mkdir -p $FLINK_PID_DIR
@@ -44,9 +48,7 @@ start() {
LOG_FILE=/var/log/flink/${DAEMON}.out
# Flink will set the pid file
- runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir $CONF_DIR --executionMode
cluster \
- > $LOG_FILE 2>&1 & "
+ runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 ${EXEC_PATH} start >
$LOG_FILE 2>&1 & "
sleep 3
diff --git a/bigtop-packages/src/common/flink/flink-taskmanager.svc
b/bigtop-packages/src/common/flink/flink-taskmanager.svc
index 5bbd616..612148c 100644
--- a/bigtop-packages/src/common/flink/flink-taskmanager.svc
+++ b/bigtop-packages/src/common/flink/flink-taskmanager.svc
@@ -16,12 +16,12 @@
TYPE="taskmanager"
DAEMON="flink-${TYPE}"
DESC="Flink ${TYPE}"
-EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
+EXEC_PATH="/usr/lib/flink/bin/taskmanager.sh"
WORKING_DIR="/var/lib/flink"
DAEMON_FLAGS="taskmanager"
SVC_USER="flink"
CONF_DIR="/etc/flink/conf"
-PIDFILE="/var/run/flink/flink-$SVC_USER-taskmanager.pid"
+PIDFILE="/var/run/flink/flink-$SVC_USER-taskexecutor.pid"
generate_start() {
@@ -31,6 +31,10 @@ start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "
+ if command -v hadoop > /dev/null; then
+ export HADOOP_CLASSPATH=$(hadoop classpath)
+ fi
+ export FLINK_CONF_DIR="${CONF_DIR}"
export FLINK_LOG_DIR="/var/log/flink"
export FLINK_PID_DIR="/var/run/flink"
mkdir -p $FLINK_PID_DIR
@@ -44,9 +48,7 @@ start() {
LOG_FILE=/var/log/flink/${DAEMON}.out
- runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir ${CONF_DIR} \
- > $LOG_FILE 2>&1 & "
+ runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 ${EXEC_PATH} start >
$LOG_FILE 2>&1 & "
sleep 3
diff --git a/bigtop-packages/src/common/flink/install_flink.sh
b/bigtop-packages/src/common/flink/install_flink.sh
index 6baa669..ac8e5b6 100644
--- a/bigtop-packages/src/common/flink/install_flink.sh
+++ b/bigtop-packages/src/common/flink/install_flink.sh
@@ -120,7 +120,7 @@ ln -s /etc/flink/conf $PREFIX/$LIB_DIR/conf
cp -ra ${BUILD_DIR}/examples/* $PREFIX/${LIB_DIR}/examples/
-cp ${BUILD_DIR}/{LICENSE,NOTICE,README.txt} ${PREFIX}/${LIB_DIR}/
+cp ${BUILD_DIR}/{LICENSE,README.txt} ${PREFIX}/${LIB_DIR}/
# Copy in the /usr/bin/flink wrapper
install -d -m 0755 $PREFIX/$BIN_DIR
diff --git
a/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
b/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
deleted file mode 100644
index c76d3c3..0000000
--- a/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git
a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
-index 8ee08db..fcd56d0 100644
----
a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
-+++
b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
-@@ -98,6 +98,7 @@ public class AbstractYarnClusterTest extends TestLogger {
- yarnApplicationState,
- null,
- null,
-+ 0,
- 1L,
- 2L,
- finalApplicationStatus,
diff --git a/bigtop-tests/smoke-tests/flink/TestFlink.groovy
b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
index b35d3cb..dba11aa 100644
--- a/bigtop-tests/smoke-tests/flink/TestFlink.groovy
+++ b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
@@ -59,7 +59,7 @@ class TestFlink {
// read JM address and port from conf
execCommand("awk '{if(/jobmanager.rpc.address:/) print \$2}' < "+
config_file);
final String jmHost = sh.out.join('\n');
- execCommand("awk '{if(/jobmanager.web.port:/) print \$2}' < "+config_file);
+ execCommand("awk '{if(/rest.port:/) print \$2}' < "+config_file);
final String webPort = sh.out.join('\n');
// check web API
execCommand("curl http://"+jmHost+":"+webPort+"/config";);
diff --git a/bigtop.bom b/bigtop.bom
index 760452b..c64a85d 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -287,7 +287,7 @@ bigtop {
'flink' {
name = 'flink'
relNotes = 'Apache Flink'
- version { base = '1.6.4'; pkg = base; release = 1 }
+ version { base = '1.11.3'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}-src.tgz" }
url { download_path = "/$name/$name-${version.base}"
