This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 3f8e47d Revert "BIGTOP-3382. Upgrade Flink to 1.11.1 (#665)"
3f8e47d is described below
commit 3f8e47d97ff4dc929765116970ebe604febfe97a
Author: Kengo Seki <[email protected]>
AuthorDate: Mon Sep 7 00:09:13 2020 +0900
Revert "BIGTOP-3382. Upgrade Flink to 1.11.1 (#665)"
This reverts commit 6f920a14401a65b072a31d5f935b3156840c6c28.
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 9 +++------
.../puppet/modules/flink/manifests/init.pp | 2 +-
.../puppet/modules/flink/templates/flink-conf.yaml | 23 ++++++++++++++++------
.../src/common/flink/flink-jobmanager.svc | 12 +++++------
.../src/common/flink/flink-taskmanager.svc | 12 +++++------
bigtop-packages/src/common/flink/install_flink.sh | 2 +-
.../flink/patch0-fix-ApplicationReport-api.diff | 12 +++++++++++
bigtop-tests/smoke-tests/flink/TestFlink.groovy | 2 +-
bigtop.bom | 2 +-
9 files changed, 46 insertions(+), 30 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 1f85244..d73b929 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -203,12 +203,9 @@ zeppelin::server::hiveserver2_password:
"%{hiera('bigtop::hiveserver2_password')
# Flink
flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}"
flink::common::jobmanager_port: "6123"
-flink::common::jobmanager_memory: "1600m"
-flink::common::taskmanager_memory: "1728m"
-flink::common::taskmanager_number_of_taskslots: "1"
-flink::common::parallelism_default: "1"
-flink::common::jobmanager_failover_strategy: "region"
-flink::common::rest_port: "8081"
+
+flink::common::ui_port: "8081"
+flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}"
gpdb::common::nodes: ["master_node_fqdn", "segment_node_fqdn"]
gpdb::common::gp_home: "/usr/lib/gpdb"
diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp
b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
index c1e6205..e31d182 100644
--- a/bigtop-deploy/puppet/modules/flink/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp
@@ -21,7 +21,7 @@ class flink {
}
}
- class common($jobmanager_host, $jobmanager_port, $jobmanager_memory,
$taskmanager_memory, $taskmanager_number_of_taskslots, $parallelism_default,
$jobmanager_failover_strategy, $rest_port) {
+ class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) {
# make sure flink is installed
package { "flink":
ensure => latest
diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
index ead8d08..cb9c1f4 100644
--- a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
+++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml
@@ -20,9 +20,20 @@
# Configuration values managed by puppet:
jobmanager.rpc.address: <%= @jobmanager_host %>
jobmanager.rpc.port: <%= @jobmanager_port %>
-jobmanager.memory.process.size: <%= @jobmanager_memory %>
-taskmanager.memory.process.size: <%= @taskmanager_memory %>
-taskmanager.numberOfTaskSlots: <%= @taskmanager_number_of_taskslots %>
-parallelism.default: <%= @parallelism_default %>
-jobmanager.execution.failover-strategy: <%= @jobmanager_failover_strategy %>
-rest.port: <%=@rest_port %>
+jobmanager.web.port: <%= @ui_port %>
+
+<% if defined?(storage_dirs) %>
+taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %>
+<% end %>
+
+# Starting from Flink v1.5, there is a rewrite of Flink’s deployment and
process
+# model (internally known as FLIP-6). A new option 'mode' was introduced to
+# indicate which executions to use. However, the way Bigtop puppet deploys
+# 'flink-jobmanager' and 'flink-taskmanager', ie. calling 'flink-daemon.sh
+# jobmanager' and 'flink-daemon.sh taskmanager', is legacy. Without setting
+# mode to 'legacy', flink's built-in exmaples fail with error
+# "JobSubmissionException: Failed to submit JobGraph"
+mode: legacy
+
+# For performance reasons its highly recommended to allocate as much memory to
the
+# Flink TaskManager as possible by setting 'taskmanager.heap.mb'.
diff --git a/bigtop-packages/src/common/flink/flink-jobmanager.svc
b/bigtop-packages/src/common/flink/flink-jobmanager.svc
index 7e74e80..ac7106e 100644
--- a/bigtop-packages/src/common/flink/flink-jobmanager.svc
+++ b/bigtop-packages/src/common/flink/flink-jobmanager.svc
@@ -16,12 +16,12 @@
TYPE="jobmanager"
DAEMON="flink-${TYPE}"
DESC="Flink ${TYPE}"
-EXEC_PATH="/usr/lib/flink/bin/jobmanager.sh"
+EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
WORKING_DIR="/var/lib/flink"
DAEMON_FLAGS=jobmanager
CONF_DIR="/etc/flink/conf"
SVC_USER="flink"
-PIDFILE="/var/run/flink/flink-$SVC_USER-standalonesession.pid"
+PIDFILE="/var/run/flink/flink-$SVC_USER-jobmanager.pid"
EXEC_MODE="cluster"
generate_start() {
@@ -31,10 +31,6 @@ start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "
- if command -v hadoop > /dev/null; then
- export HADOOP_CLASSPATH=$(hadoop classpath)
- fi
- export FLINK_CONF_DIR="${CONF_DIR}"
export FLINK_LOG_DIR="/var/log/flink"
export FLINK_PID_DIR="/var/run/flink"
mkdir -p $FLINK_PID_DIR
@@ -48,7 +44,9 @@ start() {
LOG_FILE=/var/log/flink/${DAEMON}.out
# Flink will set the pid file
- runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 ${EXEC_PATH} start >
$LOG_FILE 2>&1 & "
+ runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
+ ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir $CONF_DIR --executionMode
cluster \
+ > $LOG_FILE 2>&1 & "
sleep 3
diff --git a/bigtop-packages/src/common/flink/flink-taskmanager.svc
b/bigtop-packages/src/common/flink/flink-taskmanager.svc
index 612148c..5bbd616 100644
--- a/bigtop-packages/src/common/flink/flink-taskmanager.svc
+++ b/bigtop-packages/src/common/flink/flink-taskmanager.svc
@@ -16,12 +16,12 @@
TYPE="taskmanager"
DAEMON="flink-${TYPE}"
DESC="Flink ${TYPE}"
-EXEC_PATH="/usr/lib/flink/bin/taskmanager.sh"
+EXEC_PATH="/usr/lib/flink/bin/flink-daemon.sh"
WORKING_DIR="/var/lib/flink"
DAEMON_FLAGS="taskmanager"
SVC_USER="flink"
CONF_DIR="/etc/flink/conf"
-PIDFILE="/var/run/flink/flink-$SVC_USER-taskexecutor.pid"
+PIDFILE="/var/run/flink/flink-$SVC_USER-taskmanager.pid"
generate_start() {
@@ -31,10 +31,6 @@ start() {
[ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
log_success_msg "Starting $DESC (${DAEMON}): "
- if command -v hadoop > /dev/null; then
- export HADOOP_CLASSPATH=$(hadoop classpath)
- fi
- export FLINK_CONF_DIR="${CONF_DIR}"
export FLINK_LOG_DIR="/var/log/flink"
export FLINK_PID_DIR="/var/run/flink"
mkdir -p $FLINK_PID_DIR
@@ -48,7 +44,9 @@ start() {
LOG_FILE=/var/log/flink/${DAEMON}.out
- runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 ${EXEC_PATH} start >
$LOG_FILE 2>&1 & "
+ runuser -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
+ ${EXEC_PATH} start ${DAEMON_FLAGS} --configDir ${CONF_DIR} \
+ > $LOG_FILE 2>&1 & "
sleep 3
diff --git a/bigtop-packages/src/common/flink/install_flink.sh
b/bigtop-packages/src/common/flink/install_flink.sh
index ac8e5b6..6baa669 100644
--- a/bigtop-packages/src/common/flink/install_flink.sh
+++ b/bigtop-packages/src/common/flink/install_flink.sh
@@ -120,7 +120,7 @@ ln -s /etc/flink/conf $PREFIX/$LIB_DIR/conf
cp -ra ${BUILD_DIR}/examples/* $PREFIX/${LIB_DIR}/examples/
-cp ${BUILD_DIR}/{LICENSE,README.txt} ${PREFIX}/${LIB_DIR}/
+cp ${BUILD_DIR}/{LICENSE,NOTICE,README.txt} ${PREFIX}/${LIB_DIR}/
# Copy in the /usr/bin/flink wrapper
install -d -m 0755 $PREFIX/$BIN_DIR
diff --git
a/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
b/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
new file mode 100644
index 0000000..c76d3c3
--- /dev/null
+++ b/bigtop-packages/src/common/flink/patch0-fix-ApplicationReport-api.diff
@@ -0,0 +1,12 @@
+diff --git
a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
+index 8ee08db..fcd56d0 100644
+---
a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
++++
b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java
+@@ -98,6 +98,7 @@ public class AbstractYarnClusterTest extends TestLogger {
+ yarnApplicationState,
+ null,
+ null,
++ 0,
+ 1L,
+ 2L,
+ finalApplicationStatus,
diff --git a/bigtop-tests/smoke-tests/flink/TestFlink.groovy
b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
index dba11aa..b35d3cb 100644
--- a/bigtop-tests/smoke-tests/flink/TestFlink.groovy
+++ b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
@@ -59,7 +59,7 @@ class TestFlink {
// read JM address and port from conf
execCommand("awk '{if(/jobmanager.rpc.address:/) print \$2}' < "+
config_file);
final String jmHost = sh.out.join('\n');
- execCommand("awk '{if(/rest.port:/) print \$2}' < "+config_file);
+ execCommand("awk '{if(/jobmanager.web.port:/) print \$2}' < "+config_file);
final String webPort = sh.out.join('\n');
// check web API
execCommand("curl http://"+jmHost+":"+webPort+"/config";);
diff --git a/bigtop.bom b/bigtop.bom
index 74defb8..ea22671 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -287,7 +287,7 @@ bigtop {
'flink' {
name = 'flink'
relNotes = 'Apache Flink'
- version { base = '1.11.1'; pkg = base; release = 1 }
+ version { base = '1.6.4'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}-src.tgz" }
url { download_path = "/$name/$name-${version.base}"
