Repository: bigtop Updated Branches: refs/heads/master b5d001d5e -> 3e0de8c04
BIGTOP-2357: Add Apache Flink to Bigtop / Create puppet recipes Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3e0de8c0 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3e0de8c0 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3e0de8c0 Branch: refs/heads/master Commit: 3e0de8c04bc8f6cb5993d3c08d1f56a837958db1 Parents: b5d001d Author: Robert Metzger <[email protected]> Authored: Tue May 24 14:51:48 2016 +0200 Committer: Roman Shaposhnik <[email protected]> Committed: Fri Jun 17 16:29:40 2016 -0700 ---------------------------------------------------------------------- .../puppet/hieradata/bigtop/cluster.yaml | 9 ++- bigtop-deploy/puppet/hieradata/site.yaml | 1 + bigtop-deploy/puppet/manifests/cluster.pp | 5 ++ .../puppet/modules/flink/manifests/init.pp | 62 ++++++++++++++++++++ .../modules/flink/templates/flink-conf.yaml | 31 ++++++++++ bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 1 + 6 files changed, 108 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index f592f64..5c2c5f1 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -16,7 +16,7 @@ # "$components" list. If $components isn't set then everything in the stack will # be installed as usual. Otherwise only a specified list will be set # Possible elements: -# hadoop,yarn,hbase,alluxio,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs, +# hadoop,yarn,hbase,alluxio,flink,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs, # hue,mahout,giraph,crunch,pig,hive,zookeeper,ycsb,qfs # Example (to deploy only HDFS and YARN server and gateway parts) # This can be a comma-separated list or an array. @@ -191,3 +191,10 @@ zeppelin::server::spark_master_url: "yarn-client" zeppelin::server::hiveserver2_url: "jdbc:hive2://%{hiera('hadoop-hive::common::hiveserver2_host')}:%{hiera('hadoop-hive::common::hiveserver2_port')}" zeppelin::server::hiveserver2_user: "%{hiera('bigtop::hiveserver2_user')}" zeppelin::server::hiveserver2_password: "%{hiera('bigtop::hiveserver2_password')}" + +# Flink +flink::common::jobmanager_host: "%{hiera('bigtop::hadoop_head_node')}" +flink::common::jobmanager_port: "6123" + +flink::common::ui_port: "8081" +flink::common::storage_dirs: "%{hiera('hadoop::hadoop_storage_dirs')}" http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index d429269..b9ed6f9 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -15,6 +15,7 @@ hadoop::hadoop_storage_dirs: # - alluxio # - apex # - crunch +# - flink # - flume # - giraph # - ignite_hadoop http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index ba980bd..9ff424c 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -56,6 +56,10 @@ $roles_map = { master => ["alluxio-master"], worker => ["alluxio-worker"], }, + flink => { + master => ["flink-jobmanager"], + worker => ["flink-taskmanager"], + }, flume => { worker => ["flume-agent"], }, @@ -161,6 +165,7 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_ "alluxio", "apex", "crunch", + "flink", "giraph", "hadoop", "hadoop_hbase", http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/modules/flink/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/flink/manifests/init.pp b/bigtop-deploy/puppet/modules/flink/manifests/init.pp new file mode 100644 index 0000000..ae79fe4 --- /dev/null +++ b/bigtop-deploy/puppet/modules/flink/manifests/init.pp @@ -0,0 +1,62 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +class flink { + + class deploy ($roles) { + if ("flink-jobmanager" in $roles) { + include flink::jobmanager + } + + if ("flink-taskmanager" in $roles) { + include flink::taskmanager + } + } + + class common($jobmanager_host, $jobmanager_port, $ui_port, $storage_dirs) { + # make sure flink is installed + package { "flink": + ensure => latest + } + + # set values in flink-conf.yaml + file { "/etc/flink/conf/flink-conf.yaml": + content => template("flink/flink-conf.yaml"), + require => Package["flink"] + } + } + + class jobmanager { + include common + + service { "flink-jobmanager": + ensure => running, + require => Package["flink"], + subscribe => File["/etc/flink/conf/flink-conf.yaml"], + hasrestart => true, + hasstatus => true + + } + + } + + class taskmanager { + include common + + service { "flink-taskmanager": + ensure => running, + require => Package["flink"], + subscribe => File["/etc/flink/conf/flink-conf.yaml"], + hasrestart => true, + hasstatus => true, + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml new file mode 100644 index 0000000..14e8853 --- /dev/null +++ b/bigtop-deploy/puppet/modules/flink/templates/flink-conf.yaml @@ -0,0 +1,31 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + + +# Configuration values managed by puppet: +jobmanager.rpc.address: <%= @jobmanager_host %> +jobmanager.rpc.port: <%= @jobmanager_port %> +jobmanager.web.port: <%= @ui_port %> + +<% if defined?(storage_dirs) %> +taskmanager.tmp.dirs: <%= @storage_dirs.join(":") %> +<% end %> + + +# For performance reasons its highly recommended to allocate as much memory to the +# Flink TaskManager as possible by setting 'taskmanager.heap.mb'. http://git-wip-us.apache.org/repos/asf/bigtop/blob/3e0de8c0/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile index 63aff20..b4017d0 100755 --- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile @@ -67,6 +67,7 @@ bigtop_master = "bigtop1.vagrant" $script = <<SCRIPT service iptables stop +service firewalld stop chkconfig iptables off # Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN, # which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP
