Repository: bigtop Updated Branches: refs/heads/master 46d0a7239 -> e7154397b
BIGTOP-1876. Update puppet recipes for Sqoop2 Signed-off-by: Evans Ye <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e7154397 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e7154397 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e7154397 Branch: refs/heads/master Commit: e7154397bc3ccdf9d3b552337d4c4963768da503 Parents: 46d0a72 Author: YoungWoo Kim <[email protected]> Authored: Wed May 20 15:30:02 2015 +0900 Committer: Evans Ye <[email protected]> Committed: Sat May 23 12:06:04 2015 +0000 ---------------------------------------------------------------------- .../puppet/hieradata/bigtop/cluster.yaml | 6 ++-- bigtop-deploy/puppet/hieradata/site.yaml | 1 + bigtop-deploy/puppet/manifests/cluster.pp | 9 +++-- .../modules/hadoop-sqoop/manifests/init.pp | 35 -------------------- .../puppet/modules/hadoop-sqoop/tests/init.pp | 17 ---------- .../puppet/modules/hue/manifests/init.pp | 2 +- .../puppet/modules/hue/templates/hue.ini | 4 +-- bigtop-deploy/puppet/modules/hue/tests/init.pp | 2 +- .../puppet/modules/sqoop/manifests/init.pp | 22 ++++++++++++ .../puppet/modules/sqoop/tests/init.pp | 16 +++++++++ .../puppet/modules/sqoop2/manifests/init.pp | 35 ++++++++++++++++++++ .../puppet/modules/sqoop2/tests/init.pp | 17 ++++++++++ 12 files changed, 104 insertions(+), 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 44cb1b7..aaa4a06 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -16,7 +16,7 @@ # "$components" list. If $components isn't set then everything in the stack will # be installed as usual. Otherwise only a specified list will be set # Possible elements: -# hadoop,yarn,hbase,tachyon,flume,solrcloud,spark,oozie,hcat,sqoop,httpfs, +# hadoop,yarn,hbase,tachyon,flume,solrcloud,spark,oozie,hcat,sqoop,sqoop2,httpfs, # hue,mahout,giraph,crunch,pig,hive,zookeeper # Example (to deploy only HDFS and YARN server and gateway parts) # This can be a comma-separated list or an array. @@ -115,7 +115,7 @@ hadoop-zookeeper::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" bigtop::hadoop_rm_http_port: "8088" bigtop::hadoop_rm_proxy_port: "8088" bigtop::hadoop_history_server_port: "19888" -bigtop::sqoop_server_port: "<never defined correctly>" +bigtop::sqoop2_server_port: "12000" bigtop::hbase_thrift_port: "9090" bigtop::hadoop_oozie_port: "11000" @@ -127,7 +127,7 @@ hue::server::history_server_url: "http://%{hiera('bigtop::hadoop_head_node')}:%{ # those use fqdn instead of hadoop_head_node because it's only ever activated # on the gatewaynode hue::server::webhdfs_url: "http://%{fqdn}:%{hiera('hadoop::httpfs::hadoop_httpfs_port')}/webhdfs/v1" -hue::server::sqoop_url: "http://%{fqdn}:%{hiera('bigtop::sqoop_server_port')}/sqoop" +hue::server::sqoop2_url: "http://%{fqdn}:%{hiera('bigtop::sqoop2_server_port')}/sqoop" hue::server::solr_url: "http://%{fqdn}:%{hiera('solr::server::port')}/solr/" hue::server::hbase_thrift_url: "%{fqdn}:%{hiera('bigtop::hbase_thrift_port')}" hue::server::oozie_url: "http://%{hiera('bigtop::hadoop_head_node')}:%{hiera('bigtop::hadoop_oozie_port')}/oozie" http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index 00861c4..826d748 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -28,6 +28,7 @@ hadoop::hadoop_storage_dirs: # - solrcloud # - spark # - sqoop +# - sqoop2 # - tachyon # - tez # - yarn http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 35ef195..77cbbcf 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -161,8 +161,8 @@ class standby_head_node inherits hadoop_cluster_node { } class hadoop_gateway_node inherits hadoop_cluster_node { - if ($all or "sqoop" in $components) { - include hadoop-sqoop::server + if ($all or "sqoop2" in $components) { + include sqoop2::server } if ($all or "httpfs" in $components) { @@ -196,8 +196,11 @@ class hadoop_gateway_node inherits hadoop_cluster_node { if ($all or "hive" in $components) { include hadoop-hive::client } + if ($all or "sqoop2" in $components) { + include sqoop2::client + } if ($all or "sqoop" in $components) { - include hadoop-sqoop::client + include sqoop::client } if ($all or "oozie" in $components) { include hadoop-oozie::client http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp deleted file mode 100644 index e0223ba..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-sqoop { - class client { - package { "sqoop-client": - ensure => latest, - } - } - - class server { - package { "sqoop-server": - ensure => latest, - } - - service { "sqoop-server": - ensure => running, - require => Package["sqoop-server"], - hasstatus => true, - hasrestart => true, - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp deleted file mode 100644 index 442323e..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hadoop-sqoop::client { "test-sqoop-client": } -hadoop-sqoop::server { "test-sqoop-server": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/hue/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp index 4fd9c1c..687a055 100644 --- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp @@ -14,7 +14,7 @@ # limitations under the License. class hue { - class server($sqoop_url = "http://localhost:12000/sqoop";, $solr_url = "http://localhost:8983/solr/";, $hbase_thrift_url = "", + class server($sqoop2_url = "http://localhost:12000/sqoop";, $solr_url = "http://localhost:8983/solr/";, $hbase_thrift_url = "", $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_proxy_url, $history_server_url, $hive_host = "", $hive_port = "10000", $rm_logical_name = undef, $rm_api_port = "8088", $app_blacklist = "impala, security", http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/hue/templates/hue.ini ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/templates/hue.ini b/bigtop-deploy/puppet/modules/hue/templates/hue.ini index fe1b472..93c82fc 100644 --- a/bigtop-deploy/puppet/modules/hue/templates/hue.ini +++ b/bigtop-deploy/puppet/modules/hue/templates/hue.ini @@ -746,8 +746,8 @@ # For autocompletion, fill out the librdbms section. # Sqoop server URL -<% if @sqoop_url != "" -%> - server_url=<%= @sqoop_url %> +<% if @sqoop2_url != "" -%> + server_url=<%= @sqoop2_url %> <% end -%> ########################################################################### http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/hue/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/tests/init.pp b/bigtop-deploy/puppet/modules/hue/tests/init.pp index 7007ea0..38fa1a0 100644 --- a/bigtop-deploy/puppet/modules/hue/tests/init.pp +++ b/bigtop-deploy/puppet/modules/hue/tests/init.pp @@ -14,7 +14,7 @@ # limitations under the License. hue::server { "test-hue-server": - sqoop_url => "http://localhost:12000/sqoop";, + sqoop2_url => "http://localhost:12000/sqoop";, solr_url => "http://localhost:8983/solr/";, hbase_thrift_url => "localhost:9090", webhdfs_url => "http://localhost:14000/webhdfs/v1";, http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/sqoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/sqoop/manifests/init.pp b/bigtop-deploy/puppet/modules/sqoop/manifests/init.pp new file mode 100644 index 0000000..d7e8011 --- /dev/null +++ b/bigtop-deploy/puppet/modules/sqoop/manifests/init.pp @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class sqoop { + class client { + package { "sqoop": + ensure => latest, + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/sqoop/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/sqoop/tests/init.pp b/bigtop-deploy/puppet/modules/sqoop/tests/init.pp new file mode 100644 index 0000000..19810bc --- /dev/null +++ b/bigtop-deploy/puppet/modules/sqoop/tests/init.pp @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sqoop::client { "test-sqoop-client": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp b/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp new file mode 100644 index 0000000..9648483 --- /dev/null +++ b/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class sqoop2 { + class client { + package { "sqoop2-client": + ensure => latest, + } + } + + class server { + package { "sqoop2-server": + ensure => latest, + } + + service { "sqoop2-server": + ensure => running, + require => Package["sqoop2-server"], + hasstatus => true, + hasrestart => true, + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/e7154397/bigtop-deploy/puppet/modules/sqoop2/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/sqoop2/tests/init.pp b/bigtop-deploy/puppet/modules/sqoop2/tests/init.pp new file mode 100644 index 0000000..6263f6a --- /dev/null +++ b/bigtop-deploy/puppet/modules/sqoop2/tests/init.pp @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sqoop2::client { "test-sqoop2-client": } +sqoop2::server { "test-sqoop2-server": }
