Repository: bigtop Updated Branches: refs/heads/master ae9dc84dd -> eab70662b
http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hbase/templates/jaas.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hbase/templates/jaas.conf b/bigtop-deploy/puppet/modules/hadoop_hbase/templates/jaas.conf new file mode 100644 index 0000000..5b6b2a5 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hbase/templates/jaas.conf @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + useTicketCache=false + keyTab="/etc/hbase.keytab" + principal="hbase/<%= @fqdn %>@<%= @kerberos_realm %>"; +}; http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hbase/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hbase/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_hbase/tests/init.pp new file mode 100644 index 0000000..968ac44 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hbase/tests/init.pp @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hadoop_hbase::client { "hbase thrift client": + thrift => true, + kerberos_realm => "KRB.YOU.ORG", +} +hadoop_hbase::master { "test-hbase-master": + rootdir => "hdfs://localhost:17020/hbase", + zookeeper_quorum => "localhost", + kerberos_realm => "KRB.YOU.ORG", +} +hadoop_hbase::server { "test-hbase-server": + rootdir => "hdfs://localhost:17020/hbase", + zookeeper_quorum => "localhost", + kerberos_realm => "KRB.YOU.ORG", +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp new file mode 100644 index 0000000..86131ff --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hadoop_hive { + + class deploy ($roles) { + if ("hive-client" in $roles) { + include hadoop_hive::client + } + } + + class client($hbase_master = "", + $hbase_zookeeper_quorum = "", + $hive_execution_engine = "mr") { + + package { "hive": + ensure => latest, + } + + file { "/etc/hive/conf/hive-site.xml": + content => template('hadoop_hive/hive-site.xml'), + require => Package["hive"], + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml new file mode 100644 index 0000000..2aca237 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml @@ -0,0 +1,68 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- Licensed to the Apache Software Foundation (ASF) under one or more --> +<!-- contributor license agreements. See the NOTICE file distributed with --> +<!-- this work for additional information regarding copyright ownership. --> +<!-- The ASF licenses this file to You under the Apache License, Version 2.0 --> +<!-- (the "License"); you may not use this file except in compliance with --> +<!-- the License. You may obtain a copy of the License at --> +<!-- --> +<!-- http://www.apache.org/licenses/LICENSE-2.0 --> +<!-- --> +<!-- Unless required by applicable law or agreed to in writing, software --> +<!-- distributed under the License is distributed on an "AS IS" BASIS, --> +<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --> +<!-- See the License for the specific language governing permissions and --> +<!-- limitations under the License. --> + +<configuration> + +<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files --> +<!-- that are implied by Hadoop setup variables. --> +<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive --> +<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized --> +<!-- resource). --> + +<!-- Hive Execution Parameters --> + +<% if @hbase_master != "" %> +<property> + <name>hbase.master</name> + <value><%= @hbase_master %></value> + <description>http://wiki.apache.org/hadoop/Hive/HBaseIntegration</description> +</property> +<% end %> + +<% if @hbase_zookeeper_quorum != "" %> +<property> + <name>hbase.zookeeper.quorum</name> + <value><%= @hbase_zookeeper_quorum %></value> + <description>http://wiki.apache.org/hadoop/Hive/HBaseIntegration</description> +</property> +<% end %> + +<property> + <name>hive.execution.engine</name> + <value><%= @hive_execution_engine %></value> +</property> + +<property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> +</property> + +<property> + <name>javax.jdo.option.ConnectionDriverName</name> + <value>org.apache.derby.jdbc.EmbeddedDriver</value> + <description>Driver class name for a JDBC metastore</description> +</property> + +<property> + <name>hive.hwi.war.file</name> + <value>/usr/lib/hive/lib/hive-hwi.war</value> + <description>This is the WAR file with the jsp content for Hive Web Interface</description> +</property> + +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hive/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_hive/tests/init.pp new file mode 100644 index 0000000..ac202e9 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hive/tests/init.pp @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hadoop_hive::client { "test-hive": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_oozie/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_oozie/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_oozie/manifests/init.pp new file mode 100644 index 0000000..8010086 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_oozie/manifests/init.pp @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hadoop_oozie { + + class deploy ($roles) { + if ("oozie-client" in $roles) { + include client + } + + if ("oozie-server" in $roles) { + include hadoop::init_hdfs + include server + Class['Hadoop::Init_hdfs'] -> Class['Hadoop_oozie::Server'] + if ("mapred-app" in $roles) { + Class['Hadoop::Mapred-app'] -> Class['Hadoop_oozie::Server'] + } + } + } + + class client { + package { "oozie-client": + ensure => latest, + } + } + + class server($kerberos_realm = "") { + if ($kerberos_realm) { + require kerberos::client + kerberos::host_keytab { "oozie": + spnego => true, + require => Package["oozie"], + } + } + + package { "oozie": + ensure => latest, + } + + file { "/etc/oozie/conf/oozie-site.xml": + content => template("hadoop_oozie/oozie-site.xml"), + require => Package["oozie"], + } + + exec { "Oozie DB init": + command => "/etc/init.d/oozie init", + cwd => "/var/lib/oozie", + creates => "/var/lib/oozie/derby.log", + require => Package["oozie"], + unless => "/etc/init.d/oozie status", + } + + service { "oozie": + ensure => running, + require => [ Package["oozie"], Exec["Oozie DB init"] ], + hasrestart => true, + hasstatus => true, + } + Kerberos::Host_keytab <| title == "oozie" |> -> Service["oozie"] + + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_oozie/templates/oozie-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_oozie/templates/oozie-site.xml b/bigtop-deploy/puppet/modules/hadoop_oozie/templates/oozie-site.xml new file mode 100644 index 0000000..93d5eaf --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_oozie/templates/oozie-site.xml @@ -0,0 +1,369 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <!-- + Refer to the oozie-default.xml file for the complete list of + Oozie configuration properties and their default values. + --> + + <property> + <name>oozie.service.ActionService.executor.ext.classes</name> + <value> + org.apache.oozie.action.email.EmailActionExecutor, + org.apache.oozie.action.hadoop.HiveActionExecutor, + org.apache.oozie.action.hadoop.ShellActionExecutor, + org.apache.oozie.action.hadoop.SqoopActionExecutor, + org.apache.oozie.action.hadoop.DistcpActionExecutor + </value> + </property> + + <property> + <name>oozie.service.SchemaService.wf.ext.schemas</name> + <value>shell-action-0.1.xsd,shell-action-0.2.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,ssh-action-0.1.xsd,distcp-action-0.1.xsd</value> + </property> + + <property> + <name>oozie.system.id</name> + <value>oozie-${user.name}</value> + <description> + The Oozie system ID. + </description> + </property> + + <property> + <name>oozie.systemmode</name> + <value>NORMAL</value> + <description> + System mode for Oozie at startup. + </description> + </property> + + <property> + <name>oozie.service.AuthorizationService.security.enabled</name> + <value>false</value> + <description> + Specifies whether security (user name/admin role) is enabled or not. + If disabled any user can manage Oozie system and manage any job. + </description> + </property> + + <property> + <name>oozie.service.PurgeService.older.than</name> + <value>30</value> + <description> + Jobs older than this value, in days, will be purged by the PurgeService. + </description> + </property> + + <property> + <name>oozie.service.PurgeService.purge.interval</name> + <value>3600</value> + <description> + Interval at which the purge service will run, in seconds. + </description> + </property> + + <property> + <name>oozie.service.CallableQueueService.queue.size</name> + <value>10000</value> + <description>Max callable queue size</description> + </property> + + <property> + <name>oozie.service.CallableQueueService.threads</name> + <value>10</value> + <description>Number of threads used for executing callables</description> + </property> + + <property> + <name>oozie.service.CallableQueueService.callable.concurrency</name> + <value>3</value> + <description> + Maximum concurrency for a given callable type. + Each command is a callable type (submit, start, run, signal, job, jobs, suspend,resume, etc). + Each action type is a callable type (Map-Reduce, Pig, SSH, FS, sub-workflow, etc). + All commands that use action executors (action-start, action-end, action-kill and action-check) use + the action type as the callable type. + </description> + </property> + + <property> + <name>oozie.service.coord.normal.default.timeout + </name> + <value>120</value> + <description>Default timeout for a coordinator action input check (in minutes) for normal job. + -1 means infinite timeout</description> + </property> + + <property> + <name>oozie.db.schema.name</name> + <value>oozie</value> + <description> + Oozie DataBase Name + </description> + </property> + + <property> + <name>oozie.service.JPAService.create.db.schema</name> + <value>false</value> + <description> + Creates Oozie DB. + + If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. + If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. + </description> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.driver</name> + <value>org.apache.derby.jdbc.EmbeddedDriver</value> + <description> + JDBC driver class. + </description> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.url</name> + <value>jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true</value> + <description> + JDBC URL. + </description> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.username</name> + <value>sa</value> + <description> + DB user name. + </description> + </property> + + <property> + <name>oozie.service.JPAService.jdbc.password</name> + <value> </value> + <description> + DB user password. + + IMPORTANT: if password is emtpy leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + </description> + </property> + + <property> + <name>oozie.service.JPAService.pool.max.active.conn</name> + <value>10</value> + <description> + Max number of connections. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.kerberos.enabled</name> + <value><%= if (@kerberos_realm != "") ; "true" else "false" end %></value> + <description> + Indicates if Oozie is configured to use Kerberos. + </description> + </property> + + <property> + <name>local.realm</name> + <value><%= @kerberos_realm %></value> + <description> + Kerberos Realm used by Oozie and Hadoop. Using 'local.realm' to be aligned with Hadoop configuration + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.keytab.file</name> + <value>/etc/oozie.keytab</value> + <description> + Location of the Oozie user keytab file. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.kerberos.principal</name> + <value>${user.name}/<%= @fqdn %>@${local.realm}</value> + <description> + Kerberos principal for Oozie service. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.jobTracker.whitelist</name> + <value> </value> + <description> + Whitelisted job tracker for Oozie service. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.nameNode.whitelist</name> + <value> </value> + <description> + Whitelisted job tracker for Oozie service. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.hadoop.configurations</name> + <value>*=/etc/hadoop/conf</value> + <description> + Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of + the Hadoop service (JobTracker, HDFS). The wildcard '*' configuration is + used when there is no exact match for an authority. The HADOOP_CONF_DIR contains + the relevant Hadoop *-site.xml files. If the path is relative is looked within + the Oozie configuration directory; though the path can be absolute (i.e. to point + to Hadoop client conf/ directories in the local filesystem. + </description> + </property> + + <property> + <name>oozie.service.WorkflowAppService.system.libpath</name> + <value>/user/${user.name}/share/lib</value> + <description> + System library path to use for workflow applications. + This path is added to workflow application if their job properties sets + the property 'oozie.use.system.libpath' to true. + </description> + </property> + + <property> + <name>use.system.libpath.for.mapreduce.and.pig.jobs</name> + <value>false</value> + <description> + If set to true, submissions of MapReduce and Pig jobs will include + automatically the system library path, thus not requiring users to + specify where the Pig JAR files are. Instead, the ones from the system + library path are used. + </description> + </property> + + <property> + <name>oozie.authentication.type</name> + <value><%= @kerberos_realm.empty? ? "simple" : "kerberos" %></value> + <description> + Defines authentication used for Oozie HTTP endpoint. + Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# + </description> + </property> + + <property> + <name>oozie.authentication.token.validity</name> + <value>36000</value> + <description> + Indicates how long (in seconds) an authentication token is valid before it has + to be renewed. + </description> + </property> + + <property> + <name>oozie.authentication.signature.secret</name> + <value>oozie</value> + <description> + The signature secret for signing the authentication tokens. + If not set a random secret is generated at startup time. + In order to authentiation to work correctly across multiple hosts + the secret must be the same across al the hosts. + </description> + </property> + + <property> + <name>oozie.authentication.cookie.domain</name> + <value></value> + <description> + The domain to use for the HTTP cookie that stores the authentication token. + In order to authentiation to work correctly across multiple hosts + the domain must be correctly set. + </description> + </property> + + <property> + <name>oozie.authentication.simple.anonymous.allowed</name> + <value>true</value> + <description> + Indicates if anonymous requests are allowed. + This setting is meaningful only when using 'simple' authentication. + </description> + </property> + + <property> + <name>oozie.authentication.kerberos.principal</name> + <value>HTTP/<%= @fqdn %>@${local.realm}</value> + <description> + Indicates the Kerberos principal to be used for HTTP endpoint. + The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification. + </description> + </property> + + <property> + <name>oozie.authentication.kerberos.keytab</name> + <value>${oozie.service.HadoopAccessorService.keytab.file}</value> + <description> + Location of the keytab file with the credentials for the principal. + Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop. + </description> + </property> + + <property> + <name>oozie.authentication.kerberos.name.rules</name> + <value>DEFAULT</value> + <description> + The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's + KerberosName for more details. + </description> + </property> + + <!-- Proxyuser Configuration --> + + <property> + <name>oozie.service.ProxyUserService.proxyuser.hue.hosts</name> + <value>*</value> + <description> + List of hosts the '#USER#' user is allowed to perform 'doAs' + operations. + + The '#USER#' must be replaced with the username o the user who is + allowed to perform 'doAs' operations. + + The value can be the '*' wildcard or a list of hostnames. + + For multiple users copy this property and replace the user name + in the property name. + </description> + </property> + + <property> + <name>oozie.service.ProxyUserService.proxyuser.hue.groups</name> + <value>*</value> + <description> + List of groups the '#USER#' user is allowed to impersonate users + from to perform 'doAs' operations. + + The '#USER#' must be replaced with the username o the user who is + allowed to perform 'doAs' operations. + + The value can be the '*' wildcard or a list of groups. + + For multiple users copy this property and replace the user name + in the property name. + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_oozie/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_oozie/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_oozie/tests/init.pp new file mode 100644 index 0000000..61d0521 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_oozie/tests/init.pp @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hadoop_oozie::server { "test-oozie-server": } +hadoop_oozie::client { "test-oozie": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp new file mode 100644 index 0000000..0a09dec --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_pig/manifests/init.pp @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hadoop_pig { + + class deploy ($roles) { + if ("pig-client" in $roles) { + include client + } + } + + class client { + package { "pig": + ensure => latest, + require => Package["hadoop"], + } + + file { "/etc/pig/conf/pig.properties": + content => template('hadoop_pig/pig.properties'), + require => Package["pig"], + owner => "root", /* FIXME: I'm really no sure about these -- we might end */ + mode => "755", /* up deploying/testing a different thing compared */ + /* to a straight rpm/deb deployment */ + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties b/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties new file mode 100644 index 0000000..1e1c297 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_pig/templates/pig.properties @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Pig configuration file. All values can be overwritten by command line arguments. +# see bin/pig -help + +# log4jconf log4j configuration file +# log4jconf=./conf/log4j.properties + +# brief logging (no timestamps) +brief=false + +# clustername, name of the hadoop jobtracker. If no port is defined port 50020 will be used. +#cluster + +#debug level, INFO is default +debug=INFO + +# a file that contains pig script +#file= + +# load jarfile, colon separated +#jar= + +#verbose print all log messages to screen (default to print only INFO and above to screen) +verbose=false + +#exectype local|mapreduce, mapreduce is default +#exectype=mapreduce +# hod realted properties +#ssh.gateway +#hod.expect.root +#hod.expect.uselatest +#hod.command +#hod.config.dir +#hod.param + + +#Do not spill temp files smaller than this size (bytes) +pig.spill.size.threshold=5000000 +#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +#This should help reduce the number of files being spilled. +pig.spill.gc.activation.size=40000000 + + +###################### +# Everything below this line is Yahoo specific. Note that I've made +# (almost) no changes to the lines above to make merging in from Apache +# easier. Any values I don't want from above I override below. +# +# This file is configured for use with HOD on the production clusters. If you +# want to run pig with a static cluster you will need to remove everything +# below this line and set the cluster value (above) to the +# hostname and port of your job tracker. + +exectype=mapreduce +log.file= http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp new file mode 100644 index 0000000..9692630 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_pig/tests/init.pp @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include hadoop_pig +hadoop_pig::client { "test-pig": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_zookeeper/files/java.env ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/files/java.env b/bigtop-deploy/puppet/modules/hadoop_zookeeper/files/java.env new file mode 100644 index 0000000..6aca488 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/files/java.env @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +export SERVER_JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/jaas.conf" http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp new file mode 100644 index 0000000..7bbfcfd --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hadoop_zookeeper { + + class deploy ($roles) { + if ("zookeeper-client" in $roles) { + include hadoop_zookeeper::client + } + + if ("zookeeper-server" in $roles) { + include hadoop_zookeeper::server + } + } + + class client { + package { "zookeeper": + ensure => latest, + require => Package["jdk"], + } + } + + class server($myid, + $port = "2181", + $datadir = "/var/lib/zookeeper", + $ensemble = ["localhost:2888:3888"], + $kerberos_realm = "") + { + package { "zookeeper-server": + ensure => latest, + require => Package["jdk"], + } + + service { "zookeeper-server": + ensure => running, + require => [ Package["zookeeper-server"], + Exec["zookeeper-server-initialize"] ], + subscribe => [ File["/etc/zookeeper/conf/zoo.cfg"], + File["/var/lib/zookeeper/myid"] ], + hasrestart => true, + hasstatus => true, + } + + file { "/etc/zookeeper/conf/zoo.cfg": + content => template("hadoop_zookeeper/zoo.cfg"), + require => Package["zookeeper-server"], + } + + file { "/var/lib/zookeeper/myid": + content => inline_template("<%= @myid %>"), + require => Package["zookeeper-server"], + } + + exec { "zookeeper-server-initialize": + command => "/usr/bin/zookeeper-server-initialize", + user => "zookeeper", + creates => "/var/lib/zookeeper/version-2", + require => Package["zookeeper-server"], + } + + if ($kerberos_realm) { + require kerberos::client + + kerberos::host_keytab { "zookeeper": + spnego => true, + notify => Service["zookeeper-server"], + require => Package["zookeeper-server"], + } + + file { "/etc/zookeeper/conf/java.env": + source => "puppet:///modules/hadoop_zookeeper/java.env", + require => Package["zookeeper-server"], + notify => Service["zookeeper-server"], + } + + file { "/etc/zookeeper/conf/jaas.conf": + content => template("hadoop_zookeeper/jaas.conf"), + require => Package["zookeeper-server"], + notify => Service["zookeeper-server"], + } + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/jaas.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/jaas.conf b/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/jaas.conf new file mode 100644 index 0000000..61b2eac --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/jaas.conf @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Server { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="/etc/zookeeper.keytab" + storeKey=true + useTicketCache=false + principal="zookeeper/<%= @fqdn %>@<%= @kerberos_realm %>"; +}; http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg b/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg new file mode 100644 index 0000000..426fe2a --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +maxClientCnxns=50 +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +dataDir=<%= @datadir %> +# the port at which the clients will connect +clientPort=<%= @port %> +<% @ensemble.each_with_index do |server,idx| %> +server.<%= idx %>=<%= server %> +<% end %> + +<% if @kerberos_realm != "" -%> +authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider +jaasLoginRenew=3600000 + +# HBase needs these configs so that different hbase daemons +# (master, regionservers), which run on different hosts, can +# read from and write to znodes that others create +kerberos.removeHostFromPrincipal=true +kerberos.removeRealmFromPrincipal=true +<% end -%> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp new file mode 100644 index 0000000..0340f10 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hadoop_zookeeper::client { "zoo visitor": } +hadoop_zookeeper::server { "test-oozie-server": + myid => "0", + ensemble => ["foo:2888:3888", "bar:2888:3888", "baz:2888:3888"], +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hue/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp index 72d08c0..fa6af88 100644 --- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp @@ -21,7 +21,7 @@ class hue { Class['Hadoop::Httpfs'] -> Class['Hue::Server'] } if ("hbase-client" in $roles) { - Class['Hadoop-hbase::Client'] -> Class['Hue::Server'] + Class['Hadoop_hbase::Client'] -> Class['Hue::Server'] } } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp deleted file mode 100644 index d8e26df..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class ignite-hadoop { - class deploy ($roles) { - if ("ignite-server" in $roles) { - ignite-hadoop::server { "ignite-hadoop-node": } - } - } - - define server() { - $hadoop_head_node = hiera("bigtop::hadoop_head_node") - $hadoop_namenode_port = hiera("hadoop::common_hdfs::hadoop_namenode_port", "8020") - - package { "ignite-hadoop": - ensure => latest, - } - - package { "ignite-hadoop-service": - ensure => latest, - } - - file { "/etc/default/ignite-hadoop": - content => template("ignite-hadoop/ignite-hadoop"), - require => Package["ignite-hadoop"], - } - - file { "/etc/hadoop/ignite.client.conf": - ensure => directory, - owner => 'root', - group => 'root', - mode => '0755', - require => Package["ignite-hadoop-service"], - } - file { "/etc/ignite-hadoop/conf/default-config.xml": - content => template('ignite-hadoop/default-config.xml'), - require => [Package["ignite-hadoop"]], - } - file { "/etc/hadoop/ignite.client.conf/core-site.xml": - content => template('ignite-hadoop/core-site.xml'), - require => [File["/etc/hadoop/ignite.client.conf"]], - } - file { - "/etc/hadoop/ignite.client.conf/mapred-site.xml": - content => template('ignite-hadoop/mapred-site.xml'), - require => [File["/etc/hadoop/ignite.client.conf"]], - } - file { - "/etc/hadoop/ignite.client.conf/hive-site.xml": - content => template('ignite-hadoop/hive-site.xml'), - require => [File["/etc/hadoop/ignite.client.conf"]], - } -## let's make sure that ignite-hadoop libs are linked properly - file {'/usr/lib/hadoop/lib/ignite-core.jar': - ensure => link, - target => '/usr/lib/ignite-hadoop/libs/ignite-core.jar', - require => [Package["ignite-hadoop-service"]], - } - file {'/usr/lib/hadoop/lib/ignite-hadoop.jar': - ensure => link, - target => '/usr/lib/ignite-hadoop/libs/ignite-hadoop/ignite-hadoop.jar', - require => [Package["ignite-hadoop-service"]], - } - - service { "ignite-hadoop": - ensure => running, - require => [ Package["ignite-hadoop", "ignite-hadoop-service"], File["/etc/default/ignite-hadoop"] ], - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/templates/core-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/core-site.xml b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/core-site.xml deleted file mode 100644 index a6edb36..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/core-site.xml +++ /dev/null @@ -1,65 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- - This template file contains settings needed to run Apache Hadoop jobs - with Ignite's distributed in-memory file system GGFS. - - You can replace '$HADOOP_HOME/etc/hadoop/core-site.xml' file with this one - to work with Ignite IGFS nodes running on localhost (these local nodes can be - a part of distributed cluster though). To work with file system on remote - hosts you need to change the host of file system URI to any host running - Ignite's IGFS node. - - Note that Ignite jars must be in Apache Hadoop client classpath to work - with this configuration. - - Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup. ---> - -<configuration> - <!-- - Set default file system to IGFS instance named "igfs" configured in Ignite. - --> - <property> - <name>fs.defaultFS</name> - <value>igfs://igfs@localhost</value> - </property> - - <!-- - Set Hadoop 1.* file system implementation class for GGFS. - --> - <property> - <name>fs.igfs.impl</name> - <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value> - </property> - - <!-- - Set Hadoop 2.* file system implementation class for GGFS. - --> - <property> - <name>fs.AbstractFileSystem.igfs.impl</name> - <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value> - </property> - - <!-- - Disallow data node replacement since it does not make sense for Ignite's IGFS nodes. - --> - <property> - <name>dfs.client.block.write.replace-datanode-on-failure.policy</name> - <value>NEVER</value> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml deleted file mode 100644 index bfaee60..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml +++ /dev/null @@ -1,187 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<beans xmlns="http://www.springframework.org/schema/beans"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:util="http://www.springframework.org/schema/util"; - xsi:schemaLocation="http://www.springframework.org/schema/beans - http://www.springframework.org/schema/beans/spring-beans.xsd - http://www.springframework.org/schema/util - http://www.springframework.org/schema/util/spring-util.xsd";> - - <!-- - Optional description. - --> - <description> - Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. - Ignite node will start with this configuration by default. - </description> - - <!-- - Initialize property configurer so we can reference environment variables. - --> - <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> - <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> - <property name="searchSystemEnvironment" value="true"/> - </bean> - - <!-- - Abstract IGFS file system configuration to be used as a template. - --> - <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true"> - <!-- Must correlate with cache affinity mapper. --> - <property name="blockSize" value="#{128 * 1024}"/> - <property name="perNodeBatchSize" value="512"/> - <property name="perNodeParallelBatchCount" value="16"/> - - <property name="prefetchBlocks" value="32"/> - </bean> - - <!-- - Abstract cache configuration for IGFS file data to be used as a template. - --> - <bean id="dataCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> - <property name="cacheMode" value="PARTITIONED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - <property name="backups" value="0"/> - <property name="affinityMapper"> - <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> - <!-- How many sequential blocks will be stored on the same node. --> - <constructor-arg value="512"/> - </bean> - </property> - </bean> - - <!-- - Abstract cache configuration for IGFS metadata to be used as a template. - --> - <bean id="metaCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> - <property name="cacheMode" value="REPLICATED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - </bean> - - <!-- - Configuration of Ignite node. - --> - <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> - <!-- - Apache Hadoop Accelerator configuration. - --> - <property name="hadoopConfiguration"> - <bean class="org.apache.ignite.configuration.HadoopConfiguration"> - <!-- Information about finished jobs will be kept for 30 seconds. --> - <property name="finishedJobInfoTtl" value="30000"/> - </bean> - </property> - - <!-- - This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. - --> - <property name="connectorConfiguration"> - <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> - <property name="port" value="11211"/> - </bean> - </property> - - <!-- - Configure one IGFS file system instance named "igfs" on this node. - --> - <property name="fileSystemConfiguration"> - <list> - <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase"> - <property name="name" value="igfs"/> - - <!-- Caches with these names must be configured. --> - <property name="metaCacheName" value="igfs-meta"/> - <property name="dataCacheName" value="igfs-data"/> - - <!-- Configure TCP endpoint for communication with the file system instance. --> - <property name="ipcEndpointConfiguration"> - <bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration"> - <property name="type" value="TCP" /> - <property name="host" value="0.0.0.0" /> - <property name="port" value="10500" /> - </bean> - </property> - - <!-- Sample secondary file system configuration. - 'uri' - the URI of the secondary file system. - 'cfgPath' - optional configuration path of the secondary file system, - e.g. /opt/foo/core-site.xml. Typically left to be null. - 'userName' - optional user name to access the secondary file system on behalf of. Use it - if Hadoop client and the Ignite node are running on behalf of different users. - --> - <property name="secondaryFileSystem"> - <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> - <constructor-arg name="uri" value='hdfs://<%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>'/> - <constructor-arg name="cfgPath"><null/></constructor-arg> - <constructor-arg name="userName" value="ignite"/> - </bean> - </property> - </bean> - </list> - </property> - - <!-- - Caches needed by IGFS. - --> - <property name="cacheConfiguration"> - <list> - <!-- File system metadata cache. --> - <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="metaCacheCfgBase"> - <property name="name" value="igfs-meta"/> - </bean> - - <!-- File system files data cache. --> - <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="dataCacheCfgBase"> - <property name="name" value="igfs-data"/> - </bean> - </list> - </property> - - <!-- - Disable events. - --> - <property name="includeEventTypes"> - <list> - <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/> - <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/> - <util:constant static-field="org.apache.ignite.events.EventType.EVT_JOB_MAPPED"/> - </list> - </property> - - <!-- - TCP discovery SPI can be configured with list of addresses if multicast is not available. - --> - <!-- - <property name="discoverySpi"> - <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> - <property name="ipFinder"> - <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> - <property name="addresses"> - <list> - <value>127.0.0.1:47500..47509</value> - </list> - </property> - </bean> - </property> - </bean> - </property> - --> - </bean> -</beans> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/hive-site.xml b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/hive-site.xml deleted file mode 100644 index 1f58898..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/hive-site.xml +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<!-- - This file contains settings needed to run Apache Hive queries - with Ignite In-Memory Accelerator. ---> -<configuration> - <!-- - Ignite requires query plan to be passed not using local resource. - --> - <property> - <name>hive.rpc.query.plan</name> - <value>true</value> - </property> -</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/templates/ignite-hadoop ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/ignite-hadoop b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/ignite-hadoop deleted file mode 100644 index e4530ca..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/ignite-hadoop +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export HADOOP_PREFIX=/usr -export IGNITE_HOME=/usr/lib/ignite-hadoop -export IGNITE_LOG_DIR=/var/log/ignite-hadoop -export IGNITE_PID_DIR=/var/run/ignite-hadoop -export IGNITE_WORK_DIR="/var/run/ignite-hadoop/work" -export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/templates/mapred-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/mapred-site.xml deleted file mode 100644 index f2a0ae6..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/mapred-site.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- - This template file contains settings needed to run Apache Hadoop jobs - with Ignite In-Memory Accelerator. - - You can replace '$HADOOP_HOME/etc/hadoop/mapred-site.xml' file with this one - to run jobs on localhost (local node can be a part of distributed cluster though). - To run jobs on remote host you have to change jobtracker address to the REST address - of any running Ignite node. - - Note that Ignite jars must be in Apache Hadoop client classpath to work - with this configuration. - - Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup. ---> - -<configuration> - <!-- - Framework name must be set to 'ignite'. - --> - <property> - <name>mapreduce.framework.name</name> - <value>ignite</value> - </property> - - <!-- - Job tracker address must be set to the REST address of any running - Ignite node. - --> - <property> - <name>mapreduce.jobtracker.address</name> - <value>localhost:11211</value> - </property> - - <!-- Parameters for job tuning. --> - <!-- - <property> - <name>mapreduce.job.reduces</name> - <value>1</value> - </property> - - <property> - <name>mapreduce.job.maps</name> - <value>4</value> - </property> - --> - -</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite-hadoop/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/tests/init.pp b/bigtop-deploy/puppet/modules/ignite-hadoop/tests/init.pp deleted file mode 100644 index 7fa5c77..0000000 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/tests/init.pp +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ignite-hadoop::server { "test-ignite-hadoop-server": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/ignite_hadoop/manifests/init.pp new file mode 100644 index 0000000..98c27fc --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/manifests/init.pp @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class ignite_hadoop { + class deploy ($roles) { + if ("ignite-server" in $roles) { + ignite_hadoop::server { "ignite-hadoop-node": } + } + } + + define server() { + $hadoop_head_node = hiera("bigtop::hadoop_head_node") + $hadoop_namenode_port = hiera("hadoop::common_hdfs::hadoop_namenode_port", "8020") + + package { "ignite-hadoop": + ensure => latest, + } + + package { "ignite-hadoop-service": + ensure => latest, + } + + file { "/etc/default/ignite-hadoop": + content => template("ignite-hadoop/ignite-hadoop"), + require => Package["ignite-hadoop"], + } + + file { "/etc/hadoop/ignite.client.conf": + ensure => directory, + owner => 'root', + group => 'root', + mode => '0755', + require => Package["ignite-hadoop-service"], + } + file { "/etc/ignite-hadoop/conf/default-config.xml": + content => template('ignite-hadoop/default-config.xml'), + require => [Package["ignite-hadoop"]], + } + file { "/etc/hadoop/ignite.client.conf/core-site.xml": + content => template('ignite-hadoop/core-site.xml'), + require => [File["/etc/hadoop/ignite.client.conf"]], + } + file { + "/etc/hadoop/ignite.client.conf/mapred-site.xml": + content => template('ignite_hadoop/mapred-site.xml'), + require => [File["/etc/hadoop/ignite.client.conf"]], + } + file { + "/etc/hadoop/ignite.client.conf/hive-site.xml": + content => template('ignite_hadoop/hive-site.xml'), + require => [File["/etc/hadoop/ignite.client.conf"]], + } +## let's make sure that ignite-hadoop libs are linked properly + file {'/usr/lib/hadoop/lib/ignite-core.jar': + ensure => link, + target => '/usr/lib/ignite-hadoop/libs/ignite-core.jar', + require => [Package["ignite-hadoop-service"]], + } + file {'/usr/lib/hadoop/lib/ignite-hadoop.jar': + ensure => link, + target => '/usr/lib/ignite-hadoop/libs/ignite-hadoop/ignite-hadoop.jar', + require => [Package["ignite-hadoop-service"]], + } + + service { "ignite-hadoop": + ensure => running, + require => [ Package["ignite-hadoop", "ignite-hadoop-service"], File["/etc/default/ignite-hadoop"] ], + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/templates/core-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/core-site.xml b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/core-site.xml new file mode 100644 index 0000000..a6edb36 --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/core-site.xml @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- + This template file contains settings needed to run Apache Hadoop jobs + with Ignite's distributed in-memory file system GGFS. + + You can replace '$HADOOP_HOME/etc/hadoop/core-site.xml' file with this one + to work with Ignite IGFS nodes running on localhost (these local nodes can be + a part of distributed cluster though). To work with file system on remote + hosts you need to change the host of file system URI to any host running + Ignite's IGFS node. + + Note that Ignite jars must be in Apache Hadoop client classpath to work + with this configuration. + + Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup. +--> + +<configuration> + <!-- + Set default file system to IGFS instance named "igfs" configured in Ignite. + --> + <property> + <name>fs.defaultFS</name> + <value>igfs://igfs@localhost</value> + </property> + + <!-- + Set Hadoop 1.* file system implementation class for GGFS. + --> + <property> + <name>fs.igfs.impl</name> + <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value> + </property> + + <!-- + Set Hadoop 2.* file system implementation class for GGFS. + --> + <property> + <name>fs.AbstractFileSystem.igfs.impl</name> + <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value> + </property> + + <!-- + Disallow data node replacement since it does not make sense for Ignite's IGFS nodes. + --> + <property> + <name>dfs.client.block.write.replace-datanode-on-failure.policy</name> + <value>NEVER</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/templates/default-config.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/default-config.xml b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/default-config.xml new file mode 100644 index 0000000..bfaee60 --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/default-config.xml @@ -0,0 +1,187 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:util="http://www.springframework.org/schema/util"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/util + http://www.springframework.org/schema/util/spring-util.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Abstract IGFS file system configuration to be used as a template. + --> + <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true"> + <!-- Must correlate with cache affinity mapper. --> + <property name="blockSize" value="#{128 * 1024}"/> + <property name="perNodeBatchSize" value="512"/> + <property name="perNodeParallelBatchCount" value="16"/> + + <property name="prefetchBlocks" value="32"/> + </bean> + + <!-- + Abstract cache configuration for IGFS file data to be used as a template. + --> + <bean id="dataCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> + <property name="cacheMode" value="PARTITIONED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + <property name="backups" value="0"/> + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <!-- How many sequential blocks will be stored on the same node. --> + <constructor-arg value="512"/> + </bean> + </property> + </bean> + + <!-- + Abstract cache configuration for IGFS metadata to be used as a template. + --> + <bean id="metaCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <!-- + Apache Hadoop Accelerator configuration. + --> + <property name="hadoopConfiguration"> + <bean class="org.apache.ignite.configuration.HadoopConfiguration"> + <!-- Information about finished jobs will be kept for 30 seconds. --> + <property name="finishedJobInfoTtl" value="30000"/> + </bean> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <!-- + Configure one IGFS file system instance named "igfs" on this node. + --> + <property name="fileSystemConfiguration"> + <list> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase"> + <property name="name" value="igfs"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="igfs-meta"/> + <property name="dataCacheName" value="igfs-data"/> + + <!-- Configure TCP endpoint for communication with the file system instance. --> + <property name="ipcEndpointConfiguration"> + <bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration"> + <property name="type" value="TCP" /> + <property name="host" value="0.0.0.0" /> + <property name="port" value="10500" /> + </bean> + </property> + + <!-- Sample secondary file system configuration. + 'uri' - the URI of the secondary file system. + 'cfgPath' - optional configuration path of the secondary file system, + e.g. /opt/foo/core-site.xml. Typically left to be null. + 'userName' - optional user name to access the secondary file system on behalf of. Use it + if Hadoop client and the Ignite node are running on behalf of different users. + --> + <property name="secondaryFileSystem"> + <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> + <constructor-arg name="uri" value='hdfs://<%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>'/> + <constructor-arg name="cfgPath"><null/></constructor-arg> + <constructor-arg name="userName" value="ignite"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + Caches needed by IGFS. + --> + <property name="cacheConfiguration"> + <list> + <!-- File system metadata cache. --> + <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="metaCacheCfgBase"> + <property name="name" value="igfs-meta"/> + </bean> + + <!-- File system files data cache. --> + <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="dataCacheCfgBase"> + <property name="name" value="igfs-data"/> + </bean> + </list> + </property> + + <!-- + Disable events. + --> + <property name="includeEventTypes"> + <list> + <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/> + <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/> + <util:constant static-field="org.apache.ignite.events.EventType.EVT_JOB_MAPPED"/> + </list> + </property> + + <!-- + TCP discovery SPI can be configured with list of addresses if multicast is not available. + --> + <!-- + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + </bean> + </property> + --> + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml new file mode 100644 index 0000000..1f58898 --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/hive-site.xml @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<!-- + This file contains settings needed to run Apache Hive queries + with Ignite In-Memory Accelerator. +--> +<configuration> + <!-- + Ignite requires query plan to be passed not using local resource. + --> + <property> + <name>hive.rpc.query.plan</name> + <value>true</value> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/templates/ignite-hadoop ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/ignite-hadoop b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/ignite-hadoop new file mode 100644 index 0000000..e4530ca --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/ignite-hadoop @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export HADOOP_PREFIX=/usr +export IGNITE_HOME=/usr/lib/ignite-hadoop +export IGNITE_LOG_DIR=/var/log/ignite-hadoop +export IGNITE_PID_DIR=/var/run/ignite-hadoop +export IGNITE_WORK_DIR="/var/run/ignite-hadoop/work" +export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/templates/mapred-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/mapred-site.xml new file mode 100644 index 0000000..f2a0ae6 --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/templates/mapred-site.xml @@ -0,0 +1,63 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- + This template file contains settings needed to run Apache Hadoop jobs + with Ignite In-Memory Accelerator. + + You can replace '$HADOOP_HOME/etc/hadoop/mapred-site.xml' file with this one + to run jobs on localhost (local node can be a part of distributed cluster though). + To run jobs on remote host you have to change jobtracker address to the REST address + of any running Ignite node. + + Note that Ignite jars must be in Apache Hadoop client classpath to work + with this configuration. + + Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup. +--> + +<configuration> + <!-- + Framework name must be set to 'ignite'. + --> + <property> + <name>mapreduce.framework.name</name> + <value>ignite</value> + </property> + + <!-- + Job tracker address must be set to the REST address of any running + Ignite node. + --> + <property> + <name>mapreduce.jobtracker.address</name> + <value>localhost:11211</value> + </property> + + <!-- Parameters for job tuning. --> + <!-- + <property> + <name>mapreduce.job.reduces</name> + <value>1</value> + </property> + + <property> + <name>mapreduce.job.maps</name> + <value>4</value> + </property> + --> + +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/ignite_hadoop/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite_hadoop/tests/init.pp b/bigtop-deploy/puppet/modules/ignite_hadoop/tests/init.pp new file mode 100644 index 0000000..75ae1df --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite_hadoop/tests/init.pp @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ignite_hadoop::server { "test-ignite-hadoop-server": }
