Repository: bigtop Updated Branches: refs/heads/master 6780b1caf -> c922418d7
BIGTOP-1970. Ignite IGFS now fully supports mutiltenancy: deployment should configure it with HDFS backing Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c922418d Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c922418d Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c922418d Branch: refs/heads/master Commit: c922418d7a9d59c36adb75fd3f147c98b0768165 Parents: 6780b1c Author: Konstantin Boudnik <[email protected]> Authored: Mon Sep 21 16:03:25 2015 -0700 Committer: Konstantin Boudnik <[email protected]> Committed: Tue Sep 22 10:45:46 2015 -0700 ---------------------------------------------------------------------- .../modules/ignite-hadoop/manifests/init.pp | 7 + .../ignite-hadoop/templates/default-config.xml | 187 +++++++++++++++++++ 2 files changed, 194 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/c922418d/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp index de658a5..81832af 100644 --- a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp @@ -21,6 +21,9 @@ class ignite-hadoop { } define server() { + $hadoop_head_node = hiera("bigtop::hadoop_head_node") + $hadoop_namenode_port = hiera("hadoop::common_hdfs::hadoop_namenode_port", "8020") + package { "ignite-hadoop": ensure => latest, } @@ -41,6 +44,10 @@ class ignite-hadoop { mode => '0755', require => Package["ignite-hadoop-service"], } + file { "/etc/ignite-hadoop/conf/default-config.xml": + content => template('ignite-hadoop/default-config.xml'), + require => [Package["ignite-hadoop"]], + } file { "/etc/hadoop/ignite.client.conf/core-site.xml": content => template('ignite-hadoop/core-site.xml'), require => [File["/etc/hadoop/ignite.client.conf"]], http://git-wip-us.apache.org/repos/asf/bigtop/blob/c922418d/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml new file mode 100644 index 0000000..bfaee60 --- /dev/null +++ b/bigtop-deploy/puppet/modules/ignite-hadoop/templates/default-config.xml @@ -0,0 +1,187 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:util="http://www.springframework.org/schema/util"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/util + http://www.springframework.org/schema/util/spring-util.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Abstract IGFS file system configuration to be used as a template. + --> + <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true"> + <!-- Must correlate with cache affinity mapper. --> + <property name="blockSize" value="#{128 * 1024}"/> + <property name="perNodeBatchSize" value="512"/> + <property name="perNodeParallelBatchCount" value="16"/> + + <property name="prefetchBlocks" value="32"/> + </bean> + + <!-- + Abstract cache configuration for IGFS file data to be used as a template. + --> + <bean id="dataCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> + <property name="cacheMode" value="PARTITIONED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + <property name="backups" value="0"/> + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <!-- How many sequential blocks will be stored on the same node. --> + <constructor-arg value="512"/> + </bean> + </property> + </bean> + + <!-- + Abstract cache configuration for IGFS metadata to be used as a template. + --> + <bean id="metaCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <!-- + Apache Hadoop Accelerator configuration. + --> + <property name="hadoopConfiguration"> + <bean class="org.apache.ignite.configuration.HadoopConfiguration"> + <!-- Information about finished jobs will be kept for 30 seconds. --> + <property name="finishedJobInfoTtl" value="30000"/> + </bean> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <!-- + Configure one IGFS file system instance named "igfs" on this node. + --> + <property name="fileSystemConfiguration"> + <list> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase"> + <property name="name" value="igfs"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="igfs-meta"/> + <property name="dataCacheName" value="igfs-data"/> + + <!-- Configure TCP endpoint for communication with the file system instance. --> + <property name="ipcEndpointConfiguration"> + <bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration"> + <property name="type" value="TCP" /> + <property name="host" value="0.0.0.0" /> + <property name="port" value="10500" /> + </bean> + </property> + + <!-- Sample secondary file system configuration. + 'uri' - the URI of the secondary file system. + 'cfgPath' - optional configuration path of the secondary file system, + e.g. /opt/foo/core-site.xml. Typically left to be null. + 'userName' - optional user name to access the secondary file system on behalf of. Use it + if Hadoop client and the Ignite node are running on behalf of different users. + --> + <property name="secondaryFileSystem"> + <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> + <constructor-arg name="uri" value='hdfs://<%= @hadoop_head_node %>:<%= @hadoop_namenode_port %>'/> + <constructor-arg name="cfgPath"><null/></constructor-arg> + <constructor-arg name="userName" value="ignite"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + Caches needed by IGFS. + --> + <property name="cacheConfiguration"> + <list> + <!-- File system metadata cache. --> + <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="metaCacheCfgBase"> + <property name="name" value="igfs-meta"/> + </bean> + + <!-- File system files data cache. --> + <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="dataCacheCfgBase"> + <property name="name" value="igfs-data"/> + </bean> + </list> + </property> + + <!-- + Disable events. + --> + <property name="includeEventTypes"> + <list> + <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/> + <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/> + <util:constant static-field="org.apache.ignite.events.EventType.EVT_JOB_MAPPED"/> + </list> + </property> + + <!-- + TCP discovery SPI can be configured with list of addresses if multicast is not available. + --> + <!-- + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + </bean> + </property> + --> + </bean> +</beans>
