http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml new file mode 100644 index 0000000..072c6db --- /dev/null +++ b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml @@ -0,0 +1,141 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="gridName" value="IGFS-cli-0"/> + + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <!-- + Configure one IGFS file system instance named "igfs" on this node. + --> + <property name="fileSystemConfiguration"> + <list> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs-0"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="PRIMARY"/> + <property name="ipcEndpointEnabled" value="false"/> + + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + <property name="localAddress" value="127.0.0.1"/> + <property name="localPort" value="47500"/> + <property name="joinTimeout" value="5000"/> + </bean> + </property> + + </bean> +</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml new file mode 100644 index 0000000..e52921d --- /dev/null +++ b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml @@ -0,0 +1,141 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="gridName" value="IGFS-cli-1"/> + + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11212"/> + </bean> + </property> + + <!-- + Configure one IGFS file system instance named "igfs" on this node. + --> + <property name="fileSystemConfiguration"> + <list> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs-1"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="PRIMARY"/> + <property name="ipcEndpointEnabled" value="false"/> + + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47600..47609</value> + </list> + </property> + </bean> + </property> + <property name="localAddress" value="127.0.0.1"/> + <property name="localPort" value="47600"/> + <property name="joinTimeout" value="5000"/> + </bean> + </property> + + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml new file mode 100644 index 0000000..9344e99 --- /dev/null +++ b/modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml @@ -0,0 +1,141 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="gridName" value="IGFS-cli-2"/> + + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11213"/> + </bean> + </property> + + <!-- + Configure one IGFS file system instance named "igfs" on this node. + --> + <property name="fileSystemConfiguration"> + <list> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs-2"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="PRIMARY"/> + <property name="ipcEndpointEnabled" value="false"/> + + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47700..47709</value> + </list> + </property> + </bean> + </property> + <property name="localAddress" value="127.0.0.1"/> + <property name="localPort" value="47700"/> + <property name="joinTimeout" value="5000"/> + </bean> + </property> + + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml b/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml new file mode 100644 index 0000000..6fc38cd --- /dev/null +++ b/modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml @@ -0,0 +1,150 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="clientMode" value="true"/> + <property name="gridName" value="test-IGFS-cli"/> + + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <property name="fileSystemConfiguration"> + <list> + <!-- + Configure PRIMARY IGFS file system instance named "igfs" + --> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="DUAL_ASYNC"/> + + <property name="secondaryFileSystem"> + <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> + <property name="defaultUserName" value="secondary-default"/> + + <property name="fileSystemFactory"> + <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory"> + <property name="uri" value="igfs://[email protected]:11500/"/> + <property name="configPaths" value="/work/core-site-test.xml"/> + </bean> + </property> + </bean> + </property> + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + </bean> + </property> + + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml b/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml new file mode 100644 index 0000000..fcd8d82 --- /dev/null +++ b/modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml @@ -0,0 +1,148 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="clientMode" value="true"/> + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <property name="fileSystemConfiguration"> + <list> + <!-- + Configure PRIMARY IGFS file system instance named "igfs" + --> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="DUAL_SYNC"/> + + <property name="secondaryFileSystem"> + <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> + <property name="defaultUserName" value="secondary-default"/> + + <property name="fileSystemFactory"> + <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory"> + <property name="uri" value="igfs://[email protected]:11500/"/> + <property name="configPaths" value="/work/core-site-test.xml"/> + </bean> + </property> + </bean> + </property> + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + </bean> + </property> + + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-primary.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/igfs-cli-config-primary.xml b/modules/hadoop/src/test/config/igfs-cli-config-primary.xml new file mode 100644 index 0000000..cc066b2 --- /dev/null +++ b/modules/hadoop/src/test/config/igfs-cli-config-primary.xml @@ -0,0 +1,139 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="clientMode" value="true"/> + <property name="gridName" value="test-IGFS-cli"/> + + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <!-- + Configure one IGFS file system instance named "igfs" on this node. + --> + <property name="fileSystemConfiguration"> + <list> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="PRIMARY"/> + <property name="ipcEndpointEnabled" value="false"/> + + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + </bean> + </property> + + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml b/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml new file mode 100644 index 0000000..f25fd50 --- /dev/null +++ b/modules/hadoop/src/test/config/igfs-cli-config-proxy.xml @@ -0,0 +1,148 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + Ignite Spring configuration file. + + When starting a standalone Ignite node, you need to execute the following command: + {IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml + + When starting Ignite from Java IDE, pass path to this file into Ignition: + Ignition.start("path-to-this-file/default-config.xml"); +--> +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd";> + + <!-- + Optional description. + --> + <description> + Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. + Ignite node will start with this configuration by default. + </description> + + <!-- + Initialize property configurer so we can reference environment variables. + --> + <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> + <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> + <property name="searchSystemEnvironment" value="true"/> + </bean> + + <!-- + Configuration of Ignite node. + --> + <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <property name="clientMode" value="true"/> + <property name="marshaller"> + <bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller"> + </bean> + </property> + + <!-- + Configure caches where IGFS will store data. + --> + <property name="cacheConfiguration"> + <list> + <!-- + Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, + and must have backups to maintain file system consistency in case of node crash. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="replicated"/> + <property name="cacheMode" value="REPLICATED"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + </bean> + + <!-- + Configure data cache where file's data will be stored. + --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="partitioned"/> + <property name="atomicityMode" value="TRANSACTIONAL"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="writeSynchronizationMode" value="FULL_SYNC"/> + + <property name="affinityMapper"> + <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> + <property name="groupSize" value="128"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- + This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. + --> + <property name="connectorConfiguration"> + <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> + <property name="port" value="11211"/> + </bean> + </property> + + <property name="fileSystemConfiguration"> + <list> + <!-- + Configure PRIMARY IGFS file system instance named "igfs" + --> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> + <!-- IGFS name you will use to access IGFS through Hadoop API. --> + <property name="name" value="igfs"/> + + <!-- Caches with these names must be configured. --> + <property name="metaCacheName" value="replicated"/> + <property name="dataCacheName" value="partitioned"/> + <property name="blockSize" value="524288"/> + <property name="defaultMode" value="PROXY"/> + + <property name="secondaryFileSystem"> + <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> + <property name="defaultUserName" value="secondary-default"/> + + <property name="fileSystemFactory"> + <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory"> + <property name="uri" value="igfs://[email protected]:11500/"/> + <property name="configPaths" value="/work/core-site-test.xml"/> + </bean> + </property> + </bean> + </property> + </bean> + </list> + </property> + + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> + <property name="addresses"> + <list> + <value>127.0.0.1:47500..47509</value> + </list> + </property> + </bean> + </property> + </bean> + </property> + + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java index 1bcbdaa..4d4f68b 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemAbstractSelfTest.java @@ -17,6 +17,30 @@ package org.apache.ignite.internal.processors.hadoop.impl.igfs; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.net.URI; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Deque; +import java.util.LinkedList; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; @@ -53,6 +77,7 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.marshaller.optimized.OptimizedMarshaller; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; @@ -60,30 +85,6 @@ import org.apache.ignite.testframework.GridTestUtils; import org.jetbrains.annotations.Nullable; import org.jsr166.ThreadLocalRandom8; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.lang.reflect.Field; -import java.net.URI; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; -import java.util.Deque; -import java.util.LinkedList; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; @@ -102,7 +103,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA private static final String PRIMARY_AUTHORITY = "igfs@"; /** Primary file systme URI. */ - private static final String PRIMARY_URI = "igfs://" + PRIMARY_AUTHORITY + "/"; + protected static final String PRIMARY_URI = "igfs://" + PRIMARY_AUTHORITY + "/"; /** Secondary file system authority. */ private static final String SECONDARY_AUTHORITY = "[email protected]:11500"; @@ -122,6 +123,9 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA /** Group size. */ public static final int GRP_SIZE = 128; + /** Group size. */ + public static final int GRID_COUNT = 4; + /** Path to the default hadoop configuration. */ public static final String HADOOP_FS_CFG = "examples/config/filesystem/core-site.xml"; @@ -135,7 +139,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA private static CyclicBarrier barrier; /** File system. */ - private static FileSystem fs; + protected static FileSystem fs; /** Default IGFS mode. */ protected final IgfsMode mode; @@ -187,6 +191,8 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA this.skipLocShmem = skipLocShmem; endpoint = skipLocShmem ? "127.0.0.1:10500" : "shmem:10500"; + + ((TcpDiscoveryVmIpFinder)IP_FINDER).setAddresses(Collections.singleton("127.0.0.1:47500..47509")); } /** @@ -199,7 +205,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { - Configuration secondaryConf = configuration(SECONDARY_AUTHORITY, true, true); + Configuration secondaryConf = configurationSecondary(SECONDARY_AUTHORITY); secondaryConf.setInt("fs.igfs.block.size", 1024); @@ -220,11 +226,22 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA } /** + * @throws Exception + */ + protected void stopNodes() throws Exception { + for (int i = 0; i < GRID_COUNT; ++i) + stopGrid(i); + + G.stop("grid_secondary", false); + + Thread.sleep(500); + } + /** * Starts the nodes for this test. * * @throws Exception If failed. */ - private void startNodes() throws Exception { + protected void startNodes() throws Exception { if (mode != PRIMARY) { // Start secondary IGFS. FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); @@ -258,8 +275,10 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA cfg.setGridName("grid_secondary"); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + discoSpi.setLocalPort(47510); - discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + TcpDiscoveryVmIpFinder finder = new TcpDiscoveryVmIpFinder(true); + discoSpi.setIpFinder(finder); cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); @@ -269,7 +288,10 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA G.start(cfg); } - startGrids(4); + for (int i = 1; i < GRID_COUNT; ++i) + startGrid(i); + + startGrid(0); } /** {@inheritDoc} */ @@ -329,12 +351,14 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(gridName); + IgniteConfiguration cfg = new IgniteConfiguration(); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); discoSpi.setIpFinder(IP_FINDER); + cfg.setGridName(gridName); + cfg.setMarshaller(new OptimizedMarshaller()); cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(cacheConfiguration(gridName)); cfg.setFileSystemConfiguration(igfsConfiguration(gridName)); @@ -2049,7 +2073,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA final FSDataOutputStream s = fs.create(filePath); // Open the stream before stopping IGFS. try { - G.stopAll(true); // Stop the server. + stopNodes(); startNodes(); // Start server again. @@ -2095,7 +2119,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA for (int i = 0; i < nClients; i++) q.add(FileSystem.get(primaryFsUri, cfg)); - G.stopAll(true); // Stop the server. + stopNodes(); startNodes(); // Start server again. @@ -2414,7 +2438,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA * @param skipLocShmem Whether to skip local shmem mode. * @return Configuration. */ - private static Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) { + protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) { Configuration cfg = new Configuration(); cfg.set("fs.defaultFS", "igfs://" + authority + "/"); @@ -2432,4 +2456,29 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA return cfg; } + + /** + * Create configuration for test. + * + * @param authority Authority. + * @param skipEmbed Whether to skip embedded mode. + * @param skipLocShmem Whether to skip local shmem mode. + * @return Configuration. + */ + protected Configuration configurationSecondary(String authority) { + Configuration cfg = new Configuration(); + + cfg.set("fs.defaultFS", "igfs://" + authority + "/"); + cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); + cfg.set("fs.AbstractFileSystem.igfs.impl", + org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); + + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true); + + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true); + + return cfg; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java new file mode 100644 index 0000000..8198cd3 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedAbstractSelfTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.FileSystemConfiguration; +import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem; +import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; +import org.apache.ignite.igfs.IgfsMode; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.testframework.GridTestUtils; + +/** + * IGFS Hadoop file system Ignite client -based self test. + */ +public abstract class IgniteHadoopFileSystemClientBasedAbstractSelfTest extends IgniteHadoopFileSystemAbstractSelfTest { + /** Alive node index. */ + private static final int ALIVE_NODE_IDX = GRID_COUNT - 1; + + /** + * Constructor. + * + * @param mode IGFS mode. + */ + IgniteHadoopFileSystemClientBasedAbstractSelfTest(IgfsMode mode) { + super(mode, true, true); + } + + /** {@inheritDoc} */ + @Override protected IgfsIpcEndpointConfiguration primaryIpcEndpointConfiguration(final String gridName) { + return null; + } + + /** {@inheritDoc} */ + @Override protected FileSystemConfiguration igfsConfiguration(String gridName) throws IgniteCheckedException { + FileSystemConfiguration cfg = super.igfsConfiguration(gridName); + + cfg.setIpcEndpointEnabled(false); + + return cfg; + } + + /** + * @return Path to Ignite client node configuration. + */ + protected abstract String getClientConfig(); + + /** {@inheritDoc} */ + @Override protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) { + Configuration cfg = new Configuration(); + + cfg.set("fs.defaultFS", "igfs://" + authority + "/"); + cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); + cfg.set("fs.AbstractFileSystem.igfs.impl", + org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); + + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true); + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true); + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_TCP, authority), true); + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_REMOTE_TCP, authority), true); + cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority), + getClientConfig()); + + return cfg; + } + + /** {@inheritDoc} */ + @Override public void testClientReconnect() throws Exception { + Path filePath = new Path(PRIMARY_URI, "file1"); + + final FSDataOutputStream s = fs.create(filePath); // Open the stream before stopping IGFS. + + try { + restartServerNodesExceptOne(); + + // Check that client is again operational. + assertTrue(fs.mkdirs(new Path(PRIMARY_URI, "dir1/dir2"))); + + s.write("test".getBytes()); + + s.flush(); // Flush data to the broken output stream. + + assertTrue(fs.exists(filePath)); + } + finally { + U.closeQuiet(s); // Safety. + } + } + + /** + * Verifies that client reconnects after connection to the server has been lost (multithreaded mode). + * + * @throws Exception If error occurs. + */ + @Override public void testClientReconnectMultithreaded() throws Exception { + final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>(); + + Configuration cfg = new Configuration(); + + for (Map.Entry<String, String> entry : primaryFsCfg) + cfg.set(entry.getKey(), entry.getValue()); + + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + + final int nClients = 1; + + // Initialize clients. + for (int i = 0; i < nClients; i++) + q.add(FileSystem.get(primaryFsUri, cfg)); + + restartServerNodesExceptOne(); + + GridTestUtils.runMultiThreaded(new Callable<Object>() { + @Override public Object call() throws Exception { + FileSystem fs = q.poll(); + + try { + // Check that client is again operational. + assertTrue(fs.mkdirs(new Path("/" + Thread.currentThread().getName()))); + + return true; + } + finally { + U.closeQuiet(fs); + } + } + }, nClients, "test-client"); + } + + /** + * + * @throws Exception If failed. + */ + private void restartServerNodesExceptOne() throws Exception { + stopAllNodesExcept(ALIVE_NODE_IDX); + + Thread.sleep(500); + + startAllNodesExcept(ALIVE_NODE_IDX); // Start server again. + + Thread.sleep(500); + + stopGrid(ALIVE_NODE_IDX); + + Thread.sleep(500); + + startGrid(ALIVE_NODE_IDX); + + Thread.sleep(500); + } + + /** + * @param nodeIdx Node index to not stop + */ + private void stopAllNodesExcept(int nodeIdx) { + for (int i = 0; i < GRID_COUNT; ++i) + if (i != nodeIdx) + stopGrid(i); + } + + /** + * @param nodeIdx Node index to not stop + * @throws Exception If failed. + */ + private void startAllNodesExcept(int nodeIdx) throws Exception { + for (int i = 0; i < GRID_COUNT; ++i) + if (i != nodeIdx) + startGrid(i); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java new file mode 100644 index 0000000..3bd61b5 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualAsyncSelfTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import org.apache.ignite.igfs.IgfsMode; + +/** + * IGFS Hadoop file system Ignite client -based self test for DUAL_ASYNC mode. + */ +public class IgniteHadoopFileSystemClientBasedDualAsyncSelfTest + extends IgniteHadoopFileSystemClientBasedAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemClientBasedDualAsyncSelfTest() { + super(IgfsMode.DUAL_ASYNC); + } + + /** {@inheritDoc} */ + @Override protected String getClientConfig() { + return "modules/hadoop/src/test/config/igfs-cli-config-dual-async.xml"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java new file mode 100644 index 0000000..c7e97d7 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedDualSyncSelfTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import org.apache.ignite.igfs.IgfsMode; + +/** + * IGFS Hadoop file system Ignite client -based self test for DUAL_SYNC mode. + */ +public class IgniteHadoopFileSystemClientBasedDualSyncSelfTest + extends IgniteHadoopFileSystemClientBasedAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemClientBasedDualSyncSelfTest() { + super(IgfsMode.DUAL_SYNC); + } + + /** {@inheritDoc} */ + @Override protected String getClientConfig() { + return "modules/hadoop/src/test/config/igfs-cli-config-dual-sync.xml"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java new file mode 100644 index 0000000..6091dca --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import java.net.URI; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.IgnitionEx; +import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; + +/** + * IGFS Hadoop file system Ignite client -based self test for PROXY mode. + */ +public class IgniteHadoopFileSystemClientBasedOpenTest extends GridCommonAbstractTest { + /** Config root path. */ + private static final String [] CONFIGS = { + "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml", + "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml", + "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml" + }; + + /** Nodes types. */ + private static NodeType[][] nodesTypes = new NodeType[][] { + {NodeType.REMOTE, NodeType.REMOTE, NodeType.REMOTE}, + {NodeType.LOCAL, NodeType.REMOTE, NodeType.REMOTE}, + {NodeType.REMOTE, NodeType.LOCAL, NodeType.REMOTE}, + {NodeType.LOCAL, NodeType.LOCAL, NodeType.REMOTE}, + {NodeType.REMOTE, NodeType.REMOTE, NodeType.LOCAL}, + {NodeType.LOCAL, NodeType.REMOTE, NodeType.LOCAL}, + {NodeType.REMOTE, NodeType.LOCAL, NodeType.LOCAL}, + {NodeType.LOCAL, NodeType.LOCAL, NodeType.LOCAL}, + + {NodeType.LOCAL_SAME_NAME, NodeType.REMOTE, NodeType.REMOTE}, + {NodeType.REMOTE, NodeType.LOCAL_SAME_NAME, NodeType.REMOTE}, + {NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME, NodeType.REMOTE}, + {NodeType.REMOTE, NodeType.REMOTE, NodeType.LOCAL_SAME_NAME}, + {NodeType.LOCAL_SAME_NAME, NodeType.REMOTE, NodeType.LOCAL_SAME_NAME}, + {NodeType.REMOTE, NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME}, + {NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME}, + }; + + /** Test threads count. */ + private static final int THREADS_COUNT = 20; + + /** Iterations count. */ + private static final int ITERATIONS = 10; + + /** Iterations count. */ + private static boolean skipInProc; + + /** + * @param idx Grid index. + * @return Path to Ignite config file. + */ + private static String cfgPath(int idx) { + return CONFIGS[idx]; + } + + /** + * @param idx Grid index. + * @return Path to Ignite config file. + */ + private static String uri(int idx) { + return "igfs://" + authority(idx) + '/'; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteBiTuple<IgniteConfiguration, GridSpringResourceContext> cfgPair = + IgnitionEx.loadConfiguration(cfgPath(getTestGridIndex(gridName))); + + IgniteConfiguration cfg = cfgPair.get1(); + + cfg.setGridName(gridName); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + + super.afterTestsStopped(); + } + + /** {@inheritDoc} */ + @Override protected long getTestTimeout() { + return 15 * 60_000; + } + + /** + * Create configuration for test. + * + * @param idx Grid index. + * @return Configuration. + */ + protected Configuration configuration(int idx) { + Configuration cfg = new Configuration(); + + cfg.set("fs.defaultFS", "igfs://" + authority(idx) + '/'); + cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); + cfg.set("fs.AbstractFileSystem.igfs.impl", + org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); + + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + + cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority(idx)), + cfgPath(idx)); + + if (skipInProc) + cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority(idx)), true); + + return cfg; + } + + /** + * Create configuration for test. + * + * @return Configuration. + */ + protected Configuration configurationWrongIgfs() { + Configuration cfg = new Configuration(); + + cfg.set("fs.defaultFS", "igfs://igfs-wrong-name@/"); + cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); + cfg.set("fs.AbstractFileSystem.igfs.impl", + org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); + + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + + cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, "igfs-wrong-name@"), + cfgPath(0)); + + return cfg; + } + + /** + * @param idx Grid index. + * @return IGFS authority. + */ + private static String authority(int idx) { + return "igfs-" + idx + "@"; + } + + + /** + * @throws Exception If failed. + */ + public void testFsOpenMultithreaded() throws Exception { + skipInProc = false; + + checkFsOpenWithAllNodesTypes(); + } + + /** + * @throws Exception If failed. + */ + private void checkFsOpenWithAllNodesTypes() throws Exception { + for (int i = 0; i < nodesTypes.length; ++i) { + log.info("Begin test case for nodes: " + S.arrayToString(NodeType.class, nodesTypes[i])); + + startNodes(nodesTypes[i]); + + // Await clusters start + Thread.sleep(10_000); + + try { + checkFsOpenMultithreaded(); + } finally { + stopAllGrids(); + } + } + } + + /** + * @throws Exception If failed. + */ + public void testFsOpenMultithreadedSkipInProc() throws Exception { + skipInProc = true; + + checkFsOpenWithAllNodesTypes(); + } + + /** + * @throws Exception If failed. + */ + public void testIgniteClientWithIgfsMisconfigure() throws Exception { + startNodes(new NodeType[] {NodeType.REMOTE, NodeType.REMOTE, NodeType.REMOTE}); + + // Await clusters start + Thread.sleep(10_000); + + try { + try (FileSystem fs = FileSystem.get(new URI("igfs://igfs-wrong-name@"), configurationWrongIgfs())) { + FSDataOutputStream out = fs.create(new Path("igfs://igfs-wrong-name@/file"), true); + + assert false : "Exception must be thrown"; + } + catch (Exception e) { + assertTrue(e.getMessage().contains("Cannot find IGFS 'igfs-wrong-name' at the node")); + } + } finally { + stopAllGrids(); + } + } + + /** + * @param nodeTypes Types of server nodes. + * @throws Exception If failed. + */ + public void startNodes(NodeType[] nodeTypes) throws Exception { + assert nodeTypes.length == CONFIGS.length; + + for (int i = 0; i < CONFIGS.length; i++) { + String name = getTestGridName(i); + + switch (nodeTypes[i]) { + case REMOTE: + startRemoteGrid(name, getConfiguration(name), + null, null, false); + break; + + case LOCAL: + startGrid(name, getConfiguration(name)); + break; + + case LOCAL_SAME_NAME: + startGrid("IGFS-cli-" + i, getConfiguration(name)); + break; + } + } + } + + /** + * @throws Exception If failed. + */ + public void checkFsOpenMultithreaded() throws Exception { + X.println("Start hadoop client file system open test"); + + final AtomicInteger fileIdx = new AtomicInteger(); + + IgniteInternalFuture fut = multithreadedAsync(new Runnable() { + @Override public void run() { + for (int iter = 0; iter < ITERATIONS; iter++) { + for (int i = 0; i < CONFIGS.length; i++) { + try (FileSystem fs = FileSystem.get(new URI(uri(i)), configuration(i))) { + FSDataOutputStream out = fs.create(new Path(uri(i) + "file" + + fileIdx.getAndIncrement()), true); + + out.close(); + } + catch (Exception e) { + e.printStackTrace(); + + assert false : "Unexpected exception"; + } + } + } + } + }, THREADS_COUNT); + + fut.get(); + } + + /** + * Node type. + */ + enum NodeType { + /** Remote node. */ + REMOTE, + + /** Node in the test VM. */ + LOCAL, + + /** Node in the test VM with the name equals to client node config. */ + LOCAL_SAME_NAME + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java new file mode 100644 index 0000000..e2995fa --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedPrimarySelfTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import org.apache.ignite.igfs.IgfsMode; + +/** + * IGFS Hadoop file system Ignite client -based self test for PRIMARY mode. + */ +public class IgniteHadoopFileSystemClientBasedPrimarySelfTest + extends IgniteHadoopFileSystemClientBasedAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemClientBasedPrimarySelfTest() { + super(IgfsMode.PRIMARY); + } + + /** {@inheritDoc} */ + @Override protected String getClientConfig() { + return "modules/hadoop/src/test/config/igfs-cli-config-primary.xml"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java new file mode 100644 index 0000000..4293b3b --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedProxySelfTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import org.apache.ignite.igfs.IgfsMode; + +/** + * IGFS Hadoop file system Ignite client -based self test for PROXY mode. + */ +public class IgniteHadoopFileSystemClientBasedProxySelfTest extends IgniteHadoopFileSystemClientBasedAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemClientBasedProxySelfTest() { + super(IgfsMode.PROXY); + } + + /** {@inheritDoc} */ + @Override protected String getClientConfig() { + return "modules/hadoop/src/test/config/igfs-cli-config-proxy.xml"; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/6bdff2c3/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java new file mode 100644 index 0000000..b3c1681 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl.igfs; + +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; +import org.apache.ignite.igfs.IgfsIpcEndpointType; +import org.apache.ignite.igfs.IgfsMode; + +import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT; + +/** + * IGFS Hadoop file system IPC loopback self test. + */ +public abstract class IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest extends + IgniteHadoopFileSystemAbstractSelfTest { + + /** + * Constructor. + * + * @param mode IGFS mode. + */ + protected IgniteHadoopFileSystemLoopbackExternalToClientAbstractSelfTest(IgfsMode mode) { + super(mode, true, true); + } + + /** {@inheritDoc} */ + @Override protected IgfsIpcEndpointConfiguration primaryIpcEndpointConfiguration(final String gridName) { + IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration(); + + endpointCfg.setType(IgfsIpcEndpointType.TCP); + endpointCfg.setPort(DFLT_IPC_PORT + getTestGridIndex(gridName)); + + return endpointCfg; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + if (getTestGridIndex(gridName) == 0) + cfg.setClientMode(true); + + return cfg; + } +} \ No newline at end of file
