http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java index 14eb296..2e5708d 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java @@ -20,7 +20,9 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; import org.apache.ignite.internal.ClusterNodeMetricsSelfTest; import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest; +import org.apache.ignite.internal.GridAffinityMappedTest; import org.apache.ignite.internal.GridAffinityNoCacheSelfTest; +import org.apache.ignite.internal.GridAffinityP2PSelfTest; import org.apache.ignite.internal.GridAffinitySelfTest; import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest; import org.apache.ignite.internal.GridCancelOnGridStopSelfTest; @@ -53,6 +55,7 @@ import org.apache.ignite.internal.GridNonHistoryMetricsSelfTest; import org.apache.ignite.internal.GridProjectionLocalJobMultipleArgumentsSelfTest; import org.apache.ignite.internal.GridStopWithWaitSelfTest; import org.apache.ignite.internal.GridTaskCancelSingleNodeSelfTest; +import org.apache.ignite.internal.GridTaskExecutionContextSelfTest; import org.apache.ignite.internal.GridTaskExecutionSelfTest; import org.apache.ignite.internal.GridTaskExecutionWithoutPeerClassLoadingSelfTest; import org.apache.ignite.internal.GridTaskFailoverAffinityRunTest; @@ -109,6 +112,7 @@ public class IgniteComputeGridTestSuite { suite.addTestSuite(GridCancelUnusedJobSelfTest.class); suite.addTestSuite(GridTaskJobRejectSelfTest.class); suite.addTestSuite(GridTaskExecutionSelfTest.class); + //suite.addTestSuite(GridTaskExecutionContextSelfTest.class); suite.addTestSuite(GridTaskExecutionWithoutPeerClassLoadingSelfTest.class); suite.addTestSuite(GridFailoverSelfTest.class); suite.addTestSuite(GridTaskListenerSelfTest.class); @@ -152,6 +156,8 @@ public class IgniteComputeGridTestSuite { suite.addTestSuite(GridProjectionLocalJobMultipleArgumentsSelfTest.class); suite.addTestSuite(GridAffinitySelfTest.class); suite.addTestSuite(GridAffinityNoCacheSelfTest.class); + //suite.addTestSuite(GridAffinityMappedTest.class); + //suite.addTestSuite(GridAffinityP2PSelfTest.class); suite.addTestSuite(GridEventStorageRuntimeConfigurationSelfTest.class); suite.addTestSuite(GridMultinodeRedeployContinuousModeSelfTest.class); suite.addTestSuite(GridMultinodeRedeploySharedModeSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java index 908390f..8f026f7 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java @@ -20,6 +20,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; import org.apache.ignite.igfs.IgfsFragmentizerSelfTest; import org.apache.ignite.igfs.IgfsFragmentizerTopologySelfTest; +import org.apache.ignite.igfs.IgfsPathSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsAtomicPrimaryMultiNodeSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsAtomicPrimarySelfTest; import org.apache.ignite.internal.processors.igfs.IgfsAttributesSelfTest; @@ -122,6 +123,7 @@ public class IgniteIgfsTestSuite extends TestSuite { suite.addTest(new TestSuite(IgfsModeResolverSelfTest.class)); + //suite.addTestSuite(IgfsPathSelfTest.class); suite.addTestSuite(IgfsFragmentizerSelfTest.class); suite.addTestSuite(IgfsFragmentizerTopologySelfTest.class); suite.addTestSuite(IgfsFileMapSelfTest.class); @@ -157,4 +159,4 @@ public class IgniteIgfsTestSuite extends TestSuite { return suite; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java index af9e151..3a943f3 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java @@ -18,11 +18,13 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.ClusterMetricsSelfTest; import org.apache.ignite.internal.ComputeJobCancelWithServiceSelfTest; import org.apache.ignite.internal.GridCommunicationSelfTest; import org.apache.ignite.internal.GridDiscoveryEventSelfTest; import org.apache.ignite.internal.GridDiscoverySelfTest; import org.apache.ignite.internal.GridFailedInputParametersSelfTest; +import org.apache.ignite.internal.GridGetOrStartSelfTest; import org.apache.ignite.internal.GridHomePathSelfTest; import org.apache.ignite.internal.GridKernalConcurrentAccessStopSelfTest; import org.apache.ignite.internal.GridListenActorSelfTest; @@ -33,6 +35,7 @@ import org.apache.ignite.internal.GridNodeVisorAttributesSelfTest; import org.apache.ignite.internal.GridRuntimeExceptionSelfTest; import org.apache.ignite.internal.GridSameVmStartupSelfTest; import org.apache.ignite.internal.GridSpiExceptionSelfTest; +import org.apache.ignite.internal.GridStartupTest; import org.apache.ignite.internal.GridVersionSelfTest; import org.apache.ignite.internal.IgniteConcurrentEntryProcessorAccessStopTest; import org.apache.ignite.internal.IgniteConnectionConcurrentReserveAndRemoveTest; @@ -51,6 +54,7 @@ import org.apache.ignite.internal.processors.port.GridPortProcessorSelfTest; import org.apache.ignite.internal.processors.service.GridServiceClientNodeTest; import org.apache.ignite.internal.processors.service.GridServiceContinuousQueryRedeployTest; import org.apache.ignite.internal.processors.service.GridServiceDeploymentCompoundFutureSelfTest; +import org.apache.ignite.internal.processors.service.GridServiceDeploymentExceptionPropagationTest; import org.apache.ignite.internal.processors.service.GridServicePackagePrivateSelfTest; import org.apache.ignite.internal.processors.service.GridServiceProcessorBatchDeploySelfTest; import org.apache.ignite.internal.processors.service.GridServiceProcessorMultiNodeConfigSelfTest; @@ -64,8 +68,10 @@ import org.apache.ignite.internal.processors.service.GridServiceReassignmentSelf import org.apache.ignite.internal.processors.service.GridServiceSerializationSelfTest; import org.apache.ignite.internal.processors.service.IgniteServiceDeployment2ClassLoadersDefaultMarshallerTest; import org.apache.ignite.internal.processors.service.IgniteServiceDeployment2ClassLoadersJdkMarshallerTest; +import org.apache.ignite.internal.processors.service.IgniteServiceDeployment2ClassLoadersOptimizedMarshallerTest; import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingDefaultMarshallerTest; import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingJdkMarshallerTest; +import org.apache.ignite.internal.processors.service.IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest; import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesSelfTest; import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest; import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest; @@ -98,6 +104,8 @@ public class IgniteKernalSelfTestSuite extends TestSuite { public static TestSuite suite(Set<Class> ignoredTests) throws Exception { TestSuite suite = new TestSuite("Ignite Kernal Test Suite"); + //suite.addTestSuite(GridStartupTest.class); + //suite.addTestSuite(GridGetOrStartSelfTest.class); suite.addTestSuite(GridSameVmStartupSelfTest.class); suite.addTestSuite(GridSpiExceptionSelfTest.class); suite.addTestSuite(GridRuntimeExceptionSelfTest.class); @@ -131,6 +139,7 @@ public class IgniteKernalSelfTestSuite extends TestSuite { suite.addTestSuite(ComputeJobCancelWithServiceSelfTest.class); suite.addTestSuite(IgniteConnectionConcurrentReserveAndRemoveTest.class); suite.addTestSuite(LongJVMPauseDetectorTest.class); + //suite.addTestSuite(ClusterMetricsSelfTest.class); // Managed Services. suite.addTestSuite(GridServiceProcessorSingleNodeSelfTest.class); @@ -153,11 +162,14 @@ public class IgniteKernalSelfTestSuite extends TestSuite { suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class); suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class); suite.addTestSuite(SystemCacheNotConfiguredTest.class); + //suite.addTestSuite(GridServiceDeploymentExceptionPropagationTest.class); suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class); suite.addTestSuite(IgniteServiceDeploymentClassLoadingJdkMarshallerTest.class); + //suite.addTestSuite(IgniteServiceDeploymentClassLoadingOptimizedMarshallerTest.class); suite.addTestSuite(IgniteServiceDeployment2ClassLoadersDefaultMarshallerTest.class); suite.addTestSuite(IgniteServiceDeployment2ClassLoadersJdkMarshallerTest.class); + //suite.addTestSuite(IgniteServiceDeployment2ClassLoadersOptimizedMarshallerTest.class); return suite; } http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java index 70aec72..187ca59 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLangSelfTestSuite.java @@ -42,6 +42,7 @@ import org.apache.ignite.lang.utils.GridLeanIdentitySetSelfTest; import org.apache.ignite.lang.utils.GridLeanMapSelfTest; import org.apache.ignite.lang.utils.GridListSetSelfTest; import org.apache.ignite.lang.utils.GridStripedLockSelfTest; +import org.apache.ignite.lang.utils.IgniteOffheapReadWriteLockSelfTest; import org.apache.ignite.util.GridConcurrentLinkedDequeSelfTest; import org.apache.ignite.util.GridConcurrentLinkedHashMapMultiThreadedSelfTest; @@ -85,6 +86,8 @@ public class IgniteLangSelfTestSuite extends TestSuite { suite.addTest(new TestSuite(IgniteFutureImplTest.class)); suite.addTest(new TestSuite(IgniteCacheFutureImplTest.class)); + //suite.addTest(new TestSuite(IgniteOffheapReadWriteLockSelfTest.class)); + // Consistent hash tests. suite.addTest(new TestSuite(GridConsistentHashSelfTest.class)); http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLostAndFoundTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLostAndFoundTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLostAndFoundTestSuite.java new file mode 100644 index 0000000..7fc40df --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteLostAndFoundTestSuite.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.GridFactoryVmShutdownTest; +import org.apache.ignite.internal.managers.GridManagerMxBeanIllegalArgumentHandleTest; +import org.apache.ignite.internal.processors.cache.datastructures.GridCacheMultiNodeDataStructureTest; +import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadUndeploysTest; +import org.apache.ignite.internal.processors.cache.persistence.file.FileDownloaderTest; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSandboxTest; +import org.apache.ignite.internal.processors.compute.GridComputeJobExecutionErrorToLogManualTest; +import org.apache.ignite.internal.util.future.GridFutureQueueTest; +import org.apache.ignite.internal.util.nio.GridRoundTripTest; +import org.apache.ignite.jvmtest.BlockingQueueTest; +import org.apache.ignite.jvmtest.FileIOTest; +import org.apache.ignite.jvmtest.FileLocksTest; +import org.apache.ignite.jvmtest.LinkedHashMapTest; +import org.apache.ignite.jvmtest.MultipleFileIOTest; +import org.apache.ignite.jvmtest.NetworkFailureTest; +import org.apache.ignite.jvmtest.QueueSizeCounterMultiThreadedTest; +import org.apache.ignite.jvmtest.ReadWriteLockMultiThreadedTest; +import org.apache.ignite.jvmtest.RegExpTest; +import org.apache.ignite.jvmtest.ServerSocketMultiThreadedTest; +import org.apache.ignite.lang.GridSystemCurrentTimeMillisTest; +import org.apache.ignite.lang.GridThreadPriorityTest; +import org.apache.ignite.startup.servlet.GridServletLoaderTest; + +/** + * Tests suite for orphaned tests. + */ +public class IgniteLostAndFoundTestSuite extends TestSuite { + /** + * @return Tests suite for orphaned tests (not in any test sute previously). + */ + public static TestSuite suite() { + TestSuite suite = new TestSuite("Ignite List And Found Test Suite"); + + suite.addTestSuite(FileDownloaderTest.class); + suite.addTestSuite(FileIOTest.class); + suite.addTestSuite(FileLocksTest.class); + suite.addTestSuite(GridComputeJobExecutionErrorToLogManualTest.class); + suite.addTestSuite(GridManagerMxBeanIllegalArgumentHandleTest.class); + suite.addTestSuite(GridRoundTripTest.class); + suite.addTestSuite(GridServletLoaderTest.class); + + suite.addTestSuite(LinkedHashMapTest.class); + suite.addTestSuite(NetworkFailureTest.class); + suite.addTestSuite(PagesWriteThrottleSandboxTest.class); + suite.addTestSuite(QueueSizeCounterMultiThreadedTest.class); + suite.addTestSuite(ReadWriteLockMultiThreadedTest.class); + suite.addTestSuite(RegExpTest.class); + suite.addTestSuite(ServerSocketMultiThreadedTest.class); + + + // Non-JUnit classes with Test in name, which should be either converted to JUnit or removed in the future + // Main classes: + Class[] _$ = new Class[] { + GridCacheReplicatedPreloadUndeploysTest.class, + GridCacheMultiNodeDataStructureTest.class, + GridFactoryVmShutdownTest.class, + GridFutureQueueTest.class, + GridThreadPriorityTest.class, + GridSystemCurrentTimeMillisTest.class, + BlockingQueueTest.class, + MultipleFileIOTest.class + }; + + return suite; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java index 524283b..7218174 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java @@ -33,6 +33,7 @@ import org.apache.ignite.p2p.GridP2PNodeLeftSelfTest; import org.apache.ignite.p2p.GridP2PRecursionTaskSelfTest; import org.apache.ignite.p2p.GridP2PRemoteClassLoadersSelfTest; import org.apache.ignite.p2p.GridP2PSameClassLoaderSelfTest; +import org.apache.ignite.p2p.GridP2PTestTaskExecutionTest; import org.apache.ignite.p2p.GridP2PTimeoutSelfTest; import org.apache.ignite.p2p.GridP2PUndeploySelfTest; import org.apache.ignite.p2p.P2PStreamingClassLoaderTest; @@ -70,6 +71,7 @@ public class IgniteP2PSelfTestSuite extends TestSuite { suite.addTest(new TestSuite(GridP2PJobClassLoaderSelfTest.class)); suite.addTest(new TestSuite(GridP2PRecursionTaskSelfTest.class)); suite.addTest(new TestSuite(GridP2PLocalDeploymentSelfTest.class)); + //suite.addTest(new TestSuite(GridP2PTestTaskExecutionTest.class)); suite.addTest(new TestSuite(GridP2PTimeoutSelfTest.class)); suite.addTest(new TestSuite(GridP2PMissedResourceCacheSizeSelfTest.class)); suite.addTest(new TestSuite(GridP2PContinuousDeploymentSelfTest.class)); http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java index d445d9b..25315ea 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java @@ -18,16 +18,20 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.internal.pagemem.impl.PageMemoryNoLoadSelfTest; import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest; +import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRemoveDuringRebalancingTest; import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest; +import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistenceSequentialCheckpointTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsDataRegionMetricsTest; import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWithTtlTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest; +import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimpleTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest; import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsPageReplacementTest; import org.apache.ignite.internal.processors.cache.persistence.metastorage.IgniteMetaStorageBasicTest; @@ -35,10 +39,12 @@ import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTree import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.FillFactorMetricTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.IndexStoragePageMemoryImplTest; +import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageIdDistributionTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryNoStoreLeakTest; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIOTest; import org.apache.ignite.internal.processors.cache.persistence.wal.CpTriggeredWalDeltaConsistencyTest; import org.apache.ignite.internal.processors.cache.persistence.wal.ExplicitWalDeltaConsistencyTest; import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBufferTest; @@ -64,10 +70,13 @@ public class IgnitePdsTestSuite extends TestSuite { addRealPageStoreTestsLongRunning(suite); // Basic PageMemory tests. + //suite.addTestSuite(PageMemoryNoLoadSelfTest.class); suite.addTestSuite(PageMemoryImplNoLoadTest.class); suite.addTestSuite(PageMemoryNoStoreLeakTest.class); suite.addTestSuite(IndexStoragePageMemoryImplTest.class); suite.addTestSuite(PageMemoryImplTest.class); + //suite.addTestSuite(PageIdDistributionTest.class); + //suite.addTestSuite(TrackingPageIOTest.class); // BTree tests with store page memory. suite.addTestSuite(BPlusTreePageMemoryImplTest.class); @@ -111,6 +120,8 @@ public class IgnitePdsTestSuite extends TestSuite { // Checkpointing smoke-test. suite.addTestSuite(IgnitePdsCheckpointSimulationWithRealCpDisabledTest.class); + //suite.addTestSuite(IgnitePdsCheckpointSimpleTest.class); + //suite.addTestSuite(IgnitePersistenceSequentialCheckpointTest.class); // Basic API tests. suite.addTestSuite(IgniteDbSingleNodePutGetTest.class); @@ -132,6 +143,7 @@ public class IgnitePdsTestSuite extends TestSuite { suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class); suite.addTestSuite(IgnitePdsDestroyCacheTest.class); + //suite.addTestSuite(IgnitePdsRemoveDuringRebalancingTest.class); suite.addTestSuite(IgnitePdsDestroyCacheWithoutCheckpointsTest.class); suite.addTestSuite(IgnitePdsCacheConfigurationFileConsistencyCheckTest.class); http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index 066a12a..e8a8576 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -58,6 +58,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalF import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorExceptionDuringReadTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoverySeveralRestartsTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests; @@ -188,6 +189,8 @@ public class IgnitePdsTestSuite2 extends TestSuite { suite.addTestSuite(IgniteNodeStoppedDuringDisableWALTest.class); + //suite.addTestSuite(IgniteWalRecoverySeveralRestartsTest.class); + suite.addTestSuite(IgniteRebalanceScheduleResendPartitionsTest.class); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java new file mode 100644 index 0000000..2e82957 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePerformanceTestSuite.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.internal.processors.cache.GridCacheConcurrentTxMultiNodeLoadTest; +import org.apache.ignite.internal.processors.cache.GridCacheIteratorPerformanceTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDhtPreloadPerformanceTest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedAffinityExcludeNeighborsPerformanceTest; +import org.apache.ignite.internal.processors.cache.eviction.sorted.SortedEvictionPolicyPerformanceTest; +import org.apache.ignite.internal.processors.datastreamer.IgniteDataStreamerPerformanceTest; +import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMapPerformanceTest; +import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMapPerformanceTest; +import org.apache.ignite.lang.GridBasicPerformanceTest; +import org.apache.ignite.lang.GridFuncPerformanceTest; +import org.apache.ignite.lang.GridFutureListenPerformanceTest; +import org.apache.ignite.lang.GridMetadataAwareAdapterLoadTest; +import org.apache.ignite.lang.utils.GridCircularBufferPerformanceTest; +import org.apache.ignite.lang.utils.GridLeanMapPerformanceTest; +import org.apache.ignite.loadtests.GridCacheMultiNodeLoadTest; +import org.apache.ignite.loadtests.GridSingleExecutionTest; +import org.apache.ignite.loadtests.cache.GridCacheDataStructuresLoadTest; +import org.apache.ignite.loadtests.cache.GridCacheLoadTest; +import org.apache.ignite.loadtests.cache.GridCacheWriteBehindStoreLoadTest; +import org.apache.ignite.loadtests.capacity.GridCapacityLoadTest; +import org.apache.ignite.loadtests.continuous.GridContinuousOperationsLoadTest; +import org.apache.ignite.loadtests.datastructures.GridCachePartitionedAtomicLongLoadTest; +import org.apache.ignite.loadtests.direct.multisplit.GridMultiSplitsLoadTest; +import org.apache.ignite.loadtests.direct.multisplit.GridMultiSplitsRedeployLoadTest; +import org.apache.ignite.loadtests.direct.newnodes.GridSingleSplitsNewNodesMulticastLoadTest; +import org.apache.ignite.loadtests.direct.redeploy.GridSingleSplitsRedeployLoadTest; +import org.apache.ignite.loadtests.direct.session.GridSessionLoadTest; +import org.apache.ignite.loadtests.direct.stealing.GridStealingLoadTest; +import org.apache.ignite.loadtests.discovery.GridGcTimeoutTest; +import org.apache.ignite.loadtests.dsi.cacheget.GridBenchmarkCacheGetLoadTest; +import org.apache.ignite.loadtests.hashmap.GridBoundedConcurrentLinkedHashSetLoadTest; +import org.apache.ignite.loadtests.hashmap.GridHashMapLoadTest; +import org.apache.ignite.loadtests.job.GridJobExecutionSingleNodeLoadTest; +import org.apache.ignite.loadtests.job.GridJobExecutionSingleNodeSemaphoreLoadTest; +import org.apache.ignite.loadtests.job.GridJobLoadTest; +import org.apache.ignite.loadtests.mergesort.GridMergeSortLoadTest; +import org.apache.ignite.loadtests.nio.GridNioBenchmarkTest; +import org.apache.ignite.marshaller.GridMarshallerPerformanceTest; +import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiLanLoadTest; + +/** + * Tests suite for performance tests tests. + * Note: Most of these are resource-consuming or non-terminating. + */ +public class IgnitePerformanceTestSuite extends TestSuite { + /** + * @return Tests suite for orphaned tests (not in any test sute previously). + */ + public static TestSuite suite() { + TestSuite suite = new TestSuite("Ignite Load-Test Suite"); + + suite.addTestSuite(GridCacheDhtPreloadPerformanceTest.class); + suite.addTestSuite(GridCacheIteratorPerformanceTest.class); + suite.addTestSuite(GridCacheMultiNodeLoadTest.class); + suite.addTestSuite(GridCacheConcurrentTxMultiNodeLoadTest.class); + suite.addTestSuite(GridCachePartitionedAffinityExcludeNeighborsPerformanceTest.class); + suite.addTestSuite(GridCachePartitionedAtomicLongLoadTest.class); + suite.addTestSuite(GridCacheWriteBehindStoreLoadTest.class); + suite.addTestSuite(GridCircularBufferPerformanceTest.class); + suite.addTestSuite(GridFuncPerformanceTest.class); + suite.addTestSuite(GridHashMapLoadTest.class); + suite.addTestSuite(GridLeanMapPerformanceTest.class); + suite.addTestSuite(GridMarshallerPerformanceTest.class); + suite.addTestSuite(GridMetadataAwareAdapterLoadTest.class); + suite.addTestSuite(GridMultiSplitsLoadTest.class); + suite.addTestSuite(GridMultiSplitsRedeployLoadTest.class); + suite.addTestSuite(GridSessionLoadTest.class); + suite.addTestSuite(GridSingleSplitsNewNodesMulticastLoadTest.class); + suite.addTestSuite(GridSingleSplitsRedeployLoadTest.class); + suite.addTestSuite(GridStealingLoadTest.class); + suite.addTestSuite(GridTcpCommunicationSpiLanLoadTest.class); + suite.addTestSuite(GridUnsafeMapPerformanceTest.class); + suite.addTestSuite(GridUnsafePartitionedMapPerformanceTest.class); + suite.addTestSuite(IgniteDataStreamerPerformanceTest.class); + suite.addTestSuite(SortedEvictionPolicyPerformanceTest.class); + + // Non-JUnit classes with Test in name, which should be either converted to JUnit or removed in the future + // Main classes: + Class[] _$ = new Class[] { + GridBasicPerformanceTest.class, + GridBenchmarkCacheGetLoadTest.class, + GridBoundedConcurrentLinkedHashSetLoadTest.class, + GridCacheDataStructuresLoadTest.class, + GridCacheLoadTest.class, + GridCapacityLoadTest.class, + GridContinuousOperationsLoadTest.class, + GridFutureListenPerformanceTest.class, + GridGcTimeoutTest.class, + GridJobExecutionSingleNodeLoadTest.class, + GridJobExecutionSingleNodeSemaphoreLoadTest.class, + GridJobLoadTest.class, + GridMergeSortLoadTest.class, + GridNioBenchmarkTest.class, + GridSingleExecutionTest.class + }; + + return suite; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCheckpointSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCheckpointSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCheckpointSelfTestSuite.java index acb63b7..a3ba8ff 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCheckpointSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCheckpointSelfTestSuite.java @@ -27,6 +27,7 @@ import org.apache.ignite.spi.checkpoint.jdbc.JdbcCheckpointSpiCustomConfigSelfTe import org.apache.ignite.spi.checkpoint.jdbc.JdbcCheckpointSpiDefaultConfigSelfTest; import org.apache.ignite.spi.checkpoint.jdbc.JdbcCheckpointSpiStartStopSelfTest; import org.apache.ignite.spi.checkpoint.sharedfs.GridSharedFsCheckpointSpiConfigSelfTest; +import org.apache.ignite.spi.checkpoint.sharedfs.GridSharedFsCheckpointSpiMultiThreadedSelfTest; import org.apache.ignite.spi.checkpoint.sharedfs.GridSharedFsCheckpointSpiMultipleDirectoriesSelfTest; import org.apache.ignite.spi.checkpoint.sharedfs.GridSharedFsCheckpointSpiSelfTest; import org.apache.ignite.spi.checkpoint.sharedfs.GridSharedFsCheckpointSpiStartStopSelfTest; @@ -59,7 +60,8 @@ public class IgniteSpiCheckpointSelfTestSuite extends TestSuite { suite.addTest(new TestSuite(GridSharedFsCheckpointSpiSelfTest.class)); suite.addTest(new TestSuite(GridSharedFsCheckpointSpiStartStopSelfTest.class)); suite.addTest(new TestSuite(GridSharedFsCheckpointSpiConfigSelfTest.class)); + //suite.addTest(new TestSuite(GridSharedFsCheckpointSpiMultiThreadedSelfTest.class)); return suite; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java index 1b962bc..ef55d36 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; +import org.apache.ignite.spi.communication.tcp.GridCacheDhtLockBackupSelfTest; import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiConcurrentConnectSelfTest; import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiConcurrentConnectSslSelfTest; import org.apache.ignite.spi.communication.tcp.GridTcpCommunicationSpiConfigSelfTest; @@ -87,6 +88,8 @@ public class IgniteSpiCommunicationSelfTestSuite extends TestSuite { suite.addTest(new TestSuite(TcpCommunicationStatisticsTest.class)); + //suite.addTest(new TestSuite(GridCacheDhtLockBackupSelfTest.class)); + return suite; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java index 4b68af4..9ba9d2d 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java @@ -30,6 +30,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoveryMarshallerCheckSelf import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest; import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryConcurrentStartTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest; @@ -43,9 +44,12 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySnapshotHistoryTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiConfigSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiMBeanTest; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiRandomStartStopTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiReconnectDelayTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSslSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiWildcardSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslTrustedSelfTest; @@ -81,6 +85,9 @@ public class IgniteSpiDiscoverySelfTestSuite extends TestSuite { suite.addTest(new TestSuite(TcpDiscoverySelfTest.class)); suite.addTest(new TestSuite(TcpDiscoverySpiSelfTest.class)); + //suite.addTest(new TestSuite(TcpDiscoverySpiRandomStartStopTest.class)); + //suite.addTest(new TestSuite(TcpDiscoverySpiSslSelfTest.class)); + //suite.addTest(new TestSuite(TcpDiscoverySpiWildcardSelfTest.class)); suite.addTest(new TestSuite(TcpDiscoverySpiFailureTimeoutSelfTest.class)); suite.addTest(new TestSuite(TcpDiscoverySpiMBeanTest.class)); suite.addTest(new TestSuite(TcpDiscoverySpiStartStopSelfTest.class)); @@ -100,6 +107,7 @@ public class IgniteSpiDiscoverySelfTestSuite extends TestSuite { suite.addTest(new TestSuite(TcpDiscoveryRestartTest.class)); suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class)); + //suite.addTest(new TestSuite(TcpDiscoveryConcurrentStartTest.class)); suite.addTest(new TestSuite(TcpDiscoverySegmentationPolicyTest.class)); http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStartUpTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStartUpTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStartUpTestSuite.java index e637dcd..b29f5c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStartUpTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStartUpTestSuite.java @@ -35,4 +35,4 @@ public class IgniteStartUpTestSuite extends TestSuite { return suite; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java index 309af8c..c0d7a8b 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.commandline.CommandHandlerParsingTest; import org.apache.ignite.internal.pagemem.impl.PageIdUtilsSelfTest; import org.apache.ignite.internal.processors.cache.GridCacheUtilsSelfTest; import org.apache.ignite.internal.util.GridArraysSelfTest; +import org.apache.ignite.internal.util.IgniteDevOnlyLogTest; import org.apache.ignite.internal.util.IgniteExceptionRegistrySelfTest; import org.apache.ignite.internal.util.IgniteUtilsSelfTest; import org.apache.ignite.internal.util.nio.GridNioDelimitedBufferSelfTest; @@ -35,18 +36,28 @@ import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafePartitionedMapSe import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeStripedLruSefTest; import org.apache.ignite.internal.util.tostring.CircularStringBuilderSelfTest; import org.apache.ignite.internal.util.tostring.GridToStringBuilderSelfTest; +import org.apache.ignite.internal.util.tostring.IncludeSensitiveAtomicTest; +import org.apache.ignite.internal.util.tostring.IncludeSensitiveTransactionalTest; import org.apache.ignite.lang.GridByteArrayListSelfTest; import org.apache.ignite.spi.discovery.ClusterMetricsSnapshotSerializeSelfTest; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest; +import org.apache.ignite.thread.GridThreadTest; import org.apache.ignite.thread.IgniteThreadPoolSizeTest; +import org.apache.ignite.util.GridConcurrentLinkedDequeMultiThreadedTest; +import org.apache.ignite.util.GridIndexFillTest; import org.apache.ignite.util.GridIntListSelfTest; +import org.apache.ignite.util.GridLogThrottleTest; import org.apache.ignite.util.GridLongListSelfTest; import org.apache.ignite.util.GridMessageCollectionTest; import org.apache.ignite.util.GridPartitionMapSelfTest; import org.apache.ignite.util.GridQueueSelfTest; +import org.apache.ignite.util.GridRandomSelfTest; +import org.apache.ignite.util.GridSnapshotLockSelfTest; import org.apache.ignite.util.GridSpinReadWriteLockSelfTest; import org.apache.ignite.util.GridStringBuilderFactorySelfTest; +import org.apache.ignite.util.GridTopologyHeapSizeSelfTest; +import org.apache.ignite.util.GridTransientTest; import org.apache.ignite.util.mbeans.GridMBeanDisableSelfTest; import org.apache.ignite.util.mbeans.GridMBeanExoticNamesSelfTest; import org.apache.ignite.util.mbeans.GridMBeanSelfTest; @@ -87,12 +98,25 @@ public class IgniteUtilSelfTestSuite extends TestSuite { suite.addTestSuite(GridMBeanDisableSelfTest.class); suite.addTestSuite(GridMBeanExoticNamesSelfTest.class); suite.addTestSuite(GridLongListSelfTest.class); + //suite.addTestSuite(GridThreadTest.class); suite.addTestSuite(GridIntListSelfTest.class); suite.addTestSuite(GridArraysSelfTest.class); suite.addTestSuite(GridCacheUtilsSelfTest.class); suite.addTestSuite(IgniteExceptionRegistrySelfTest.class); suite.addTestSuite(GridMessageCollectionTest.class); suite.addTestSuite(WorkersControlMXBeanTest.class); + //suite.addTestSuite(GridConcurrentLinkedDequeMultiThreadedTest.class); + //suite.addTestSuite(GridIndexFillTest.class); + //suite.addTestSuite(GridLogThrottleTest.class); + //suite.addTestSuite(GridRandomSelfTest.class); + //suite.addTestSuite(GridSnapshotLockSelfTest.class); + //suite.addTestSuite(GridTopologyHeapSizeSelfTest.class); + //suite.addTestSuite(GridTransientTest.class); + //suite.addTestSuite(IgniteDevOnlyLogTest.class); + + // Sensitive toString. + //suite.addTestSuite(IncludeSensitiveAtomicTest.class); + //suite.addTestSuite(IncludeSensitiveTransactionalTest.class); // Metrics. suite.addTestSuite(ClusterMetricsSnapshotSerializeSelfTest.class); http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java index 337d4d4..0bd9584 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/TxDeadlockDetectionTestSuite.java @@ -19,6 +19,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.transactions.DepthFirstSearchTest; +import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockCauseTest; import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionMessageMarshallingTest; import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionNoHangsTest; import org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetectionTest; @@ -44,6 +45,7 @@ public class TxDeadlockDetectionTestSuite extends TestSuite { suite.addTestSuite(TxOptimisticDeadlockDetectionCrossCacheTest.class); suite.addTestSuite(TxPessimisticDeadlockDetectionTest.class); suite.addTestSuite(TxPessimisticDeadlockDetectionCrossCacheTest.class); + //suite.addTestSuite(TxDeadlockCauseTest.class); suite.addTestSuite(TxDeadlockDetectionTest.class); suite.addTestSuite(TxDeadlockDetectionNoHangsTest.class); suite.addTestSuite(TxDeadlockDetectionUnmasrhalErrorsTest.class); http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopPopularWordsTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopPopularWordsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopPopularWordsTest.java deleted file mode 100644 index 8d46483..0000000 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopPopularWordsTest.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.impl; - -import com.google.common.collect.MinMaxPriorityQueue; -import java.io.IOException; -import java.util.Comparator; -import java.util.Map.Entry; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; -import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.ignite.internal.util.typedef.X; -import org.apache.ignite.internal.util.typedef.internal.U; - -import static com.google.common.collect.Maps.immutableEntry; -import static com.google.common.collect.MinMaxPriorityQueue.orderedBy; -import static java.util.Collections.reverseOrder; - -/** - * Hadoop-based 10 popular words example: all files in a given directory are tokenized and for each word longer than - * 3 characters the number of occurrences ins calculated. Finally, 10 words with the highest occurrence count are - * output. - * - * NOTE: in order to run this example on Windows please ensure that cygwin is installed and available in the system - * path. - */ -public class HadoopPopularWordsTest { - /** Ignite home. */ - private static final String IGNITE_HOME = U.getIgniteHome(); - - /** The path to the input directory. ALl files in that directory will be processed. */ - private static final Path BOOKS_LOCAL_DIR = - new Path("file:" + IGNITE_HOME, "modules/tests/java/org/apache/ignite/grid/hadoop/books"); - - /** The path to the output directory. THe result file will be written to this location. */ - private static final Path RESULT_LOCAL_DIR = - new Path("file:" + IGNITE_HOME, "modules/tests/java/org/apache/ignite/grid/hadoop/output"); - - /** Popular books source dir in DFS. */ - private static final Path BOOKS_DFS_DIR = new Path("tmp/word-count-example/in"); - - /** Popular books source dir in DFS. */ - private static final Path RESULT_DFS_DIR = new Path("tmp/word-count-example/out"); - - /** Path to the distributed file system configuration. */ - private static final String DFS_CFG = "examples/config/filesystem/core-site.xml"; - - /** Top N words to select **/ - private static final int POPULAR_WORDS_CNT = 10; - - /** - * For each token in the input string the mapper emits a {word, 1} pair. - */ - private static class TokenizingMapper extends Mapper<LongWritable, Text, Text, IntWritable> { - /** Constant value. */ - private static final IntWritable ONE = new IntWritable(1); - - /** The word converted into the Text. */ - private Text word = new Text(); - - /** - * Emits a entry where the key is the word and the value is always 1. - * - * @param key the current position in the input file (not used here) - * @param val the text string - * @param ctx mapper context - * @throws IOException - * @throws InterruptedException - */ - @Override protected void map(LongWritable key, Text val, Context ctx) - throws IOException, InterruptedException { - // Get the mapped object. - final String line = val.toString(); - - // Splits the given string to words. - final String[] words = line.split("[^a-zA-Z0-9]"); - - for (final String w : words) { - // Only emit counts for longer words. - if (w.length() <= 3) - continue; - - word.set(w); - - // Write the word into the context with the initial count equals 1. - ctx.write(word, ONE); - } - } - } - - /** - * The reducer uses a priority queue to rank the words based on its number of occurrences. - */ - private static class TopNWordsReducer extends Reducer<Text, IntWritable, Text, IntWritable> { - private MinMaxPriorityQueue<Entry<Integer, String>> q; - - TopNWordsReducer() { - q = orderedBy(reverseOrder(new Comparator<Entry<Integer, String>>() { - @Override public int compare(Entry<Integer, String> o1, Entry<Integer, String> o2) { - return o1.getKey().compareTo(o2.getKey()); - } - })).expectedSize(POPULAR_WORDS_CNT).maximumSize(POPULAR_WORDS_CNT).create(); - } - - /** - * This method doesn't emit anything, but just keeps track of the top N words. - * - * @param key The word. - * @param vals The words counts. - * @param ctx Reducer context. - * @throws IOException If failed. - * @throws InterruptedException If failed. - */ - @Override public void reduce(Text key, Iterable<IntWritable> vals, Context ctx) throws IOException, - InterruptedException { - int sum = 0; - - for (IntWritable val : vals) - sum += val.get(); - - q.add(immutableEntry(sum, key.toString())); - } - - /** - * This method is called after all the word entries have been processed. It writes the accumulated - * statistics to the job output file. - * - * @param ctx The job context. - * @throws IOException If failed. - * @throws InterruptedException If failed. - */ - @Override protected void cleanup(Context ctx) throws IOException, InterruptedException { - IntWritable i = new IntWritable(); - - Text txt = new Text(); - - // iterate in desc order - while (!q.isEmpty()) { - Entry<Integer, String> e = q.removeFirst(); - - i.set(e.getKey()); - - txt.set(e.getValue()); - - ctx.write(txt, i); - } - } - } - - /** - * Configures the Hadoop MapReduce job. - * - * @return Instance of the Hadoop MapRed job. - * @throws IOException If failed. - */ - @SuppressWarnings("deprecation") - private Job createConfigBasedHadoopJob() throws IOException { - Job jobCfg = new Job(); - - Configuration cfg = jobCfg.getConfiguration(); - - // Use explicit configuration of distributed file system, if provided. - cfg.addResource(U.resolveIgniteUrl(DFS_CFG)); - - jobCfg.setJobName("HadoopPopularWordExample"); - jobCfg.setJarByClass(HadoopPopularWordsTest.class); - jobCfg.setInputFormatClass(TextInputFormat.class); - jobCfg.setOutputKeyClass(Text.class); - jobCfg.setOutputValueClass(IntWritable.class); - jobCfg.setMapperClass(TokenizingMapper.class); - jobCfg.setReducerClass(TopNWordsReducer.class); - - FileInputFormat.setInputPaths(jobCfg, BOOKS_DFS_DIR); - FileOutputFormat.setOutputPath(jobCfg, RESULT_DFS_DIR); - - // Local job tracker allows the only task per wave, but text input format - // replaces it with the calculated value based on input split size option. - if ("local".equals(cfg.get("mapred.job.tracker", "local"))) { - // Split job into tasks using 32MB split size. - FileInputFormat.setMinInputSplitSize(jobCfg, 32L * 1024 * 1024); - FileInputFormat.setMaxInputSplitSize(jobCfg, Long.MAX_VALUE); - } - - return jobCfg; - } - - /** - * Runs the Hadoop job. - * - * @return {@code True} if succeeded, {@code false} otherwise. - * @throws Exception If failed. - */ - private boolean runWordCountConfigBasedHadoopJob() throws Exception { - Job job = createConfigBasedHadoopJob(); - - // Distributed file system this job will work with. - FileSystem fs = FileSystem.get(job.getConfiguration()); - - X.println(">>> Using distributed file system: " + fs.getHomeDirectory()); - - // Prepare input and output job directories. - prepareDirectories(fs); - - long time = System.currentTimeMillis(); - - // Run job. - boolean res = job.waitForCompletion(true); - - X.println(">>> Job execution time: " + (System.currentTimeMillis() - time) / 1000 + " sec."); - - // Move job results into local file system, so you can view calculated results. - publishResults(fs); - - return res; - } - - /** - * Prepare job's data: cleanup result directories that might have left over - * after previous runs, copy input files from the local file system into DFS. - * - * @param fs Distributed file system to use in job. - * @throws IOException If failed. - */ - private void prepareDirectories(FileSystem fs) throws IOException { - X.println(">>> Cleaning up DFS result directory: " + RESULT_DFS_DIR); - - fs.delete(RESULT_DFS_DIR, true); - - X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR); - - fs.delete(BOOKS_DFS_DIR, true); - - X.println(">>> Copy local files into DFS input directory: " + BOOKS_DFS_DIR); - - fs.copyFromLocalFile(BOOKS_LOCAL_DIR, BOOKS_DFS_DIR); - } - - /** - * Publish job execution results into local file system, so you can view them. - * - * @param fs Distributed file sytem used in job. - * @throws IOException If failed. - */ - private void publishResults(FileSystem fs) throws IOException { - X.println(">>> Cleaning up DFS input directory: " + BOOKS_DFS_DIR); - - fs.delete(BOOKS_DFS_DIR, true); - - X.println(">>> Cleaning up LOCAL result directory: " + RESULT_LOCAL_DIR); - - fs.delete(RESULT_LOCAL_DIR, true); - - X.println(">>> Moving job results into LOCAL result directory: " + RESULT_LOCAL_DIR); - - fs.copyToLocalFile(true, RESULT_DFS_DIR, RESULT_LOCAL_DIR); - } - - /** - * Executes a modified version of the Hadoop word count example. Here, in addition to counting the number of - * occurrences of the word in the source files, the N most popular words are selected. - * - * @param args None. - */ - public static void main(String[] args) { - try { - new HadoopPopularWordsTest().runWordCountConfigBasedHadoopJob(); - } - catch (Exception e) { - X.println(">>> Failed to run word count example: " + e.getMessage()); - } - - System.exit(0); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksAllVersionsTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksAllVersionsTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksAllVersionsTest.java deleted file mode 100644 index 5d34989..0000000 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksAllVersionsTest.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.hadoop.impl; - -import com.google.common.base.Joiner; -import java.io.IOException; -import java.io.PrintWriter; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.igfs.IgfsPath; -import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock; -import org.apache.ignite.internal.processors.hadoop.HadoopJobEx; -import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo; -import org.apache.ignite.internal.processors.hadoop.HadoopTaskType; -import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount2; - -/** - * Tests of Map, Combine and Reduce task executions of any version of hadoop API. - */ -abstract class HadoopTasksAllVersionsTest extends HadoopAbstractWordCountTest { - /** Empty hosts array. */ - private static final String[] HOSTS = new String[0]; - - /** - * Creates some grid hadoop job. Override this method to create tests for any job implementation. - * - * @param inFile Input file name for the job. - * @param outFile Output file name for the job. - * @return Hadoop job. - * @throws IOException If fails. - */ - public abstract HadoopJobEx getHadoopJob(String inFile, String outFile) throws Exception; - - /** - * @return prefix of reducer output file name. It's "part-" for v1 and "part-r-" for v2 API - */ - public abstract String getOutputFileNamePrefix(); - - /** - * Tests map task execution. - * - * @throws Exception If fails. - */ - @SuppressWarnings("ConstantConditions") - public void testMapTask() throws Exception { - IgfsPath inDir = new IgfsPath(PATH_INPUT); - - igfs.mkdirs(inDir); - - IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input"); - - URI inFileUri = URI.create(igfsScheme() + inFile.toString()); - - try (PrintWriter pw = new PrintWriter(igfs.create(inFile, true))) { - pw.println("hello0 world0"); - pw.println("world1 hello1"); - } - - HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, igfs.info(inFile).length() - 1); - - try (PrintWriter pw = new PrintWriter(igfs.append(inFile, false))) { - pw.println("hello2 world2"); - pw.println("world3 hello3"); - } - HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, fileBlock1.length(), - igfs.info(inFile).length() - fileBlock1.length()); - - HadoopJobEx gridJob = getHadoopJob(igfsScheme() + inFile.toString(), igfsScheme() + PATH_OUTPUT); - - HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock1); - - HadoopTestTaskContext ctx = new HadoopTestTaskContext(taskInfo, gridJob); - - ctx.mockOutput().clear(); - - ctx.run(); - - assertEquals("hello0,1; world0,1; world1,1; hello1,1", Joiner.on("; ").join(ctx.mockOutput())); - - ctx.mockOutput().clear(); - - ctx.taskInfo(new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock2)); - - ctx.run(); - - assertEquals("hello2,1; world2,1; world3,1; hello3,1", Joiner.on("; ").join(ctx.mockOutput())); - } - - /** - * Generates input data for reduce-like operation into mock context input and runs the operation. - * - * @param gridJob Job is to create reduce task from. - * @param taskType Type of task - combine or reduce. - * @param taskNum Number of task in job. - * @param words Pairs of words and its counts. - * @return Context with mock output. - * @throws IgniteCheckedException If fails. - */ - private HadoopTestTaskContext runTaskWithInput(HadoopJobEx gridJob, HadoopTaskType taskType, - int taskNum, String... words) throws IgniteCheckedException { - HadoopTaskInfo taskInfo = new HadoopTaskInfo(taskType, gridJob.id(), taskNum, 0, null); - - HadoopTestTaskContext ctx = new HadoopTestTaskContext(taskInfo, gridJob); - - for (int i = 0; i < words.length; i+=2) { - List<IntWritable> valList = new ArrayList<>(); - - for (int j = 0; j < Integer.parseInt(words[i + 1]); j++) - valList.add(new IntWritable(1)); - - ctx.mockInput().put(new Text(words[i]), valList); - } - - ctx.run(); - - return ctx; - } - - /** - * Tests reduce task execution. - * - * @throws Exception If fails. - */ - public void testReduceTask() throws Exception { - HadoopJobEx gridJob = getHadoopJob(igfsScheme() + PATH_INPUT, igfsScheme() + PATH_OUTPUT); - - runTaskWithInput(gridJob, HadoopTaskType.REDUCE, 0, "word1", "5", "word2", "10"); - runTaskWithInput(gridJob, HadoopTaskType.REDUCE, 1, "word3", "7", "word4", "15"); - - assertEquals( - "word1\t5\n" + - "word2\t10\n", - readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000000/" + - getOutputFileNamePrefix() + "00000") - ); - - assertEquals( - "word3\t7\n" + - "word4\t15\n", - readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000001/" + - getOutputFileNamePrefix() + "00001") - ); - } - - /** - * Tests combine task execution. - * - * @throws Exception If fails. - */ - public void testCombinerTask() throws Exception { - HadoopJobEx gridJob = getHadoopJob("/", "/"); - - HadoopTestTaskContext ctx = - runTaskWithInput(gridJob, HadoopTaskType.COMBINE, 0, "word1", "5", "word2", "10"); - - assertEquals("word1,5; word2,10", Joiner.on("; ").join(ctx.mockOutput())); - - ctx = runTaskWithInput(gridJob, HadoopTaskType.COMBINE, 1, "word3", "7", "word4", "15"); - - assertEquals("word3,7; word4,15", Joiner.on("; ").join(ctx.mockOutput())); - } - - /** - * Runs chain of map-combine task on file block. - * - * @param fileBlock block of input file to be processed. - * @param gridJob Hadoop job implementation. - * @return Context of combine task with mock output. - * @throws IgniteCheckedException If fails. - */ - private HadoopTestTaskContext runMapCombineTask(HadoopFileBlock fileBlock, HadoopJobEx gridJob) - throws IgniteCheckedException { - HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock); - - HadoopTestTaskContext mapCtx = new HadoopTestTaskContext(taskInfo, gridJob); - - mapCtx.run(); - - //Prepare input for combine - taskInfo = new HadoopTaskInfo(HadoopTaskType.COMBINE, gridJob.id(), 0, 0, null); - - HadoopTestTaskContext combineCtx = new HadoopTestTaskContext(taskInfo, gridJob); - - combineCtx.makeTreeOfWritables(mapCtx.mockOutput()); - - combineCtx.run(); - - return combineCtx; - } - - /** - * Tests all job in complex. - * Runs 2 chains of map-combine tasks and sends result into one reduce task. - * - * @throws Exception If fails. - */ - @SuppressWarnings("ConstantConditions") - public void testAllTasks() throws Exception { - IgfsPath inDir = new IgfsPath(PATH_INPUT); - - igfs.mkdirs(inDir); - - IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input"); - - URI inFileUri = URI.create(igfsScheme() + inFile.toString()); - - generateTestFile(inFile.toString(), "red", 100, "blue", 200, "green", 150, "yellow", 70); - - //Split file into two blocks - long fileLen = igfs.info(inFile).length(); - - Long l = fileLen / 2; - - HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, l); - HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, l, fileLen - l); - - HadoopJobEx gridJob = getHadoopJob(inFileUri.toString(), igfsScheme() + PATH_OUTPUT); - - HadoopTestTaskContext combine1Ctx = runMapCombineTask(fileBlock1, gridJob); - - HadoopTestTaskContext combine2Ctx = runMapCombineTask(fileBlock2, gridJob); - - //Prepare input for combine - HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, gridJob.id(), 0, 0, null); - - HadoopTestTaskContext reduceCtx = new HadoopTestTaskContext(taskInfo, gridJob); - - reduceCtx.makeTreeOfWritables(combine1Ctx.mockOutput()); - reduceCtx.makeTreeOfWritables(combine2Ctx.mockOutput()); - - reduceCtx.run(); - - reduceCtx.taskInfo(new HadoopTaskInfo(HadoopTaskType.COMMIT, gridJob.id(), 0, 0, null)); - - reduceCtx.run(); - - assertEquals( - "blue\t200\n" + - "green\t150\n" + - "red\t100\n" + - "yellow\t70\n", - readAndSortFile(PATH_OUTPUT + "/" + getOutputFileNamePrefix() + "00000") - ); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV1Test.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV1Test.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV1Test.java index ca96551..0ca166f 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV1Test.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV1Test.java @@ -32,7 +32,7 @@ import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.crea /** * Tests of Map, Combine and Reduce task executions via running of job of hadoop API v1. */ -public class HadoopTasksV1Test extends HadoopTasksAllVersionsTest { +public class HadoopTasksV1Test extends HadoopTasksVersionsAbstractTest { /** * Creates WordCount hadoop job for API v1. * @@ -59,4 +59,4 @@ public class HadoopTasksV1Test extends HadoopTasksAllVersionsTest { @Override public String getOutputFileNamePrefix() { return "part-"; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV2Test.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV2Test.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV2Test.java index 0fcd358..0172956 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV2Test.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksV2Test.java @@ -37,7 +37,7 @@ import static org.apache.ignite.internal.processors.hadoop.impl.HadoopUtils.crea /** * Tests of Map, Combine and Reduce task executions via running of job of hadoop API v2. */ -public class HadoopTasksV2Test extends HadoopTasksAllVersionsTest { +public class HadoopTasksV2Test extends HadoopTasksVersionsAbstractTest { /** * Creates WordCount hadoop job for API v2. * @@ -78,4 +78,4 @@ public class HadoopTasksV2Test extends HadoopTasksAllVersionsTest { @Override public String getOutputFileNamePrefix() { return "part-r-"; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/ignite/blob/de9227d7/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksVersionsAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksVersionsAbstractTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksVersionsAbstractTest.java new file mode 100644 index 0000000..836073a --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTasksVersionsAbstractTest.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.hadoop.impl; + +import com.google.common.base.Joiner; +import java.io.IOException; +import java.io.PrintWriter; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.igfs.IgfsPath; +import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock; +import org.apache.ignite.internal.processors.hadoop.HadoopJobEx; +import org.apache.ignite.internal.processors.hadoop.HadoopTaskInfo; +import org.apache.ignite.internal.processors.hadoop.HadoopTaskType; +import org.apache.ignite.internal.processors.hadoop.impl.examples.HadoopWordCount2; + +/** + * Tests of Map, Combine and Reduce task executions of any version of hadoop API. + */ +abstract class HadoopTasksVersionsAbstractTest extends HadoopAbstractWordCountTest { + /** Empty hosts array. */ + private static final String[] HOSTS = new String[0]; + + /** + * Creates some grid hadoop job. Override this method to create tests for any job implementation. + * + * @param inFile Input file name for the job. + * @param outFile Output file name for the job. + * @return Hadoop job. + * @throws IOException If fails. + */ + public abstract HadoopJobEx getHadoopJob(String inFile, String outFile) throws Exception; + + /** + * @return prefix of reducer output file name. It's "part-" for v1 and "part-r-" for v2 API + */ + public abstract String getOutputFileNamePrefix(); + + /** + * Tests map task execution. + * + * @throws Exception If fails. + */ + @SuppressWarnings("ConstantConditions") + public void testMapTask() throws Exception { + IgfsPath inDir = new IgfsPath(PATH_INPUT); + + igfs.mkdirs(inDir); + + IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input"); + + URI inFileUri = URI.create(igfsScheme() + inFile.toString()); + + try (PrintWriter pw = new PrintWriter(igfs.create(inFile, true))) { + pw.println("hello0 world0"); + pw.println("world1 hello1"); + } + + HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, igfs.info(inFile).length() - 1); + + try (PrintWriter pw = new PrintWriter(igfs.append(inFile, false))) { + pw.println("hello2 world2"); + pw.println("world3 hello3"); + } + HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, fileBlock1.length(), + igfs.info(inFile).length() - fileBlock1.length()); + + HadoopJobEx gridJob = getHadoopJob(igfsScheme() + inFile.toString(), igfsScheme() + PATH_OUTPUT); + + HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock1); + + HadoopTestTaskContext ctx = new HadoopTestTaskContext(taskInfo, gridJob); + + ctx.mockOutput().clear(); + + ctx.run(); + + assertEquals("hello0,1; world0,1; world1,1; hello1,1", Joiner.on("; ").join(ctx.mockOutput())); + + ctx.mockOutput().clear(); + + ctx.taskInfo(new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock2)); + + ctx.run(); + + assertEquals("hello2,1; world2,1; world3,1; hello3,1", Joiner.on("; ").join(ctx.mockOutput())); + } + + /** + * Generates input data for reduce-like operation into mock context input and runs the operation. + * + * @param gridJob Job is to create reduce task from. + * @param taskType Type of task - combine or reduce. + * @param taskNum Number of task in job. + * @param words Pairs of words and its counts. + * @return Context with mock output. + * @throws IgniteCheckedException If fails. + */ + private HadoopTestTaskContext runTaskWithInput(HadoopJobEx gridJob, HadoopTaskType taskType, + int taskNum, String... words) throws IgniteCheckedException { + HadoopTaskInfo taskInfo = new HadoopTaskInfo(taskType, gridJob.id(), taskNum, 0, null); + + HadoopTestTaskContext ctx = new HadoopTestTaskContext(taskInfo, gridJob); + + for (int i = 0; i < words.length; i+=2) { + List<IntWritable> valList = new ArrayList<>(); + + for (int j = 0; j < Integer.parseInt(words[i + 1]); j++) + valList.add(new IntWritable(1)); + + ctx.mockInput().put(new Text(words[i]), valList); + } + + ctx.run(); + + return ctx; + } + + /** + * Tests reduce task execution. + * + * @throws Exception If fails. + */ + public void testReduceTask() throws Exception { + HadoopJobEx gridJob = getHadoopJob(igfsScheme() + PATH_INPUT, igfsScheme() + PATH_OUTPUT); + + runTaskWithInput(gridJob, HadoopTaskType.REDUCE, 0, "word1", "5", "word2", "10"); + runTaskWithInput(gridJob, HadoopTaskType.REDUCE, 1, "word3", "7", "word4", "15"); + + assertEquals( + "word1\t5\n" + + "word2\t10\n", + readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000000/" + + getOutputFileNamePrefix() + "00000") + ); + + assertEquals( + "word3\t7\n" + + "word4\t15\n", + readAndSortFile(PATH_OUTPUT + "/_temporary/0/task_00000000-0000-0000-0000-000000000000_0000_r_000001/" + + getOutputFileNamePrefix() + "00001") + ); + } + + /** + * Tests combine task execution. + * + * @throws Exception If fails. + */ + public void testCombinerTask() throws Exception { + HadoopJobEx gridJob = getHadoopJob("/", "/"); + + HadoopTestTaskContext ctx = + runTaskWithInput(gridJob, HadoopTaskType.COMBINE, 0, "word1", "5", "word2", "10"); + + assertEquals("word1,5; word2,10", Joiner.on("; ").join(ctx.mockOutput())); + + ctx = runTaskWithInput(gridJob, HadoopTaskType.COMBINE, 1, "word3", "7", "word4", "15"); + + assertEquals("word3,7; word4,15", Joiner.on("; ").join(ctx.mockOutput())); + } + + /** + * Runs chain of map-combine task on file block. + * + * @param fileBlock block of input file to be processed. + * @param gridJob Hadoop job implementation. + * @return Context of combine task with mock output. + * @throws IgniteCheckedException If fails. + */ + private HadoopTestTaskContext runMapCombineTask(HadoopFileBlock fileBlock, HadoopJobEx gridJob) + throws IgniteCheckedException { + HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.MAP, gridJob.id(), 0, 0, fileBlock); + + HadoopTestTaskContext mapCtx = new HadoopTestTaskContext(taskInfo, gridJob); + + mapCtx.run(); + + //Prepare input for combine + taskInfo = new HadoopTaskInfo(HadoopTaskType.COMBINE, gridJob.id(), 0, 0, null); + + HadoopTestTaskContext combineCtx = new HadoopTestTaskContext(taskInfo, gridJob); + + combineCtx.makeTreeOfWritables(mapCtx.mockOutput()); + + combineCtx.run(); + + return combineCtx; + } + + /** + * Tests all job in complex. + * Runs 2 chains of map-combine tasks and sends result into one reduce task. + * + * @throws Exception If fails. + */ + @SuppressWarnings("ConstantConditions") + public void testAllTasks() throws Exception { + IgfsPath inDir = new IgfsPath(PATH_INPUT); + + igfs.mkdirs(inDir); + + IgfsPath inFile = new IgfsPath(inDir, HadoopWordCount2.class.getSimpleName() + "-input"); + + URI inFileUri = URI.create(igfsScheme() + inFile.toString()); + + generateTestFile(inFile.toString(), "red", 100, "blue", 200, "green", 150, "yellow", 70); + + //Split file into two blocks + long fileLen = igfs.info(inFile).length(); + + Long l = fileLen / 2; + + HadoopFileBlock fileBlock1 = new HadoopFileBlock(HOSTS, inFileUri, 0, l); + HadoopFileBlock fileBlock2 = new HadoopFileBlock(HOSTS, inFileUri, l, fileLen - l); + + HadoopJobEx gridJob = getHadoopJob(inFileUri.toString(), igfsScheme() + PATH_OUTPUT); + + HadoopTestTaskContext combine1Ctx = runMapCombineTask(fileBlock1, gridJob); + + HadoopTestTaskContext combine2Ctx = runMapCombineTask(fileBlock2, gridJob); + + //Prepare input for combine + HadoopTaskInfo taskInfo = new HadoopTaskInfo(HadoopTaskType.REDUCE, gridJob.id(), 0, 0, null); + + HadoopTestTaskContext reduceCtx = new HadoopTestTaskContext(taskInfo, gridJob); + + reduceCtx.makeTreeOfWritables(combine1Ctx.mockOutput()); + reduceCtx.makeTreeOfWritables(combine2Ctx.mockOutput()); + + reduceCtx.run(); + + reduceCtx.taskInfo(new HadoopTaskInfo(HadoopTaskType.COMMIT, gridJob.id(), 0, 0, null)); + + reduceCtx.run(); + + assertEquals( + "blue\t200\n" + + "green\t150\n" + + "red\t100\n" + + "yellow\t70\n", + readAndSortFile(PATH_OUTPUT + "/" + getOutputFileNamePrefix() + "00000") + ); + } +}
