IGNITE-3625: IGFS: "meta" and "data" caches are now configured inside IGFS bean. This closes #923. This closes #972.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3ca46d1a Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3ca46d1a Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3ca46d1a Branch: refs/heads/ignite-4565-ddl Commit: 3ca46d1a1bebf940f916ef47eaae48c3a181433d Parents: f55370b Author: tledkov-gridgain <[email protected]> Authored: Tue Feb 21 13:55:30 2017 +0300 Committer: devozerov <[email protected]> Committed: Tue Feb 21 13:55:30 2017 +0300 ---------------------------------------------------------------------- config/hadoop/default-config.xml | 29 -- examples/config/filesystem/example-igfs.xml | 27 -- .../JettyRestProcessorAbstractSelfTest.java | 23 -- .../configuration/FileSystemConfiguration.java | 75 +++-- .../org/apache/ignite/internal/IgnitionEx.java | 9 +- .../processors/cache/GridCacheAdapter.java | 3 +- .../processors/cache/GridCacheProcessor.java | 5 +- .../processors/igfs/IgfsDataManager.java | 11 +- .../internal/processors/igfs/IgfsImpl.java | 2 +- .../processors/igfs/IgfsMetaManager.java | 13 +- .../internal/processors/igfs/IgfsProcessor.java | 130 +------- .../internal/processors/igfs/IgfsUtils.java | 215 +++++++++++-- .../visor/node/VisorIgfsConfiguration.java | 4 +- .../visor/node/VisorNodeDataCollectorJob.java | 6 +- modules/core/src/test/config/igfs-loopback.xml | 27 -- modules/core/src/test/config/igfs-shmem.xml | 27 -- .../ignite/igfs/IgfsEventsAbstractSelfTest.java | 51 ++- .../igfs/IgfsFragmentizerAbstractSelfTest.java | 17 +- .../ignite/igfs/IgfsFragmentizerSelfTest.java | 3 +- .../igfs/IgfsAbstractBaseSelfTest.java | 17 +- .../igfs/IgfsBackupFailoverSelfTest.java | 7 +- ...lockMessageSystemPoolStarvationSelfTest.java | 17 +- ...sCachePerBlockLruEvictionPolicySelfTest.java | 18 +- .../processors/igfs/IgfsCacheSelfTest.java | 19 +- .../igfs/IgfsDataManagerSelfTest.java | 33 +- .../processors/igfs/IgfsMaxSizeSelfTest.java | 6 +- .../igfs/IgfsMetaManagerSelfTest.java | 6 +- .../processors/igfs/IgfsMetricsSelfTest.java | 16 +- .../processors/igfs/IgfsModesSelfTest.java | 51 ++- .../processors/igfs/IgfsOneClientNodeTest.java | 13 +- .../processors/igfs/IgfsProcessorSelfTest.java | 25 +- .../igfs/IgfsProcessorValidationSelfTest.java | 320 ++++--------------- ...gfsSecondaryFileSystemInjectionSelfTest.java | 8 +- ...IpcEndpointRegistrationAbstractSelfTest.java | 17 +- .../processors/igfs/IgfsSizeSelfTest.java | 27 +- .../processors/igfs/IgfsStartCacheTest.java | 28 +- .../processors/igfs/IgfsStreamsSelfTest.java | 46 +-- .../processors/igfs/IgfsTaskSelfTest.java | 8 +- .../IgfsAbstractRecordResolverSelfTest.java | 8 +- .../ipc/shmem/IpcSharedMemoryNodeStartup.java | 30 +- .../test/config/hadoop-fs-open-test/grid-0.xml | 57 ++-- .../test/config/hadoop-fs-open-test/grid-1.xml | 57 ++-- .../test/config/hadoop-fs-open-test/grid-2.xml | 57 ++-- .../test/config/igfs-cli-config-dual-async.xml | 57 ++-- .../test/config/igfs-cli-config-dual-sync.xml | 57 ++-- .../src/test/config/igfs-cli-config-primary.xml | 57 ++-- .../src/test/config/igfs-cli-config-proxy.xml | 57 ++-- .../impl/HadoopAbstractMapReduceTest.java | 6 +- .../hadoop/impl/HadoopAbstractSelfTest.java | 18 +- .../igfs/HadoopFIleSystemFactorySelfTest.java | 8 +- .../HadoopIgfs20FileSystemAbstractSelfTest.java | 52 +-- .../igfs/HadoopIgfsDualAbstractSelfTest.java | 8 +- ...oopSecondaryFileSystemConfigurationTest.java | 69 ++-- .../igfs/IgfsNearOnlyMultiNodeSelfTest.java | 6 +- .../IgniteHadoopFileSystemAbstractSelfTest.java | 67 ++-- ...niteHadoopFileSystemClientBasedOpenTest.java | 3 +- .../IgniteHadoopFileSystemClientSelfTest.java | 56 ++-- ...IgniteHadoopFileSystemHandshakeSelfTest.java | 8 +- .../IgniteHadoopFileSystemIpcCacheSelfTest.java | 47 +-- ...niteHadoopFileSystemLoggerStateSelfTest.java | 22 +- ...condaryFileSystemInitializationSelfTest.java | 30 +- .../org/apache/ignite/spring/sprint-exclude.xml | 2 - 62 files changed, 889 insertions(+), 1309 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/config/hadoop/default-config.xml ---------------------------------------------------------------------- diff --git a/config/hadoop/default-config.xml b/config/hadoop/default-config.xml index d8d3636..83d3679 100644 --- a/config/hadoop/default-config.xml +++ b/config/hadoop/default-config.xml @@ -54,31 +54,6 @@ --> <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> <!-- - Configure caches where IGFS will store data. - --> - <property name="cacheConfiguration"> - <list> - <!-- - Configure metadata cache where file system structure will be stored. It must be TRANSACTIONAL, - and must have backups to maintain file system consistency in case of node crash. - --> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-meta"/> - <property name="cacheMode" value="REPLICATED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - </bean> - - <!-- - Configure data cache where file's data will be stored. - --> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-data"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - </bean> - </list> - </property> - - <!-- This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. --> <property name="connectorConfiguration"> @@ -96,10 +71,6 @@ <!-- IGFS name you will use to access IGFS through Hadoop API. --> <property name="name" value="igfs"/> - <!-- Caches with these names must be configured. --> - <property name="metaCacheName" value="igfs-meta"/> - <property name="dataCacheName" value="igfs-data"/> - <!-- Configure TCP endpoint for communication with the file system instance. --> <property name="ipcEndpointConfiguration"> <bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration"> http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/examples/config/filesystem/example-igfs.xml ---------------------------------------------------------------------- diff --git a/examples/config/filesystem/example-igfs.xml b/examples/config/filesystem/example-igfs.xml index a72ddce..2236821 100644 --- a/examples/config/filesystem/example-igfs.xml +++ b/examples/config/filesystem/example-igfs.xml @@ -61,8 +61,6 @@ <list> <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> <property name="name" value="igfs"/> - <property name="metaCacheName" value="igfs-meta"/> - <property name="dataCacheName" value="igfs-data"/> <!-- Must correlate with cache affinity mapper. --> <property name="blockSize" value="#{128 * 1024}"/> @@ -97,31 +95,6 @@ </list> </property> - <property name="cacheConfiguration"> - <list> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-data"/> - <property name="cacheMode" value="PARTITIONED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - <property name="backups" value="0"/> - <property name="affinityMapper"> - <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> - <!-- Haw many blocks in row will be stored on the same node. --> - <constructor-arg value="512"/> - </bean> - </property> - </bean> - - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-meta"/> - <property name="cacheMode" value="REPLICATED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - </bean> - </list> - </property> - <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. --> <property name="discoverySpi"> <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java index 8d3ab74..84071ea 100644 --- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java @@ -2274,32 +2274,9 @@ public abstract class JettyRestProcessorAbstractSelfTest extends AbstractRestPro @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); - CacheConfiguration cacheIgfs_data = new CacheConfiguration(); - - cacheIgfs_data.setName("igfs-data"); - cacheIgfs_data.setCacheMode(CacheMode.PARTITIONED); - cacheIgfs_data.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - cacheIgfs_data.setBackups(0); - - cacheIgfs_data.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - - cacheIgfs_data.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(512)); - - CacheConfiguration cacheIgfs_meta = new CacheConfiguration(); - - cacheIgfs_meta.setName("igfs-meta"); - cacheIgfs_meta.setCacheMode(CacheMode.REPLICATED); - cacheIgfs_meta.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - - cacheIgfs_meta.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - - cfg.setCacheConfiguration(cfg.getCacheConfiguration()[0], cacheIgfs_data, cacheIgfs_meta); - FileSystemConfiguration igfs = new FileSystemConfiguration(); igfs.setName("igfs"); - igfs.setDataCacheName("igfs-data"); - igfs.setMetaCacheName("igfs-meta"); igfs.setIpcEndpointConfiguration(new IgfsIpcEndpointConfiguration()); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java index e665e84..1a8d576 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java @@ -98,12 +98,6 @@ public class FileSystemConfiguration { /** IGFS instance name. */ private String name; - /** Cache name to store IGFS meta information. */ - private String metaCacheName; - - /** Cache name to store file's data blocks. */ - private String dataCacheName; - /** File's data block size (bytes). */ private int blockSize = DFLT_BLOCK_SIZE; @@ -185,6 +179,12 @@ public class FileSystemConfiguration { /** Update file length on flush flag. */ private boolean updateFileLenOnFlush = DFLT_UPDATE_FILE_LEN_ON_FLUSH; + /** Meta cache config. */ + private CacheConfiguration metaCacheCfg; + + /** Data cache config. */ + private CacheConfiguration dataCacheCfg; + /** * Constructs default configuration. */ @@ -206,7 +206,7 @@ public class FileSystemConfiguration { blockSize = cfg.getBlockSize(); bufSize = cfg.getStreamBufferSize(); colocateMeta = cfg.isColocateMetadata(); - dataCacheName = cfg.getDataCacheName(); + dataCacheCfg = cfg.getDataCacheConfiguration(); dfltMode = cfg.getDefaultMode(); dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize(); dualModePutExec = cfg.getDualModePutExecutorService(); @@ -222,7 +222,7 @@ public class FileSystemConfiguration { ipcEndpointEnabled = cfg.isIpcEndpointEnabled(); maxSpace = cfg.getMaxSpaceSize(); maxTaskRangeLen = cfg.getMaximumTaskRangeLength(); - metaCacheName = cfg.getMetaCacheName(); + metaCacheCfg = cfg.getMetaCacheConfiguration(); mgmtPort = cfg.getManagementPort(); name = cfg.getName(); pathModes = cfg.getPathModes(); @@ -255,40 +255,55 @@ public class FileSystemConfiguration { } /** - * Cache name to store IGFS meta information. If {@code null}, then instance - * with default meta-cache name will be used. + * Cache config to store IGFS meta information. * - * @return Cache name to store IGFS meta information. + * @return Cache configuration object. */ - @Nullable public String getMetaCacheName() { - return metaCacheName; + @Nullable public CacheConfiguration getMetaCacheConfiguration() { + return metaCacheCfg; } /** - * Sets cache name to store IGFS meta information. + * Cache config to store IGFS meta information. If {@code null}, then default config for + * meta-cache will be used. + * + * Default configuration for the meta cache is: + * <ul> + * <li>atomicityMode = TRANSACTIONAL</li> + * <li>cacheMode = PARTITIONED</li> + * <li>backups = 1</li> + * </ul> * - * @param metaCacheName Cache name to store IGFS meta information. + * @param metaCacheCfg Cache configuration object. */ - public void setMetaCacheName(String metaCacheName) { - this.metaCacheName = metaCacheName; + public void setMetaCacheConfiguration(CacheConfiguration metaCacheCfg) { + this.metaCacheCfg = metaCacheCfg; } /** - * Cache name to store IGFS data. + * Cache config to store IGFS data. * - * @return Cache name to store IGFS data. + * @return Cache configuration object. */ - @Nullable public String getDataCacheName() { - return dataCacheName; + @Nullable public CacheConfiguration getDataCacheConfiguration() { + return dataCacheCfg; } /** - * Sets cache name to store IGFS data. + * Cache config to store IGFS data. If {@code null}, then default config for + * data cache will be used. + * + * Default configuration for the data cache is: + * <ul> + * <<li>atomicityMode = TRANSACTIONAL</li> + * <li>cacheMode = PARTITIONED</li> + * <li>backups = 0</li> + * </ul> * - * @param dataCacheName Cache name to store IGFS data. + * @param dataCacheCfg Cache configuration object. */ - public void setDataCacheName(String dataCacheName) { - this.dataCacheName = dataCacheName; + public void setDataCacheConfiguration(CacheConfiguration dataCacheCfg) { + this.dataCacheCfg = dataCacheCfg; } /** @@ -878,11 +893,11 @@ public class FileSystemConfiguration { * transaction with keys owned only by a single node. * <p> * IGFS stores information about file system structure (metadata) inside a transactional cache configured through - * {@link #getMetaCacheName()} property. Metadata updates caused by operations on IGFS usually require several - * internal keys to be updated. As IGFS metadata cache usually operates in {@link CacheMode#REPLICATED} mode, - * meaning that all nodes have all metadata locally, it makes sense to give a hint to Ignite to co-locate - * ownership of all metadata keys on a single node. This will decrease amount of network trips required to update - * metadata and hence could improve performance. + * {@link #getMetaCacheConfiguration()} property. Metadata updates caused by operations on IGFS usually require + * several internal keys to be updated. As IGFS metadata cache usually operates + * in {@link CacheMode#REPLICATED} mode, meaning that all nodes have all metadata locally, it makes sense to give + * a hint to Ignite to co-locate ownership of all metadata keys on a single node. + * This will decrease amount of network trips required to update metadata and hence could improve performance. * <p> * This property should be disabled if you see excessive CPU and network load on a single node, which * degrades performance and cannot be explained by business logic of your application. http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index 05d19c0..9fe2dba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -2126,17 +2126,20 @@ public class IgnitionEx { throw new IgniteCheckedException("Cache name cannot be \"" + CU.UTILITY_CACHE_NAME + "\" because it is reserved for internal purposes."); + if (IgfsUtils.matchIgfsCacheName(ccfg.getName())) + throw new IgniteCheckedException( + "Cache name cannot start with \""+ IgfsUtils.IGFS_CACHE_PREFIX + + "\" because it is reserved for IGFS internal purposes."); + cacheCfgs.add(ccfg); } } cfg.setCacheConfiguration(cacheCfgs.toArray(new CacheConfiguration[cacheCfgs.size()])); - // Iterate over IGFS caches and prepare their configurations if needed. assert cfg.getCacheConfiguration() != null; - for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) - IgfsUtils.prepareCacheConfiguration(cfg, ccfg); + IgfsUtils.prepareCacheConfigurations(cfg); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 4507c53..285ba11 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -97,6 +97,7 @@ import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry; import org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl; import org.apache.ignite.internal.processors.dr.IgniteDrDataStreamerCacheUpdater; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.processors.platform.cache.PlatformCacheEntryFilter; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException; @@ -338,7 +339,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V if (igfsCfgs != null) { for (FileSystemConfiguration igfsCfg : igfsCfgs) { - if (F.eq(ctx.name(), igfsCfg.getDataCacheName())) { + if (F.eq(ctx.name(), igfsCfg.getDataCacheConfiguration().getName())) { if (!ctx.isNear()) { igfsDataCache = true; igfsDataCacheSize = new LongAdder8(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 17f9c2a..b1644d1 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -99,6 +99,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTransactio import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.processors.plugin.CachePluginManager; import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.processors.timeout.GridTimeoutObject; @@ -726,8 +727,8 @@ public class GridCacheProcessor extends GridProcessorAdapter { if (igfsCfgs != null) { for (FileSystemConfiguration igfsCfg : igfsCfgs) { - internalCaches.add(maskNull(igfsCfg.getMetaCacheName())); - internalCaches.add(maskNull(igfsCfg.getDataCacheName())); + internalCaches.add(maskNull(igfsCfg.getMetaCacheConfiguration().getName())); + internalCaches.add(maskNull(igfsCfg.getDataCacheConfiguration().getName())); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java index d6297b9..dc845ee 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java @@ -130,6 +130,9 @@ public class IgfsDataManager extends IgfsManager { /** Async file delete worker. */ private AsyncDeleteWorker delWorker; + /** Async file delete worker. */ + private String dataCacheName; + /** On-going remote reads futures. */ private final ConcurrentHashMap8<IgfsBlockKey, IgniteInternalFuture<byte[]>> rmtReadFuts = new ConcurrentHashMap8<>(); @@ -180,19 +183,21 @@ public class IgfsDataManager extends IgfsManager { delWorker = new AsyncDeleteWorker(igfsCtx.kernalContext().gridName(), "igfs-" + igfsName + "-delete-worker", log); + + dataCacheName = igfsCtx.configuration().getDataCacheConfiguration().getName(); } /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override protected void onKernalStart0() throws IgniteCheckedException { - dataCachePrj = igfsCtx.kernalContext().cache().getOrStartCache(igfsCtx.configuration().getDataCacheName()); + dataCachePrj = igfsCtx.kernalContext().cache().getOrStartCache(dataCacheName); assert dataCachePrj != null; dataCache = (IgniteInternalCache)dataCachePrj; AffinityKeyMapper mapper = igfsCtx.kernalContext().cache() - .internalCache(igfsCtx.configuration().getDataCacheName()).configuration().getAffinityMapper(); + .internalCache(dataCacheName).configuration().getAffinityMapper(); grpSize = mapper instanceof IgfsGroupDataBlocksKeyMapper ? ((IgfsGroupDataBlocksKeyMapper)mapper).getGroupSize() : 1; @@ -201,7 +206,7 @@ public class IgfsDataManager extends IgfsManager { assert grpBlockSize != 0; - igfsCtx.kernalContext().cache().internalCache(igfsCtx.configuration().getDataCacheName()).preloader() + igfsCtx.kernalContext().cache().internalCache(dataCacheName).preloader() .startFuture().listen(new CI1<IgniteInternalFuture<Object>>() { @Override public void apply(IgniteInternalFuture<Object> f) { dataCacheStartLatch.countDown(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index 56da121..7165f31 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -263,7 +263,7 @@ public final class IgfsImpl implements IgfsEx { secondaryPaths = new IgfsPaths(secondaryFsPayload, dfltMode, modeRslvr.modesOrdered()); // Check whether IGFS LRU eviction policy is set on data cache. - String dataCacheName = igfsCtx.configuration().getDataCacheName(); + String dataCacheName = igfsCtx.configuration().getDataCacheConfiguration().getName(); for (CacheConfiguration cacheCfg : igfsCtx.kernalContext().config().getCacheConfiguration()) { if (F.eq(dataCacheName, cacheCfg.getName())) { http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java index 95d73a8..35d0776 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java @@ -155,6 +155,9 @@ public class IgfsMetaManager extends IgfsManager { /** Compute facade for client tasks. */ private IgniteCompute cliCompute; + /** Compute facade for client tasks. */ + private String metaCacheName; + /** * Constructor. * @@ -189,16 +192,18 @@ public class IgfsMetaManager extends IgfsManager { sampling = new IgfsSamplingKey(cfg.getName()); log = igfsCtx.kernalContext().log(IgfsMetaManager.class); + + metaCacheName = cfg.getMetaCacheConfiguration().getName(); } /** {@inheritDoc} */ @SuppressWarnings("RedundantCast") @Override protected void onKernalStart0() throws IgniteCheckedException { - metaCache = igfsCtx.kernalContext().cache().getOrStartCache(cfg.getMetaCacheName()); + metaCache = igfsCtx.kernalContext().cache().getOrStartCache(metaCacheName); assert metaCache != null; - igfsCtx.kernalContext().cache().internalCache(cfg.getMetaCacheName()).preloader().startFuture() + igfsCtx.kernalContext().cache().internalCache(metaCacheName).preloader().startFuture() .listen(new CI1<IgniteInternalFuture<Object>>() { @Override public void apply(IgniteInternalFuture<Object> f) { metaCacheStartLatch.countDown(); @@ -267,7 +272,7 @@ public class IgfsMetaManager extends IgfsManager { <T> T runClientTask(IgniteUuid affinityFileId, IgfsClientAbstractCallable<T> task) { try { return (cfg.isColocateMetadata()) ? - clientCompute().affinityCall(cfg.getMetaCacheName(), affinityFileId, task) : + clientCompute().affinityCall(metaCacheName, affinityFileId, task) : clientCompute().call(task); } catch (ClusterTopologyException e) { @@ -288,7 +293,7 @@ public class IgfsMetaManager extends IgfsManager { if (cliCompute0 == null) { IgniteEx ignite = igfsCtx.kernalContext().grid(); - ClusterGroup cluster = ignite.cluster().forIgfsMetadataDataNodes(cfg.getName(), cfg.getMetaCacheName()); + ClusterGroup cluster = ignite.cluster().forIgfsMetadataDataNodes(cfg.getName(), metaCacheName); cliCompute0 = ignite.compute(cluster); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java index 57bf70f..405016c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java @@ -26,14 +26,11 @@ import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper; -import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; -import org.apache.ignite.igfs.IgfsMode; import org.apache.ignite.igfs.IgfsPath; import org.apache.ignite.igfs.mapreduce.IgfsJob; import org.apache.ignite.igfs.mapreduce.IgfsRecordResolver; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteNodeAttributes; -import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.util.ipc.IpcServerEndpoint; import org.apache.ignite.internal.util.typedef.C1; import org.apache.ignite.internal.util.typedef.F; @@ -46,7 +43,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.ListIterator; import java.util.Map; @@ -55,8 +51,6 @@ import java.util.concurrent.ConcurrentMap; import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK; import static org.apache.ignite.IgniteSystemProperties.getBoolean; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.igfs.IgfsMode.PROXY; import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGFS; /** @@ -66,12 +60,6 @@ public class IgfsProcessor extends IgfsProcessorAdapter { /** Null IGFS name. */ private static final String NULL_NAME = UUID.randomUUID().toString(); - /** Min available TCP port. */ - private static final int MIN_TCP_PORT = 1; - - /** Max available TCP port. */ - private static final int MAX_TCP_PORT = 0xFFFF; - /** Converts context to IGFS. */ private static final IgniteClosure<IgfsContext,IgniteFileSystem> CTX_TO_IGFS = new C1<IgfsContext, IgniteFileSystem>() { @Override public IgniteFileSystem apply(IgfsContext igfsCtx) { @@ -101,8 +89,6 @@ public class IgfsProcessor extends IgfsProcessorAdapter { assert cfgs != null && cfgs.length > 0; - validateLocalIgfsConfigurations(cfgs); - // Start IGFS instances. for (FileSystemConfiguration cfg : cfgs) { FileSystemConfiguration cfg0 = new FileSystemConfiguration(cfg); @@ -111,9 +97,11 @@ public class IgfsProcessor extends IgfsProcessorAdapter { CacheConfiguration[] cacheCfgs = igniteCfg.getCacheConfiguration(); + String metaCacheName = cfg.getMetaCacheConfiguration().getName(); + if (cacheCfgs != null) { for (CacheConfiguration cacheCfg : cacheCfgs) { - if (F.eq(cacheCfg.getName(), cfg.getMetaCacheName())) { + if (F.eq(cacheCfg.getName(), metaCacheName)) { metaClient = false; break; @@ -162,7 +150,9 @@ public class IgfsProcessor extends IgfsProcessorAdapter { assert igniteCfg.getFileSystemConfiguration() != null; for (FileSystemConfiguration igfsCfg : igniteCfg.getFileSystemConfiguration()) { - CacheConfiguration cacheCfg = cacheCfgs.get(igfsCfg.getDataCacheName()); + String dataCacheName = igfsCfg.getDataCacheConfiguration().getName(); + + CacheConfiguration cacheCfg = cacheCfgs.get(dataCacheName); if (cacheCfg == null) continue; // No cache for the given IGFS configuration. @@ -178,8 +168,8 @@ public class IgfsProcessor extends IgfsProcessorAdapter { igfsCfg.getName(), igfsCfg.getBlockSize(), ((IgfsGroupDataBlocksKeyMapper)affMapper).getGroupSize(), - igfsCfg.getMetaCacheName(), - igfsCfg.getDataCacheName(), + igfsCfg.getMetaCacheConfiguration().getName(), + dataCacheName, igfsCfg.getDefaultMode(), igfsCfg.getPathModes(), igfsCfg.isFragmentizerEnabled())); @@ -288,110 +278,6 @@ public class IgfsProcessor extends IgfsProcessorAdapter { } /** - * Validates local IGFS configurations. Compares attributes only for IGFSes with same name. - * @param cfgs IGFS configurations - * @throws IgniteCheckedException If any of IGFS configurations is invalid. - */ - private void validateLocalIgfsConfigurations(FileSystemConfiguration[] cfgs) throws IgniteCheckedException { - Collection<String> cfgNames = new HashSet<>(); - - Collection<String> dataCacheNames = new HashSet<>(); - Collection<String> metaCacheNames = new HashSet<>(); - - for (FileSystemConfiguration cfg : cfgs) { - String name = cfg.getName(); - - if (cfgNames.contains(name)) - throw new IgniteCheckedException("Duplicate IGFS name found (check configuration and " + - "assign unique name to each): " + name); - - CacheConfiguration dataCacheCfg = config(cfg.getDataCacheName()); - CacheConfiguration metaCacheCfg = config(cfg.getMetaCacheName()); - - if (dataCacheCfg == null) - throw new IgniteCheckedException("Data cache is not configured locally for IGFS: " + cfg); - - if (GridQueryProcessor.isEnabled(dataCacheCfg)) - throw new IgniteCheckedException("IGFS data cache cannot start with enabled query indexing."); - - if (dataCacheCfg.getAtomicityMode() != TRANSACTIONAL && cfg.isFragmentizerEnabled()) - throw new IgniteCheckedException("Data cache should be transactional: " + cfg.getDataCacheName() + - " when fragmentizer is enabled"); - - if (metaCacheCfg == null) - throw new IgniteCheckedException("Metadata cache is not configured locally for IGFS: " + cfg); - - if (GridQueryProcessor.isEnabled(metaCacheCfg)) - throw new IgniteCheckedException("IGFS metadata cache cannot start with enabled query indexing."); - - if (metaCacheCfg.getAtomicityMode() != TRANSACTIONAL) - throw new IgniteCheckedException("Meta cache should be transactional: " + cfg.getMetaCacheName()); - - if (F.eq(cfg.getDataCacheName(), cfg.getMetaCacheName())) - throw new IgniteCheckedException("Cannot use same cache as both data and meta cache: " + cfg.getName()); - - if (dataCacheNames.contains(cfg.getDataCacheName())) - throw new IgniteCheckedException("Data cache names should be different for different IGFS instances: " - + cfg.getName()); - - if (metaCacheNames.contains(cfg.getMetaCacheName())) - throw new IgniteCheckedException("Meta cache names should be different for different IGFS instances: " - + cfg.getName()); - - if (!(dataCacheCfg.getAffinityMapper() instanceof IgfsGroupDataBlocksKeyMapper)) - throw new IgniteCheckedException("Invalid IGFS data cache configuration (key affinity mapper class should be " + - IgfsGroupDataBlocksKeyMapper.class.getSimpleName() + "): " + cfg); - - IgfsIpcEndpointConfiguration ipcCfg = cfg.getIpcEndpointConfiguration(); - - if (ipcCfg != null) { - final int tcpPort = ipcCfg.getPort(); - - if (!(tcpPort >= MIN_TCP_PORT && tcpPort <= MAX_TCP_PORT)) - throw new IgniteCheckedException("IGFS endpoint TCP port is out of range [" + MIN_TCP_PORT + - ".." + MAX_TCP_PORT + "]: " + tcpPort); - - if (ipcCfg.getThreadCount() <= 0) - throw new IgniteCheckedException("IGFS endpoint thread count must be positive: " + - ipcCfg.getThreadCount()); - } - - boolean secondary = cfg.getDefaultMode() == PROXY; - - if (cfg.getPathModes() != null) { - for (Map.Entry<String, IgfsMode> mode : cfg.getPathModes().entrySet()) { - if (mode.getValue() == PROXY) - secondary = true; - } - } - - if (secondary) { - // When working in any mode except of primary, secondary FS config must be provided. - assertParameter(cfg.getSecondaryFileSystem() != null, - "secondaryFileSystem cannot be null when mode is not " + IgfsMode.PRIMARY); - } - - dataCacheNames.add(cfg.getDataCacheName()); - metaCacheNames.add(cfg.getMetaCacheName()); - - cfgNames.add(name); - } - } - - /** - * @param cacheName Cache name. - * @return Configuration. - */ - private CacheConfiguration config(String cacheName) { - for (CacheConfiguration ccfg : ctx.config().getCacheConfiguration()) { - if (F.eq(cacheName, ccfg.getName())) - return ccfg; - } - - return null; - } - - /** * Check IGFS config on remote node. * * @param rmtNode Remote node. http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java index 6763d2a..ec12884 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java @@ -17,11 +17,16 @@ package org.apache.ignite.internal.processors.igfs; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.binary.BinaryRawReader; import org.apache.ignite.binary.BinaryRawWriter; +import org.apache.ignite.cache.CacheAtomicWriteOrderMode; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterTopologyException; @@ -31,6 +36,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.IgfsEvent; import org.apache.ignite.igfs.IgfsException; import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper; +import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; import org.apache.ignite.igfs.IgfsMode; import org.apache.ignite.igfs.IgfsPath; import org.apache.ignite.internal.GridKernalContext; @@ -38,6 +44,7 @@ import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException; import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; +import org.apache.ignite.internal.processors.query.GridQueryProcessor; import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.future.IgniteFutureImpl; import org.apache.ignite.internal.util.lang.IgniteOutClosureX; @@ -65,6 +72,8 @@ import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_RETRIES_COUNT; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.igfs.IgfsMode.DUAL_ASYNC; import static org.apache.ignite.igfs.IgfsMode.DUAL_SYNC; import static org.apache.ignite.igfs.IgfsMode.PRIMARY; @@ -127,9 +136,27 @@ public class IgfsUtils { /** Flag: this is a file. */ private static final byte FLAG_FILE = 0x2; + /** Filesystem cache prefix. */ + public static final String IGFS_CACHE_PREFIX = "igfs-internal-"; + + /** Data cache suffix. */ + public static final String DATA_CACHE_SUFFIX = "-data"; + + /** Meta cache suffix. */ + public static final String META_CACHE_SUFFIX = "-meta"; + /** Maximum string length to be written at once. */ private static final int MAX_STR_LEN = 0xFFFF / 4; + /** Min available TCP port. */ + private static final int MIN_TCP_PORT = 1; + + /** Max available TCP port. */ + private static final int MAX_TCP_PORT = 0xFFFF; + + /* + * Static initializer. + */ static { TRASH_IDS = new IgniteUuid[TRASH_CONCURRENCY]; @@ -311,6 +338,7 @@ public class IgfsUtils { /** * Sends a series of event. * + * @param kernalCtx Kernal context. * @param path The path of the created file. * @param type The type of event to send. */ @@ -321,6 +349,7 @@ public class IgfsUtils { /** * Sends a series of event. * + * @param kernalCtx Kernal context. * @param path The path of the created file. * @param newPath New path. * @param type The type of event to send. @@ -342,70 +371,189 @@ public class IgfsUtils { } /** + * @param cacheName Cache name. + * @return {@code True} in this is IGFS data or meta cache. + */ + public static boolean matchIgfsCacheName(@Nullable String cacheName) { + return cacheName != null && cacheName.startsWith(IGFS_CACHE_PREFIX); + } + + /** * @param cfg Grid configuration. * @param cacheName Cache name. * @return {@code True} in this is IGFS data or meta cache. */ public static boolean isIgfsCache(IgniteConfiguration cfg, @Nullable String cacheName) { + return matchIgfsCacheName(cacheName); + } + + /** + * Prepare cache configuration if this is IGFS meta or data cache. + * + * @param cfg Configuration. + * @throws IgniteCheckedException If failed. + */ + public static void prepareCacheConfigurations(IgniteConfiguration cfg) throws IgniteCheckedException { FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); + List<CacheConfiguration> ccfgs = new ArrayList<>(Arrays.asList(cfg.getCacheConfiguration())); if (igfsCfgs != null) { for (FileSystemConfiguration igfsCfg : igfsCfgs) { - // IGFS config probably has not been validated yet => possible NPE, so we check for null. - if (igfsCfg != null) { - if (F.eq(cacheName, igfsCfg.getDataCacheName()) || F.eq(cacheName, igfsCfg.getMetaCacheName())) - return true; + if (igfsCfg == null) + continue; + + CacheConfiguration ccfgMeta = igfsCfg.getMetaCacheConfiguration(); + + if (ccfgMeta == null) { + ccfgMeta = defaultMetaCacheConfig(); + + igfsCfg.setMetaCacheConfiguration(ccfgMeta); + } + + ccfgMeta.setName(IGFS_CACHE_PREFIX + igfsCfg.getName() + META_CACHE_SUFFIX); + + ccfgs.add(ccfgMeta); + + CacheConfiguration ccfgData = igfsCfg.getDataCacheConfiguration(); + + if (ccfgData == null) { + ccfgData = defaultDataCacheConfig(); + + igfsCfg.setDataCacheConfiguration(ccfgData); } + + ccfgData.setName(IGFS_CACHE_PREFIX + igfsCfg.getName() + DATA_CACHE_SUFFIX); + + ccfgs.add(ccfgData); + + + // No copy-on-read. + ccfgMeta.setCopyOnRead(false); + ccfgData.setCopyOnRead(false); + + // Always full-sync to maintain consistency. + ccfgMeta.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + ccfgData.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + + // Set co-located affinity mapper if needed. + if (igfsCfg.isColocateMetadata() && ccfgMeta.getAffinityMapper() == null) + ccfgMeta.setAffinityMapper(new IgfsColocatedMetadataAffinityKeyMapper()); + + // Set affinity mapper if needed. + if (ccfgData.getAffinityMapper() == null) + ccfgData.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper()); } + + cfg.setCacheConfiguration(ccfgs.toArray(new CacheConfiguration[ccfgs.size()])); } - return false; + validateLocalIgfsConfigurations(cfg); } /** - * Prepare cache configuration if this is IGFS meta or data cache. + * Validates local IGFS configurations. Compares attributes only for IGFSes with same name. * - * @param cfg Configuration. - * @param ccfg Cache configuration. + * @param igniteCfg Ignite config. + * @throws IgniteCheckedException If any of IGFS configurations is invalid. */ - public static void prepareCacheConfiguration(IgniteConfiguration cfg, CacheConfiguration ccfg) { - FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); + private static void validateLocalIgfsConfigurations(IgniteConfiguration igniteCfg) + throws IgniteCheckedException { - if (igfsCfgs != null) { - for (FileSystemConfiguration igfsCfg : igfsCfgs) { - if (igfsCfg != null) { - if (F.eq(ccfg.getName(), igfsCfg.getMetaCacheName())) { - // No copy-on-read. - ccfg.setCopyOnRead(false); + if (igniteCfg.getFileSystemConfiguration() == null || igniteCfg.getFileSystemConfiguration().length == 0) + return; + + Collection<String> cfgNames = new HashSet<>(); - // Always full-sync to maintain consistency. - ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + for (FileSystemConfiguration cfg : igniteCfg.getFileSystemConfiguration()) { + String name = cfg.getName(); - // Set co-located affinity mapper if needed. - if (igfsCfg.isColocateMetadata() && ccfg.getAffinityMapper() == null) - ccfg.setAffinityMapper(new IgfsColocatedMetadataAffinityKeyMapper()); + if (cfgNames.contains(name)) + throw new IgniteCheckedException("Duplicate IGFS name found (check configuration and " + + "assign unique name to each): " + name); - return; - } + CacheConfiguration ccfgData = cfg.getDataCacheConfiguration(); - if (F.eq(ccfg.getName(), igfsCfg.getDataCacheName())) { - // No copy-on-read. - ccfg.setCopyOnRead(false); + CacheConfiguration ccfgMeta = cfg.getMetaCacheConfiguration(); - // Always full-sync to maintain consistency. - ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + if (GridQueryProcessor.isEnabled(ccfgData)) + throw new IgniteCheckedException("IGFS data cache cannot start with enabled query indexing."); - // Set affinity mapper if needed. - if (ccfg.getAffinityMapper() == null) - ccfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper()); + if (GridQueryProcessor.isEnabled(ccfgMeta)) + throw new IgniteCheckedException("IGFS metadata cache cannot start with enabled query indexing."); - return; - } + if (ccfgMeta.getAtomicityMode() != TRANSACTIONAL) + throw new IgniteCheckedException("IGFS metadata cache should be transactional: " + cfg.getName()); + + if (!(ccfgData.getAffinityMapper() instanceof IgfsGroupDataBlocksKeyMapper)) + throw new IgniteCheckedException( + "Invalid IGFS data cache configuration (key affinity mapper class should be " + + IgfsGroupDataBlocksKeyMapper.class.getSimpleName() + "): " + cfg); + + IgfsIpcEndpointConfiguration ipcCfg = cfg.getIpcEndpointConfiguration(); + + if (ipcCfg != null) { + final int tcpPort = ipcCfg.getPort(); + + if (!(tcpPort >= MIN_TCP_PORT && tcpPort <= MAX_TCP_PORT)) + throw new IgniteCheckedException("IGFS endpoint TCP port is out of range [" + MIN_TCP_PORT + + ".." + MAX_TCP_PORT + "]: " + tcpPort); + + if (ipcCfg.getThreadCount() <= 0) + throw new IgniteCheckedException("IGFS endpoint thread count must be positive: " + + ipcCfg.getThreadCount()); + } + + boolean secondary = cfg.getDefaultMode() == IgfsMode.PROXY; + + if (cfg.getPathModes() != null) { + for (Map.Entry<String, IgfsMode> mode : cfg.getPathModes().entrySet()) { + if (mode.getValue() == IgfsMode.PROXY) + secondary = true; } } + + if (secondary && cfg.getSecondaryFileSystem() == null) { + // When working in any mode except of primary, secondary FS config must be provided. + throw new IgniteCheckedException("Grid configuration parameter invalid: " + + "secondaryFileSystem cannot be null when mode is not " + IgfsMode.PRIMARY); + } + + cfgNames.add(name); } } + + /** + * @return Default IGFS cache configuration. + */ + private static CacheConfiguration defaultCacheConfig() { + CacheConfiguration cfg = new CacheConfiguration(); + cfg.setAtomicWriteOrderMode(CacheAtomicWriteOrderMode.PRIMARY); + cfg.setAtomicityMode(TRANSACTIONAL); + cfg.setWriteSynchronizationMode(FULL_SYNC); + cfg.setCacheMode(CacheMode.PARTITIONED); + + return cfg; + } + + /** + * @return Default IGFS meta cache configuration. + */ + private static CacheConfiguration defaultMetaCacheConfig() { + CacheConfiguration cfg = defaultCacheConfig(); + + cfg.setBackups(1); + + return cfg; + } + + /** + * @return Default IGFS data cache configuration. + */ + private static CacheConfiguration defaultDataCacheConfig() { + return defaultCacheConfig(); + } + /** * Create empty directory with the given ID. * @@ -532,6 +680,7 @@ public class IgfsUtils { * * @param in Reader. * @return Entry. + * @throws IOException If failed. */ @Nullable public static IgfsListingEntry readListingEntry(DataInput in) throws IOException { if (in.readBoolean()) { http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java index cb10c1c..7f8e3b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java @@ -130,8 +130,8 @@ public class VisorIgfsConfiguration implements Serializable, LessNamingBean { VisorIgfsConfiguration cfg = new VisorIgfsConfiguration(); cfg.name = igfs.getName(); - cfg.metaCacheName = igfs.getMetaCacheName(); - cfg.dataCacheName = igfs.getDataCacheName(); + cfg.metaCacheName = igfs.getMetaCacheConfiguration().getName(); + cfg.dataCacheName = igfs.getDataCacheConfiguration().getName(); cfg.blockSize = igfs.getBlockSize(); cfg.prefetchBlocks = igfs.getPrefetchBlocks(); cfg.streamBufSize = igfs.getStreamBufferSize(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java index abe1364..48487ed 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java @@ -188,7 +188,8 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa if (proxyCache(cacheName)) continue; - if (arg.systemCaches() || !(isSystemCache(cacheName) || isIgfsCache(cfg, cacheName))) { + if (arg.systemCaches() || + !(isSystemCache(cacheName) || isIgfsCache(ignite.configuration(), cacheName))) { long start0 = U.currentTimeMillis(); try { @@ -227,7 +228,8 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa FileSystemConfiguration igfsCfg = igfs.configuration(); - if (proxyCache(igfsCfg.getDataCacheName()) || proxyCache(igfsCfg.getMetaCacheName())) + if (proxyCache(igfsCfg.getDataCacheConfiguration().getName()) + || proxyCache(igfsCfg.getMetaCacheConfiguration().getName())) continue; try { http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/config/igfs-loopback.xml ---------------------------------------------------------------------- diff --git a/modules/core/src/test/config/igfs-loopback.xml b/modules/core/src/test/config/igfs-loopback.xml index dbbfb4e..5bfbdae 100644 --- a/modules/core/src/test/config/igfs-loopback.xml +++ b/modules/core/src/test/config/igfs-loopback.xml @@ -75,8 +75,6 @@ <list> <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> <property name="name" value="igfs"/> - <property name="metaCacheName" value="igfs-meta"/> - <property name="dataCacheName" value="igfs-data"/> <!-- Must correlate with cache affinity mapper. --> <property name="blockSize" value="#{128 * 1024}"/> @@ -97,31 +95,6 @@ </list> </property> - <property name="cacheConfiguration"> - <list> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-data"/> - <property name="cacheMode" value="PARTITIONED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - <property name="backups" value="0"/> - <property name="affinityMapper"> - <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> - <!-- Haw many blocks in row will be stored on the same node. --> - <constructor-arg value="512"/> - </bean> - </property> - </bean> - - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-meta"/> - <property name="cacheMode" value="REPLICATED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - </bean> - </list> - </property> - <!-- Disable events. --> http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/config/igfs-shmem.xml ---------------------------------------------------------------------- diff --git a/modules/core/src/test/config/igfs-shmem.xml b/modules/core/src/test/config/igfs-shmem.xml index c4c8704..5876b41 100644 --- a/modules/core/src/test/config/igfs-shmem.xml +++ b/modules/core/src/test/config/igfs-shmem.xml @@ -75,8 +75,6 @@ <list> <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> <property name="name" value="igfs"/> - <property name="metaCacheName" value="igfs-meta"/> - <property name="dataCacheName" value="igfs-data"/> <!-- Must correlate with cache affinity mapper. --> <property name="blockSize" value="#{128 * 1024}"/> @@ -97,31 +95,6 @@ </list> </property> - <property name="cacheConfiguration"> - <list> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-data"/> - <property name="cacheMode" value="PARTITIONED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - <property name="backups" value="0"/> - <property name="affinityMapper"> - <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> - <!-- Haw many blocks in row will be stored on the same node. --> - <constructor-arg value="512"/> - </bean> - </property> - </bean> - - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="name" value="igfs-meta"/> - <property name="cacheMode" value="REPLICATED"/> - <property name="atomicityMode" value="TRANSACTIONAL"/> - <property name="writeSynchronizationMode" value="FULL_SYNC"/> - </bean> - </list> - </property> - <!-- Disable events. --> http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java index 9c2c6e5..ea2be11 100644 --- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java @@ -71,46 +71,33 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest private IgnitePredicate<Event> lsnr; /** - * Gets cache configuration. - * - * @param gridName Grid name. - * @return Cache configuration. + * @return IGFS configuration for this test. */ - @SuppressWarnings("deprecation") - protected CacheConfiguration[] getCacheConfiguration(String gridName) { - CacheConfiguration cacheCfg = defaultCacheConfiguration(); - - cacheCfg.setName("dataCache"); - cacheCfg.setCacheMode(PARTITIONED); - cacheCfg.setNearConfiguration(null); - cacheCfg.setWriteSynchronizationMode(FULL_SYNC); - cacheCfg.setEvictionPolicy(null); - cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); - cacheCfg.setBackups(0); - cacheCfg.setAtomicityMode(TRANSACTIONAL); + protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException { + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); + + igfsCfg.setName("igfs"); + igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. + + CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); + + dataCacheCfg.setCacheMode(PARTITIONED); + dataCacheCfg.setNearConfiguration(null); + dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC); + dataCacheCfg.setEvictionPolicy(null); + dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); + dataCacheCfg.setBackups(0); + dataCacheCfg.setAtomicityMode(TRANSACTIONAL); CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - metaCacheCfg.setName("metaCache"); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC); metaCacheCfg.setEvictionPolicy(null); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); - return new CacheConfiguration[] {cacheCfg, metaCacheCfg}; - } - - /** - * @return IGFS configuration for this test. - */ - protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException { - FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - - igfsCfg.setDataCacheName("dataCache"); - igfsCfg.setMetaCacheName("metaCache"); - igfsCfg.setName("igfs"); - - igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. + igfsCfg.setMetaCacheConfiguration(metaCacheCfg); + igfsCfg.setDataCacheConfiguration(dataCacheCfg); return igfsCfg; } @@ -139,8 +126,6 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest cfg.setFileSystemConfiguration(igfsCfg); - cfg.setCacheConfiguration(getCacheConfiguration(gridName)); - cfg.setHadoopConfiguration(null); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java index d52a918..6536267 100644 --- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java @@ -51,12 +51,6 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { /** IGFS group size. */ protected static final int IGFS_GROUP_SIZE = 32; - /** Metadata cache name. */ - private static final String META_CACHE_NAME = "meta"; - - /** File data cache name. */ - protected static final String DATA_CACHE_NAME = "data"; - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); @@ -67,13 +61,9 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - cfg.setCacheConfiguration(metaConfiguration(), dataConfiguration()); - FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); - igfsCfg.setMetaCacheName(META_CACHE_NAME); - igfsCfg.setDataCacheName(DATA_CACHE_NAME); igfsCfg.setBlockSize(IGFS_BLOCK_SIZE); // Need to set this to avoid thread starvation. @@ -82,6 +72,9 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { igfsCfg.setFragmentizerThrottlingBlockLength(16 * IGFS_BLOCK_SIZE); igfsCfg.setFragmentizerThrottlingDelay(10); + igfsCfg.setMetaCacheConfiguration(metaConfiguration()); + igfsCfg.setDataCacheConfiguration(dataConfiguration()); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; @@ -95,8 +88,6 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { protected CacheConfiguration metaConfiguration() { CacheConfiguration cfg = defaultCacheConfiguration(); - cfg.setName(META_CACHE_NAME); - cfg.setCacheMode(REPLICATED); cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); cfg.setAtomicityMode(TRANSACTIONAL); @@ -112,8 +103,6 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { protected CacheConfiguration dataConfiguration() { CacheConfiguration cfg = defaultCacheConfiguration(); - cfg.setName(DATA_CACHE_NAME); - cfg.setCacheMode(PARTITIONED); cfg.setBackups(0); cfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(IGFS_GROUP_SIZE)); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java index 4e0f12b..5149493 100644 --- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java @@ -244,7 +244,8 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest { for (int i = 0; i < NODE_CNT; i++) { IgniteEx g = grid(i); - GridCacheAdapter<Object, Object> cache = ((IgniteKernal)g).internalCache(DATA_CACHE_NAME); + GridCacheAdapter<Object, Object> cache = ((IgniteKernal)g).internalCache( + g.igfsx("igfs").configuration().getDataCacheConfiguration().getName()); assertTrue("Data cache is not empty [keys=" + cache.keySet() + ", node=" + g.localNode().id() + ']', cache.isEmpty()); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java index 374d3d3..90ac4b1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractBaseSelfTest.java @@ -375,8 +375,6 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { TcpDiscoveryIpFinder ipFinder) throws Exception { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - igfsCfg.setDataCacheName("dataCache"); - igfsCfg.setMetaCacheName("metaCache"); igfsCfg.setName(igfsName); igfsCfg.setBlockSize(IGFS_BLOCK_SIZE); igfsCfg.setDefaultMode(mode); @@ -391,7 +389,6 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); - dataCacheCfg.setName("dataCache"); dataCacheCfg.setNearConfiguration(null); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setNearConfiguration(null); @@ -404,12 +401,16 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - metaCacheCfg.setName("metaCache"); metaCacheCfg.setNearConfiguration(null); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + prepareCacheConfigurations(dataCacheCfg, metaCacheCfg); + + igfsCfg.setDataCacheConfiguration(dataCacheCfg); + igfsCfg.setMetaCacheConfiguration(metaCacheCfg); + IgniteConfiguration cfg = new IgniteConfiguration(); if (useOptimizedMarshaller()) @@ -421,10 +422,7 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { discoSpi.setIpFinder(ipFinder); - prepareCacheConfigurations(dataCacheCfg, metaCacheCfg); - cfg.setDiscoverySpi(discoSpi); - cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); @@ -545,6 +543,7 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { /** * Create the file in the given IGFS and write provided data chunks to it. * + * @param uni FS tests adaptor. * @param file File. * @param chunks Data chunks. * @throws IOException In case of IO exception. @@ -942,7 +941,7 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { * @return The data cache. */ protected static GridCacheAdapter<IgfsBlockKey, byte[]> getDataCache(IgniteFileSystem igfs) { - String dataCacheName = igfs.configuration().getDataCacheName(); + String dataCacheName = igfs.configuration().getDataCacheConfiguration().getName(); IgniteEx igniteEx = ((IgfsEx)igfs).context().kernalContext().grid(); @@ -956,7 +955,7 @@ public abstract class IgfsAbstractBaseSelfTest extends IgfsCommonAbstractTest { * @return The data cache. */ protected static GridCacheAdapter<IgniteUuid, IgfsEntryInfo> getMetaCache(IgniteFileSystem igfs) { - String dataCacheName = igfs.configuration().getMetaCacheName(); + String dataCacheName = igfs.configuration().getMetaCacheConfiguration().getName(); IgniteEx igniteEx = ((IgfsEx)igfs).context().kernalContext().grid(); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBackupFailoverSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBackupFailoverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBackupFailoverSelfTest.java index 187aeeb..51b9ace 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBackupFailoverSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBackupFailoverSelfTest.java @@ -133,8 +133,6 @@ public class IgfsBackupFailoverSelfTest extends IgfsCommonAbstractTest { throws Exception { final FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - igfsCfg.setDataCacheName("dataCache"); - igfsCfg.setMetaCacheName("metaCache"); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(igfsBlockSize); igfsCfg.setDefaultMode(mode); @@ -142,7 +140,6 @@ public class IgfsBackupFailoverSelfTest extends IgfsCommonAbstractTest { CacheConfiguration<?,?> dataCacheCfg = defaultCacheConfiguration(); - dataCacheCfg.setName("dataCache"); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(affGrpSize)); @@ -151,11 +148,13 @@ public class IgfsBackupFailoverSelfTest extends IgfsCommonAbstractTest { CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - metaCacheCfg.setName("metaCache"); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + igfsCfg.setMetaCacheConfiguration(metaCacheCfg); + igfsCfg.setDataCacheConfiguration(dataCacheCfg); + IgniteConfiguration cfg = new IgniteConfiguration(); cfg.setGridName(gridName); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java index 57c709b..fba78c8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsBlockMessageSystemPoolStarvationSelfTest.java @@ -57,12 +57,6 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst /** Second node name. */ private static final String NODE_2_NAME = "node2"; - /** Data cache name. */ - private static final String DATA_CACHE_NAME = "data"; - - /** Meta cache name. */ - private static final String META_CACHE_NAME = "meta"; - /** Key in data caceh we will use to reproduce the issue. */ private static final Integer DATA_KEY = 1; @@ -212,13 +206,15 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst * @throws Exception If failed. */ private GridCacheAdapter dataCache(Ignite node) throws Exception { - return ((IgniteKernal)node).internalCache(DATA_CACHE_NAME); + return ((IgniteKernal)node).internalCache(((IgniteKernal)node).igfsx(null).configuration() + .getDataCacheConfiguration().getName()); } /** * Create node configuration. * * @param name Node name. + * @param ipFinder IpFinder. * @return Configuration. * @throws Exception If failed. */ @@ -226,7 +222,6 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst // Data cache configuration. CacheConfiguration dataCcfg = new CacheConfiguration(); - dataCcfg.setName(DATA_CACHE_NAME); dataCcfg.setCacheMode(CacheMode.REPLICATED); dataCcfg.setAtomicityMode(TRANSACTIONAL); dataCcfg.setWriteSynchronizationMode(FULL_SYNC); @@ -236,7 +231,6 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst // Meta cache configuration. CacheConfiguration metaCcfg = new CacheConfiguration(); - metaCcfg.setName(META_CACHE_NAME); metaCcfg.setCacheMode(CacheMode.REPLICATED); metaCcfg.setAtomicityMode(TRANSACTIONAL); metaCcfg.setWriteSynchronizationMode(FULL_SYNC); @@ -245,10 +239,10 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDefaultMode(IgfsMode.PRIMARY); - igfsCfg.setDataCacheName(DATA_CACHE_NAME); - igfsCfg.setMetaCacheName(META_CACHE_NAME); igfsCfg.setFragmentizerEnabled(false); igfsCfg.setBlockSize(1024); + igfsCfg.setDataCacheConfiguration(dataCcfg); + igfsCfg.setMetaCacheConfiguration(metaCcfg); // Ignite configuration. IgniteConfiguration cfg = getConfiguration(name); @@ -258,7 +252,6 @@ public class IgfsBlockMessageSystemPoolStarvationSelfTest extends IgfsCommonAbst discoSpi.setIpFinder(ipFinder); cfg.setDiscoverySpi(discoSpi); - cfg.setCacheConfiguration(dataCcfg, metaCcfg); cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java index 93171ba..1b8ab58 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java @@ -98,8 +98,6 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra private void startPrimary() throws Exception { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - igfsCfg.setDataCacheName("dataCache"); - igfsCfg.setMetaCacheName("metaCache"); igfsCfg.setName(IGFS_PRIMARY); igfsCfg.setBlockSize(512); igfsCfg.setDefaultMode(DUAL_SYNC); @@ -115,7 +113,6 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); - dataCacheCfg.setName("dataCache"); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setNearConfiguration(null); dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); @@ -129,12 +126,14 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - metaCacheCfg.setName("metaCache"); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setNearConfiguration(null); metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + igfsCfg.setMetaCacheConfiguration(metaCacheCfg); + igfsCfg.setDataCacheConfiguration(dataCacheCfg); + IgniteConfiguration cfg = new IgniteConfiguration(); cfg.setGridName("grid-primary"); @@ -144,7 +143,6 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); cfg.setDiscoverySpi(discoSpi); - cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); @@ -155,7 +153,7 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra igfsPrimary = (IgfsImpl)g.fileSystem(IGFS_PRIMARY); dataCache = igfsPrimary.context().kernalContext().cache().internalCache( - igfsPrimary.context().configuration().getDataCacheName()); + igfsPrimary.context().configuration().getDataCacheConfiguration().getName()); } /** @@ -166,8 +164,6 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra private void startSecondary() throws Exception { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - igfsCfg.setDataCacheName("dataCache"); - igfsCfg.setMetaCacheName("metaCache"); igfsCfg.setName(IGFS_SECONDARY); igfsCfg.setBlockSize(512); igfsCfg.setDefaultMode(PRIMARY); @@ -175,7 +171,6 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); - dataCacheCfg.setName("dataCache"); dataCacheCfg.setCacheMode(PARTITIONED); dataCacheCfg.setNearConfiguration(null); dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); @@ -185,12 +180,14 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - metaCacheCfg.setName("metaCache"); metaCacheCfg.setCacheMode(REPLICATED); metaCacheCfg.setNearConfiguration(null); metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + igfsCfg.setMetaCacheConfiguration(metaCacheCfg); + igfsCfg.setDataCacheConfiguration(dataCacheCfg); + IgniteConfiguration cfg = new IgniteConfiguration(); cfg.setGridName("grid-secondary"); @@ -200,7 +197,6 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); cfg.setDiscoverySpi(discoSpi); - cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); http://git-wip-us.apache.org/repos/asf/ignite/blob/3ca46d1a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java index 81caa91..090c86a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java @@ -41,12 +41,6 @@ import static org.apache.ignite.cache.CacheMode.REPLICATED; * Tests ensuring that IGFS data and meta caches are not "visible" through public API. */ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { - /** Meta-information cache name. */ - private static final String META_CACHE_NAME = "meta"; - - /** Data cache name. */ - private static final String DATA_CACHE_NAME = null; - /** Regular cache name. */ private static final String CACHE_NAME = "cache"; @@ -54,8 +48,7 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); - cfg.setCacheConfiguration(cacheConfiguration(META_CACHE_NAME), cacheConfiguration(DATA_CACHE_NAME), - cacheConfiguration(CACHE_NAME)); + cfg.setCacheConfiguration(cacheConfiguration(CACHE_NAME)); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); @@ -65,9 +58,9 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - igfsCfg.setMetaCacheName(META_CACHE_NAME); - igfsCfg.setDataCacheName(DATA_CACHE_NAME); igfsCfg.setName("igfs"); + igfsCfg.setMetaCacheConfiguration(cacheConfiguration("meta")); + igfsCfg.setDataCacheConfiguration(cacheConfiguration("data")); cfg.setFileSystemConfiguration(igfsCfg); @@ -80,7 +73,7 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { cacheCfg.setName(cacheName); - if (META_CACHE_NAME.equals(cacheName)) + if ("meta".equals(cacheName)) cacheCfg.setCacheMode(REPLICATED); else { cacheCfg.setCacheMode(PARTITIONED); @@ -128,7 +121,7 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { GridTestUtils.assertThrows(log(), new Callable<Object>() { @Override public Object call() throws Exception { - g.cache(META_CACHE_NAME); + g.cache(((IgniteKernal)g).igfsx("igfs").configuration().getMetaCacheConfiguration().getName()); return null; } @@ -136,7 +129,7 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { GridTestUtils.assertThrows(log(), new Callable<Object>() { @Override public Object call() throws Exception { - g.cache(DATA_CACHE_NAME); + g.cache(((IgniteKernal)g).igfsx("igfs").configuration().getDataCacheConfiguration().getName()); return null; }
