IGNITE-2834: IGFS: Implemented optional metadata co-location.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4af5316b Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4af5316b Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4af5316b Branch: refs/heads/ignite-1786 Commit: 4af5316bfe87b9cba179ce9307beb5eadd3b6a7f Parents: 88ffed1 Author: vozerov-gridgain <[email protected]> Authored: Fri Mar 18 17:07:58 2016 +0300 Committer: vozerov-gridgain <[email protected]> Committed: Fri Mar 18 17:07:58 2016 +0300 ---------------------------------------------------------------------- .../configuration/FileSystemConfiguration.java | 47 ++++++++++++++++ .../org/apache/ignite/internal/IgnitionEx.java | 10 ++-- .../processors/cache/GridCacheUtils.java | 14 +---- .../IgfsColocatedMetadataAffinityKeyMapper.java | 47 ++++++++++++++++ .../internal/processors/igfs/IgfsUtils.java | 57 ++++++++++++++++++++ 5 files changed, 157 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/4af5316b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java index 1a9c0fe..99d364e 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java @@ -19,6 +19,7 @@ package org.apache.ignite.configuration; import java.util.Map; import java.util.concurrent.ExecutorService; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration; import org.apache.ignite.igfs.IgfsMode; import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem; @@ -83,6 +84,9 @@ public class FileSystemConfiguration { /** Default value of whether to initialize default path modes. */ public static final boolean DFLT_INIT_DFLT_PATH_MODES = true; + /** Default value of metadata co-location flag. */ + public static boolean DFLT_COLOCATE_META = true; + /** IGFS instance name. */ private String name; @@ -164,6 +168,9 @@ public class FileSystemConfiguration { /** Whether to initialize default path modes. */ private boolean initDfltPathModes = DFLT_INIT_DFLT_PATH_MODES; + /** Metadata co-location flag. */ + private boolean colocateMeta = DFLT_COLOCATE_META; + /** * Constructs default configuration. */ @@ -184,6 +191,7 @@ public class FileSystemConfiguration { */ blockSize = cfg.getBlockSize(); bufSize = cfg.getStreamBufferSize(); + colocateMeta = cfg.isColocateMetadata(); dataCacheName = cfg.getDataCacheName(); dfltMode = cfg.getDefaultMode(); dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize(); @@ -830,6 +838,45 @@ public class FileSystemConfiguration { this.initDfltPathModes = initDfltPathModes; } + /** + * Get whether to co-locate metadata on a single node. + * <p> + * Normally Ignite spread ownership of particular keys among all cache nodes. Transaction with keys owned by + * different nodes will produce more network traffic and will require more time to complete comparing to + * transaction with keys owned only by a single node. + * <p> + * IGFS stores information about file system structure (metadata) inside a transactional cache configured through + * {@link #getMetaCacheName()} property. Metadata updates caused by operations on IGFS usually require several + * intearnal keys to be updated. As IGFS metadata cache usually operates in {@link CacheMode#REPLICATED} mode, + * meaning that all nodes have all metadata locally, it makes sense to give a hint to Ignite to co-locate + * ownership of all metadata keys on a single node. This will decrease amount of network trips required to update + * metadata and hence could improve performance. + * <p> + * This property should be disabled if you see excessive CPU and network load on a single node, which + * degrades performance and cannot be explained by business logic of your application. + * <p> + * This settings is only used if metadata cache is configured in {@code CacheMode#REPLICATED} mode. Otherwise it + * is ignored. + * <p> + * Defaults to {@link #DFLT_COLOCATE_META}. + * + * @return {@code True} if metadata co-location is enabled. + */ + public boolean isColocateMetadata() { + return colocateMeta; + } + + /** + * Set metadata co-location flag. + * <p> + * See {@link #isColocateMetadata()} for more information. + * + * @param colocateMeta Whether metadata co-location is enabled. + */ + public void setColocateMetadata(boolean colocateMeta) { + this.colocateMeta = colocateMeta; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(FileSystemConfiguration.class, this); http://git-wip-us.apache.org/repos/asf/ignite/blob/4af5316b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java index 4796581..533b6d8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java @@ -62,6 +62,7 @@ import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; import org.apache.ignite.internal.util.GridConcurrentHashSet; import org.apache.ignite.internal.util.IgniteUtils; @@ -1950,14 +1951,11 @@ public class IgnitionEx { cfg.setCacheConfiguration(cacheCfgs.toArray(new CacheConfiguration[cacheCfgs.size()])); - // Iterate over IGFS caches and set "copyOnRead" flag to "false". Note that we do this after cloning - // to leave user object unchanged. + // Iterate over IGFS caches and prepare their configurations if needed. assert cfg.getCacheConfiguration() != null; - for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) { - if (CU.isIgfsCache(cfg, ccfg.getName())) - ccfg.setCopyOnRead(false); - } + for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) + IgfsUtils.prepareCacheConfiguration(cfg, ccfg); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/4af5316b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java index 94307ca..1cdd303 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java @@ -73,6 +73,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException; import org.apache.ignite.internal.util.lang.IgniteInClosureX; import org.apache.ignite.internal.util.typedef.C1; @@ -1382,18 +1383,7 @@ public class GridCacheUtils { * @return {@code True} in this is IGFS data or meta cache. */ public static boolean isIgfsCache(IgniteConfiguration cfg, @Nullable String cacheName) { - FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); - - if (igfsCfgs != null) { - for (FileSystemConfiguration igfsCfg : igfsCfgs) { - // IGFS config probably has not been validated yet => possible NPE, so we check for null. - if (igfsCfg != null && - (F.eq(cacheName, igfsCfg.getDataCacheName()) || F.eq(cacheName, igfsCfg.getMetaCacheName()))) - return true; - } - } - - return false; + return IgfsUtils.isIgfsCache(cfg, cacheName); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/4af5316b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsColocatedMetadataAffinityKeyMapper.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsColocatedMetadataAffinityKeyMapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsColocatedMetadataAffinityKeyMapper.java new file mode 100644 index 0000000..d8c10b0 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsColocatedMetadataAffinityKeyMapper.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.igfs; + +import org.apache.ignite.cache.affinity.AffinityKeyMapper; +import org.apache.ignite.internal.util.typedef.internal.S; + +/** + * Special implementation of affinity mapper which maps all metadata to the same primary. + */ +public class IgfsColocatedMetadataAffinityKeyMapper implements AffinityKeyMapper { + /** */ + private static final long serialVersionUID = 0L; + + /** Affinity. */ + private static final Integer AFF = 1; + + /** {@inheritDoc} */ + @Override public Object affinityKey(Object key) { + return AFF; + } + + /** {@inheritDoc} */ + @Override public void reset() { + // No-op. + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IgfsColocatedMetadataAffinityKeyMapper.class, this); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/4af5316b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java index b176e21..ef7d5c7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java @@ -20,9 +20,12 @@ package org.apache.ignite.internal.processors.igfs; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.cluster.ClusterTopologyException; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.FileSystemConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.IgfsEvent; import org.apache.ignite.igfs.IgfsException; import org.apache.ignite.igfs.IgfsPath; @@ -264,4 +267,58 @@ public class IgfsUtils { if (evts.isRecordable(type)) evts.record(new IgfsEvent(path, locNode, type)); } + + /** + * @param cfg Grid configuration. + * @param cacheName Cache name. + * @return {@code True} in this is IGFS data or meta cache. + */ + public static boolean isIgfsCache(IgniteConfiguration cfg, @Nullable String cacheName) { + FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); + + if (igfsCfgs != null) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { + // IGFS config probably has not been validated yet => possible NPE, so we check for null. + if (igfsCfg != null) { + if (F.eq(cacheName, igfsCfg.getDataCacheName()) || F.eq(cacheName, igfsCfg.getMetaCacheName())) + return true; + } + } + } + + return false; + } + + /** + * Prepare cache configuration if this is IGFS meta or data cache. + * + * @param cfg Configuration. + * @param ccfg Cache configuration. + */ + public static void prepareCacheConfiguration(IgniteConfiguration cfg, CacheConfiguration ccfg) { + FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); + + if (igfsCfgs != null) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { + if (igfsCfg != null) { + if (F.eq(ccfg.getName(), igfsCfg.getMetaCacheName())) { + ccfg.setCopyOnRead(false); + + // Set co-located affinity mapper if needed. + if (igfsCfg.isColocateMetadata() && ccfg.getCacheMode() == CacheMode.REPLICATED && + ccfg.getAffinityMapper() == null) + ccfg.setAffinityMapper(new IgfsColocatedMetadataAffinityKeyMapper()); + + return; + } + + if (F.eq(ccfg.getName(), igfsCfg.getDataCacheName())) { + ccfg.setCopyOnRead(false); + + return; + } + } + } + } + } } \ No newline at end of file
