Repository: ignite Updated Branches: refs/heads/ignite-2.0 6cc43cae9 -> 800fb87a1
IGNITE-5023 .NET: IgniteConfiguration.MemoryConfiguration Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/800fb87a Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/800fb87a Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/800fb87a Branch: refs/heads/ignite-2.0 Commit: 800fb87a1c095edfdbeb090ea2125a6fcc0f742f Parents: 6cc43ca Author: Pavel Tupitsyn <ptupit...@apache.org> Authored: Thu Apr 20 19:08:05 2017 +0300 Committer: Pavel Tupitsyn <ptupit...@apache.org> Committed: Thu Apr 20 19:08:05 2017 +0300 ---------------------------------------------------------------------- .../configuration/DataPageEvictionMode.java | 10 +- .../utils/PlatformConfigurationUtils.java | 170 ++++++++++++++++--- .../Cache/CacheConfigurationTest.cs | 15 +- .../IgniteConfigurationSerializerTest.cs | 48 ++++++ .../IgniteConfigurationTest.cs | 56 ++++++ .../Apache.Ignite.Core.csproj | 3 + .../Cache/Configuration/CacheConfiguration.cs | 8 + .../Cache/Configuration/DataPageEvictionMode.cs | 59 +++++++ .../Cache/Configuration/MemoryConfiguration.cs | 151 ++++++++++++++++ .../Configuration/MemoryPolicyConfiguration.cs | 119 +++++++++++++ .../Apache.Ignite.Core/IgniteConfiguration.cs | 21 +++ .../IgniteConfigurationSection.xsd | 85 ++++++++++ 12 files changed, 714 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/core/src/main/java/org/apache/ignite/configuration/DataPageEvictionMode.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataPageEvictionMode.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataPageEvictionMode.java index 91f707e..437721b 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/DataPageEvictionMode.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataPageEvictionMode.java @@ -18,16 +18,16 @@ package org.apache.ignite.configuration; /** - * The enumeration defines an algorithm to be used for memory pages eviction. A mode is set for a specific + * Defines memory page eviction algorithm. A mode is set for a specific * {@link MemoryPolicyConfiguration}. Only data pages, that store key-value entries, are eligible for eviction. The - * other types of pages like index or system pages are evictable. + * other types of pages, like index or system pages, are not evictable. */ public enum DataPageEvictionMode { /** Eviction is disabled. */ DISABLED, /** - * Activates Random-LRU algorithm that works the way below: + * Random-LRU algorithm. * <ul> * <li>Once a memory region defined by a memory policy is configured, an off-heap array is allocated to track * last usage timestamp for every individual data page. The size of the array is calculated this way - size = @@ -36,13 +36,13 @@ public enum DataPageEvictionMode { * tracking array is calculated this way - index = (pageAddress / {@link MemoryPolicyConfiguration#getSize()}</li> * <li>When it's required to evict some pages, the algorithm randomly chooses 5 indexes from the tracking array and * evicts a page with the latest timestamp. If some of the indexes point to non-data pages (index or system pages) - * then the algorithm peaks another ones.</li> + * then the algorithm picks other pages.</li> * </ul> */ RANDOM_LRU, /** - * Activates Random-2-LRU algorithm which is a scan resistant version of Random-LRU. + * Random-2-LRU algorithm: scan-resistant version of Random-LRU. * <p> * This algorithm differs from Random-LRU only in a way that two latest access timestamps are stored for every * data page. At the eviction time, a minimum between two latest timestamps is taken for further comparison with http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java index 98ce61d..b79ab4b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/utils/PlatformConfigurationUtils.java @@ -52,7 +52,10 @@ import org.apache.ignite.cache.eviction.lru.LruEvictionPolicy; import org.apache.ignite.configuration.AtomicConfiguration; import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.configuration.TransactionConfiguration; import org.apache.ignite.internal.binary.BinaryArrayIdentityResolver; @@ -169,6 +172,11 @@ public class PlatformConfigurationUtils { ccfg.setWriteThrough(in.readBoolean()); ccfg.setStatisticsEnabled(in.readBoolean()); + String memoryPolicyName = in.readString(); + + if (memoryPolicyName != null) + ccfg.setMemoryPolicyName(memoryPolicyName); + Object storeFactory = in.readObjectDetached(); if (storeFactory != null) @@ -516,20 +524,36 @@ public class PlatformConfigurationUtils { * @param cfg Configuration. */ public static void readIgniteConfiguration(BinaryRawReaderEx in, IgniteConfiguration cfg) { - if (in.readBoolean()) cfg.setClientMode(in.readBoolean()); - int[] eventTypes = in.readIntArray(); if (eventTypes != null) cfg.setIncludeEventTypes(eventTypes); - if (in.readBoolean()) cfg.setMetricsExpireTime(in.readLong()); - if (in.readBoolean()) cfg.setMetricsHistorySize(in.readInt()); - if (in.readBoolean()) cfg.setMetricsLogFrequency(in.readLong()); - if (in.readBoolean()) cfg.setMetricsUpdateFrequency(in.readLong()); - if (in.readBoolean()) cfg.setNetworkSendRetryCount(in.readInt()); - if (in.readBoolean()) cfg.setNetworkSendRetryDelay(in.readLong()); - if (in.readBoolean()) cfg.setNetworkTimeout(in.readLong()); - String workDir = in.readString(); if (workDir != null) cfg.setWorkDirectory(workDir); - String localHost = in.readString(); if (localHost != null) cfg.setLocalHost(localHost); - if (in.readBoolean()) cfg.setDaemon(in.readBoolean()); - if (in.readBoolean()) cfg.setLateAffinityAssignment(in.readBoolean()); - if (in.readBoolean()) cfg.setFailureDetectionTimeout(in.readLong()); + if (in.readBoolean()) + cfg.setClientMode(in.readBoolean()); + int[] eventTypes = in.readIntArray(); + if (eventTypes != null) cfg.setIncludeEventTypes(eventTypes); + if (in.readBoolean()) + cfg.setMetricsExpireTime(in.readLong()); + if (in.readBoolean()) + cfg.setMetricsHistorySize(in.readInt()); + if (in.readBoolean()) + cfg.setMetricsLogFrequency(in.readLong()); + if (in.readBoolean()) + cfg.setMetricsUpdateFrequency(in.readLong()); + if (in.readBoolean()) + cfg.setNetworkSendRetryCount(in.readInt()); + if (in.readBoolean()) + cfg.setNetworkSendRetryDelay(in.readLong()); + if (in.readBoolean()) + cfg.setNetworkTimeout(in.readLong()); + String workDir = in.readString(); + if (workDir != null) + cfg.setWorkDirectory(workDir); + String localHost = in.readString(); + if (localHost != null) + cfg.setLocalHost(localHost); + if (in.readBoolean()) + cfg.setDaemon(in.readBoolean()); + if (in.readBoolean()) + cfg.setLateAffinityAssignment(in.readBoolean()); + if (in.readBoolean()) + cfg.setFailureDetectionTimeout(in.readLong()); readCacheConfigurations(in, cfg); readDiscoveryConfiguration(in, cfg); @@ -616,6 +640,8 @@ public class PlatformConfigurationUtils { break; } + cfg.setMemoryConfiguration(readMemoryConfiguration(in)); + readPluginConfiguration(cfg, in); } @@ -772,6 +798,7 @@ public class PlatformConfigurationUtils { writer.writeBoolean(ccfg.isReadThrough()); writer.writeBoolean(ccfg.isWriteThrough()); writer.writeBoolean(ccfg.isStatisticsEnabled()); + writer.writeString(ccfg.getMemoryPolicyName()); if (ccfg.getCacheStoreFactory() instanceof PlatformDotNetCacheStoreFactoryNative) writer.writeObject(((PlatformDotNetCacheStoreFactoryNative)ccfg.getCacheStoreFactory()).getNativeFactory()); @@ -914,20 +941,30 @@ public class PlatformConfigurationUtils { assert w != null; assert cfg != null; - w.writeBoolean(true); w.writeBoolean(cfg.isClientMode()); + w.writeBoolean(true); + w.writeBoolean(cfg.isClientMode()); w.writeIntArray(cfg.getIncludeEventTypes()); - w.writeBoolean(true); w.writeLong(cfg.getMetricsExpireTime()); - w.writeBoolean(true); w.writeInt(cfg.getMetricsHistorySize()); - w.writeBoolean(true); w.writeLong(cfg.getMetricsLogFrequency()); - w.writeBoolean(true); w.writeLong(cfg.getMetricsUpdateFrequency()); - w.writeBoolean(true); w.writeInt(cfg.getNetworkSendRetryCount()); - w.writeBoolean(true); w.writeLong(cfg.getNetworkSendRetryDelay()); - w.writeBoolean(true); w.writeLong(cfg.getNetworkTimeout()); + w.writeBoolean(true); + w.writeLong(cfg.getMetricsExpireTime()); + w.writeBoolean(true); + w.writeInt(cfg.getMetricsHistorySize()); + w.writeBoolean(true); + w.writeLong(cfg.getMetricsLogFrequency()); + w.writeBoolean(true); + w.writeLong(cfg.getMetricsUpdateFrequency()); + w.writeBoolean(true); + w.writeInt(cfg.getNetworkSendRetryCount()); + w.writeBoolean(true);w.writeLong(cfg.getNetworkSendRetryDelay()); + w.writeBoolean(true); + w.writeLong(cfg.getNetworkTimeout()); w.writeString(cfg.getWorkDirectory()); w.writeString(cfg.getLocalHost()); - w.writeBoolean(true); w.writeBoolean(cfg.isDaemon()); - w.writeBoolean(true); w.writeBoolean(cfg.isLateAffinityAssignment()); - w.writeBoolean(true); w.writeLong(cfg.getFailureDetectionTimeout()); + w.writeBoolean(true); + w.writeBoolean(cfg.isDaemon()); + w.writeBoolean(true); + w.writeBoolean(cfg.isLateAffinityAssignment()); + w.writeBoolean(true); + w.writeLong(cfg.getFailureDetectionTimeout()); CacheConfiguration[] cacheCfg = cfg.getCacheConfiguration(); @@ -1033,6 +1070,8 @@ public class PlatformConfigurationUtils { w.writeLong(((MemoryEventStorageSpi)eventStorageSpi).getExpireAgeMs()); } + writeMemoryConfiguration(w, cfg.getMemoryConfiguration()); + w.writeString(cfg.getIgniteHome()); w.writeLong(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getInit()); @@ -1273,6 +1312,87 @@ public class PlatformConfigurationUtils { } /** + * Reads the memory configuration. + * + * @param in Reader + * @return Config. + */ + private static MemoryConfiguration readMemoryConfiguration(BinaryRawReader in) { + if (!in.readBoolean()) + return null; + + MemoryConfiguration res = new MemoryConfiguration(); + + res.setSystemCacheMemorySize(in.readLong()) + .setPageSize(in.readInt()) + .setConcurrencyLevel(in.readInt()) + .setDefaultMemoryPolicyName(in.readString()); + + int cnt = in.readInt(); + + if (cnt > 0) { + MemoryPolicyConfiguration[] plcs = new MemoryPolicyConfiguration[cnt]; + + for (int i = 0; i < cnt; i++) { + MemoryPolicyConfiguration cfg = new MemoryPolicyConfiguration(); + + cfg.setName(in.readString()) + .setSize(in.readLong()) + .setSwapFilePath(in.readString()) + .setPageEvictionMode(DataPageEvictionMode.values()[in.readInt()]) + .setEvictionThreshold(in.readDouble()) + .setEmptyPagesPoolSize(in.readInt()); + + plcs[i] = cfg; + } + + res.setMemoryPolicies(plcs); + } + + return res; + } + + /** + * Writes the memory configuration. + * + * @param w Writer. + * @param cfg Config. + */ + private static void writeMemoryConfiguration(BinaryRawWriter w, MemoryConfiguration cfg) { + if (cfg == null) { + w.writeBoolean(false); + return; + } + + w.writeBoolean(true); + + w.writeLong(cfg.getSystemCacheMemorySize()); + w.writeInt(cfg.getPageSize()); + w.writeInt(cfg.getConcurrencyLevel()); + w.writeString(cfg.getDefaultMemoryPolicyName()); + + MemoryPolicyConfiguration[] plcs = cfg.getMemoryPolicies(); + + if (plcs != null) { + w.writeInt(plcs.length); + + for (MemoryPolicyConfiguration plc : plcs) { + w.writeString(plc.getName()); + w.writeLong(plc.getSize()); + w.writeString(plc.getSwapFilePath()); + w.writeInt(plc.getPageEvictionMode().ordinal()); + w.writeDouble(plc.getEvictionThreshold()); + w.writeInt(plc.getEmptyPagesPoolSize()); + } + } + else { + w.writeInt(0); + } + } + + + + /** * Private constructor. */ private PlatformConfigurationUtils() { http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs index 7a120bc..6406d32 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheConfigurationTest.cs @@ -64,7 +64,18 @@ namespace Apache.Ignite.Core.Tests.Cache GetCustomCacheConfiguration2() }, IgniteInstanceName = CacheName, - BinaryConfiguration = new BinaryConfiguration(typeof (Entity)) + BinaryConfiguration = new BinaryConfiguration(typeof(Entity)), + MemoryConfiguration = new MemoryConfiguration + { + MemoryPolicies = new[] + { + new MemoryPolicyConfiguration + { + Name = "myMemPolicy", + Size = 99 * 1024 * 1024 + } + } + } }; _ignite = Ignition.Start(cfg); @@ -233,6 +244,7 @@ namespace Apache.Ignite.Core.Tests.Cache Assert.AreEqual(x.WriteBehindFlushFrequency, y.WriteBehindFlushFrequency); Assert.AreEqual(x.WriteBehindFlushSize, y.WriteBehindFlushSize); Assert.AreEqual(x.EnableStatistics, y.EnableStatistics); + Assert.AreEqual(x.MemoryPolicyName, y.MemoryPolicyName); if (x.ExpiryPolicyFactory != null) Assert.AreEqual(x.ExpiryPolicyFactory.CreateInstance().GetType(), @@ -546,6 +558,7 @@ namespace Apache.Ignite.Core.Tests.Cache }, ExpiryPolicyFactory = new ExpiryFactory(), EnableStatistics = true, + MemoryPolicyName = "myMemPolicy", PluginConfigurations = new[] { new MyPluginConfiguration() } }; } http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs index 4a197e5..2d8cc20 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationSerializerTest.cs @@ -134,6 +134,11 @@ namespace Apache.Ignite.Core.Tests <iPluginConfiguration type='Apache.Ignite.Core.Tests.Plugin.TestIgnitePluginConfiguration, Apache.Ignite.Core.Tests' /> </pluginConfigurations> <eventStorageSpi type='MemoryEventStorageSpi' expirationTimeout='00:00:23.45' maxEventCount='129' /> + <memoryConfiguration concurrencyLevel='3' defaultMemoryPolicyName='dfPlc' pageSize='45' systemCacheMemorySize='67'> + <memoryPolicies> + <memoryPolicyConfiguration emptyPagesPoolSize='1' evictionThreshold='0.2' name='dfPlc' pageEvictionMode='RandomLru' size='89' swapFilePath='abc' /> + </memoryPolicies> + </memoryConfiguration> </igniteConfig>"; var cfg = IgniteConfiguration.FromXml(xml); @@ -248,6 +253,21 @@ namespace Apache.Ignite.Core.Tests Assert.IsNotNull(eventStorage); Assert.AreEqual(23.45, eventStorage.ExpirationTimeout.TotalSeconds); Assert.AreEqual(129, eventStorage.MaxEventCount); + + var memCfg = cfg.MemoryConfiguration; + Assert.IsNotNull(memCfg); + Assert.AreEqual(3, memCfg.ConcurrencyLevel); + Assert.AreEqual("dfPlc", memCfg.DefaultMemoryPolicyName); + Assert.AreEqual(45, memCfg.PageSize); + Assert.AreEqual(67, memCfg.SystemCacheMemorySize); + + var memPlc = memCfg.MemoryPolicies.Single(); + Assert.AreEqual(1, memPlc.EmptyPagesPoolSize); + Assert.AreEqual(0.2, memPlc.EvictionThreshold); + Assert.AreEqual("dfPlc", memPlc.Name); + Assert.AreEqual(DataPageEvictionMode.RandomLru, memPlc.PageEvictionMode); + Assert.AreEqual("abc", memPlc.SwapFilePath); + Assert.AreEqual(89, memPlc.Size); } /// <summary> @@ -774,6 +794,34 @@ namespace Apache.Ignite.Core.Tests { ExpirationTimeout = TimeSpan.FromMilliseconds(12345), MaxEventCount = 257 + }, + MemoryConfiguration = new MemoryConfiguration + { + ConcurrencyLevel = 3, + DefaultMemoryPolicyName = "somePolicy", + PageSize = 4, + SystemCacheMemorySize = 5, + MemoryPolicies = new[] + { + new MemoryPolicyConfiguration + { + Name = "myDefaultPlc", + PageEvictionMode = DataPageEvictionMode.Random2Lru, + Size = 345 * 1024 * 1024, + EvictionThreshold = 0.88, + EmptyPagesPoolSize = 77, + SwapFilePath = "myPath1" + }, + new MemoryPolicyConfiguration + { + Name = "customPlc", + PageEvictionMode = DataPageEvictionMode.RandomLru, + Size = 456 * 1024 * 1024, + EvictionThreshold = 0.77, + EmptyPagesPoolSize = 66, + SwapFilePath = "somePath2" + } + } } }; } http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs index 19053f2..8da2616 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteConfigurationTest.cs @@ -79,6 +79,8 @@ namespace Apache.Ignite.Core.Tests CheckDefaultValueAttributes(new AtomicConfiguration()); CheckDefaultValueAttributes(new TransactionConfiguration()); CheckDefaultValueAttributes(new MemoryEventStorageSpi()); + CheckDefaultValueAttributes(new MemoryConfiguration()); + CheckDefaultValueAttributes(new MemoryPolicyConfiguration()); } /// <summary> @@ -190,6 +192,32 @@ namespace Apache.Ignite.Core.Tests Assert.IsNotNull(resEventCfg); Assert.AreEqual(eventCfg.ExpirationTimeout, resEventCfg.ExpirationTimeout); Assert.AreEqual(eventCfg.MaxEventCount, resEventCfg.MaxEventCount); + + var memCfg = cfg.MemoryConfiguration; + var resMemCfg = resCfg.MemoryConfiguration; + Assert.IsNotNull(memCfg); + Assert.IsNotNull(resMemCfg); + Assert.AreEqual(memCfg.PageSize, resMemCfg.PageSize); + Assert.AreEqual(memCfg.ConcurrencyLevel, resMemCfg.ConcurrencyLevel); + Assert.AreEqual(memCfg.DefaultMemoryPolicyName, resMemCfg.DefaultMemoryPolicyName); + Assert.AreEqual(memCfg.SystemCacheMemorySize, resMemCfg.SystemCacheMemorySize); + Assert.IsNotNull(memCfg.MemoryPolicies); + Assert.IsNotNull(resMemCfg.MemoryPolicies); + Assert.AreEqual(2, memCfg.MemoryPolicies.Count); + Assert.AreEqual(2, resMemCfg.MemoryPolicies.Count); + + for (var i = 0; i < memCfg.MemoryPolicies.Count; i++) + { + var plc = memCfg.MemoryPolicies.Skip(i).First(); + var resPlc = resMemCfg.MemoryPolicies.Skip(i).First(); + + Assert.AreEqual(plc.PageEvictionMode, resPlc.PageEvictionMode); + Assert.AreEqual(plc.Size, resPlc.Size); + Assert.AreEqual(plc.EmptyPagesPoolSize, resPlc.EmptyPagesPoolSize); + Assert.AreEqual(plc.EvictionThreshold, resPlc.EvictionThreshold); + Assert.AreEqual(plc.Name, resPlc.Name); + Assert.AreEqual(plc.SwapFilePath, resPlc.SwapFilePath); + } } } @@ -534,6 +562,34 @@ namespace Apache.Ignite.Core.Tests { ExpirationTimeout = TimeSpan.FromSeconds(5), MaxEventCount = 10 + }, + MemoryConfiguration = new MemoryConfiguration + { + ConcurrencyLevel = 3, + DefaultMemoryPolicyName = "myDefaultPlc", + PageSize = 2048, + SystemCacheMemorySize = 13 * 1024 * 1024, + MemoryPolicies = new[] + { + new MemoryPolicyConfiguration + { + Name = "myDefaultPlc", + PageEvictionMode = DataPageEvictionMode.Random2Lru, + Size = 345 * 1024 * 1024, + EvictionThreshold = 0.88, + EmptyPagesPoolSize = 77, + SwapFilePath = "myPath1" + }, + new MemoryPolicyConfiguration + { + Name = "customPlc", + PageEvictionMode = DataPageEvictionMode.RandomLru, + Size = 456 * 1024 * 1024, + EvictionThreshold = 0.77, + EmptyPagesPoolSize = 66, + SwapFilePath = "somePath2" + } + } } }; } http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj index a298f6f..9d1f9fc 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj @@ -92,6 +92,8 @@ </ItemGroup> <ItemGroup> <Compile Include="Binary\BinaryBasicNameMapper.cs" /> + <Compile Include="Cache\Configuration\DataPageEvictionMode.cs" /> + <Compile Include="Cache\Configuration\MemoryPolicyConfiguration.cs" /> <Compile Include="Common\ExceptionFactory.cs" /> <Compile Include="Events\IEventStorageSpi.cs" /> <Compile Include="Events\MemoryEventStorageSpi.cs" /> @@ -458,6 +460,7 @@ <Compile Include="Log\ILogger.cs" /> <Compile Include="Log\LoggerExtensions.cs" /> <Compile Include="Log\LogLevel.cs" /> + <Compile Include="Cache\Configuration\MemoryConfiguration.cs" /> <Compile Include="Messaging\Package-Info.cs" /> <Compile Include="Package-Info.cs" /> <Compile Include="Lifecycle\ILifecycleHandler.cs" /> http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs index 91a5b1e..3586989 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/CacheConfiguration.cs @@ -233,6 +233,7 @@ namespace Apache.Ignite.Core.Cache.Configuration ReadThrough = reader.ReadBoolean(); WriteThrough = reader.ReadBoolean(); EnableStatistics = reader.ReadBoolean(); + MemoryPolicyName = reader.ReadString(); CacheStoreFactory = reader.ReadObject<IFactory<ICacheStore>>(); var count = reader.ReadInt(); @@ -288,6 +289,7 @@ namespace Apache.Ignite.Core.Cache.Configuration writer.WriteBoolean(ReadThrough); writer.WriteBoolean(WriteThrough); writer.WriteBoolean(EnableStatistics); + writer.WriteString(MemoryPolicyName); writer.WriteObject(CacheStoreFactory); if (QueryEntities != null) @@ -623,5 +625,11 @@ namespace Apache.Ignite.Core.Cache.Configuration /// </summary> [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")] public ICollection<ICachePluginConfiguration> PluginConfigurations { get; set; } + + /// <summary> + /// Gets or sets the name of the <see cref="MemoryPolicyConfiguration"/> for this cache. + /// See <see cref="IgniteConfiguration.MemoryConfiguration"/>. + /// </summary> + public string MemoryPolicyName { get; set; } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/DataPageEvictionMode.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/DataPageEvictionMode.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/DataPageEvictionMode.cs new file mode 100644 index 0000000..f3897e6 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/DataPageEvictionMode.cs @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Cache.Configuration +{ + /// <summary> + /// Memory page eviction mode. + /// Only data pages, that store key-value entries, are eligible for eviction. + /// The other types of pages, like index or system pages, are not evictable. + /// </summary> + public enum DataPageEvictionMode + { + /// <summary> + /// Eviction is disabled. + /// </summary> + Disabled, + + /// <summary> + /// Random-LRU algorithm. + /// <para /> + /// Once a memory region defined by a memory policy is configured, an off-heap array is allocated to track + /// last usage timestamp for every individual data page. The size of the array equals to + /// <see cref="MemoryPolicyConfiguration.Size"/> / <see cref="MemoryConfiguration.PageSize"/>. + /// <para /> + /// When a data page is accessed, its timestamp gets updated in the tracking array. The page index in the + /// tracking array equals to pageAddress / <see cref="MemoryPolicyConfiguration.Size"/>. + /// <para /> + /// When some pages need to be evicted, the algorithm randomly chooses 5 indexes from the tracking array and + /// evicts a page with the latest timestamp. If some of the indexes point to non-data pages + /// (index or system pages) then the algorithm picks other pages. + /// </summary> + RandomLru, + + /// <summary> + /// Activates Random-2-LRU algorithm which is a scan resistant version of Random-LRU. + /// <para /> + /// This algorithm differs from Random-LRU only in a way that two latest access timestamps are stored for every + /// data page. At the eviction time, a minimum between two latest timestamps is taken for further + /// comparison with minimums of other pages that might be evicted. LRU-2 outperforms LRU by + /// resolving "one-hit wonder" problem - if a data page is accessed rarely, but accidentally accessed once, + /// its protected from eviction for a long time. + /// </summary> + Random2Lru + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryConfiguration.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryConfiguration.cs new file mode 100644 index 0000000..9b1016d --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryConfiguration.cs @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Cache.Configuration +{ + using System.Collections.Generic; + using System.ComponentModel; + using System.Diagnostics; + using System.Diagnostics.CodeAnalysis; + using System.Linq; + using Apache.Ignite.Core.Binary; + using Apache.Ignite.Core.Common; + + /// <summary> + /// A page memory configuration for an Apache Ignite node. The page memory is a manageable off-heap based + /// memory architecture that divides all continuously allocated memory regions into pages of fixed size. + /// An individual page can store one or many cache key-value entries that allows reusing the memory + /// in the most efficient way and avoid memory fragmentation issues. + /// <para /> + /// By default, the page memory allocates a single continuous memory region. All the caches that + /// will be configured in an application will be mapped to this memory region by default, + /// thus, all the cache data will reside in that memory region. + /// <para /> + /// If initial size of the default memory region doesn't satisfy requirements or it's + /// required to have multiple memory regions with different properties + /// then <see cref="MemoryPolicyConfiguration" /> can be used for both scenarios. + /// For instance, using memory policies you can define memory regions of different maximum size, + /// eviction policies, swapping options, etc. Once you define a new memory region you can bind + /// particular Ignite caches to it. <para /> + /// To learn more about memory policies refer to <see cref="MemoryPolicyConfiguration" /> documentation. + /// </summary> + public class MemoryConfiguration + { + /// <summary> + /// The default system cache memory size. + /// </summary> + public const long DefaultSystemCacheMemorySize = 100 * 1024 * 1024; + + /// <summary> + /// The default page size. + /// </summary> + public const int DefaultPageSize = 2 * 1024; + + /// <summary> + /// Initializes a new instance of the <see cref="MemoryConfiguration"/> class. + /// </summary> + public MemoryConfiguration() + { + SystemCacheMemorySize = DefaultSystemCacheMemorySize; + PageSize = DefaultPageSize; + } + + /// <summary> + /// Initializes a new instance of the <see cref="MemoryConfiguration"/> class. + /// </summary> + /// <param name="reader">The reader.</param> + public MemoryConfiguration(IBinaryRawReader reader) + { + Debug.Assert(reader != null); + + SystemCacheMemorySize = reader.ReadLong(); + PageSize = reader.ReadInt(); + ConcurrencyLevel = reader.ReadInt(); + DefaultMemoryPolicyName = reader.ReadString(); + + var count = reader.ReadInt(); + + if (count > 0) + { + MemoryPolicies = Enumerable.Range(0, count) + .Select(x => new MemoryPolicyConfiguration(reader)) + .ToArray(); + } + } + + /// <summary> + /// Writes this instance to a writer. + /// </summary> + /// <param name="writer">The writer.</param> + internal void Write(IBinaryRawWriter writer) + { + Debug.Assert(writer != null); + + writer.WriteLong(SystemCacheMemorySize); + writer.WriteInt(PageSize); + writer.WriteInt(ConcurrencyLevel); + writer.WriteString(DefaultMemoryPolicyName); + + if (MemoryPolicies != null) + { + writer.WriteInt(MemoryPolicies.Count); + + foreach (var policy in MemoryPolicies) + { + if (policy == null) + { + throw new IgniteException("MemoryConfiguration.MemoryPolicies must not contain null items."); + } + + policy.Write(writer); + } + } + else + { + writer.WriteInt(0); + } + } + + /// <summary> + /// Gets or sets the size of a memory chunk reserved for system cache needs. + /// </summary> + [DefaultValue(DefaultSystemCacheMemorySize)] + public long SystemCacheMemorySize { get; set; } + + /// <summary> + /// Gets or sets the size of the memory page. + /// </summary> + [DefaultValue(DefaultPageSize)] + public int PageSize { get; set; } + + /// <summary> + /// Gets or sets the number of concurrent segments in Ignite internal page mapping tables. + /// </summary> + public int ConcurrencyLevel { get;set; } + + /// <summary> + /// Gets or sets the name of the default memory policy in <see cref="MemoryPolicies"/>. + /// </summary> + public string DefaultMemoryPolicyName { get; set; } + + /// <summary> + /// Gets or sets the memory policies. + /// </summary> + [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")] + public ICollection<MemoryPolicyConfiguration> MemoryPolicies { get; set; } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs new file mode 100644 index 0000000..9e21910 --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/Configuration/MemoryPolicyConfiguration.cs @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Cache.Configuration +{ + using System.ComponentModel; + using Apache.Ignite.Core.Binary; + + /// <summary> + /// Defines page memory policy configuration. See <see cref="MemoryConfiguration.MemoryPolicies"/>. + /// </summary> + public class MemoryPolicyConfiguration + { + /// <summary> + /// The default eviction threshold. + /// </summary> + public const double DefaultEvictionThreshold = 0.9; + + /// <summary> + /// The default empty pages pool size. + /// </summary> + public const int DefaultEmptyPagesPoolSize = 100; + + /// <summary> + /// Initializes a new instance of the <see cref="MemoryPolicyConfiguration"/> class. + /// </summary> + public MemoryPolicyConfiguration() + { + EvictionThreshold = DefaultEvictionThreshold; + EmptyPagesPoolSize = DefaultEmptyPagesPoolSize; + } + + /// <summary> + /// Initializes a new instance of the <see cref="MemoryPolicyConfiguration"/> class. + /// </summary> + /// <param name="reader">The reader.</param> + internal MemoryPolicyConfiguration(IBinaryRawReader reader) + { + Name = reader.ReadString(); + Size = reader.ReadLong(); + SwapFilePath = reader.ReadString(); + PageEvictionMode = (DataPageEvictionMode) reader.ReadInt(); + EvictionThreshold = reader.ReadDouble(); + EmptyPagesPoolSize = reader.ReadInt(); + } + + /// <summary> + /// Writes this instance to a writer. + /// </summary> + internal void Write(IBinaryRawWriter writer) + { + writer.WriteString(Name); + writer.WriteLong(Size); + writer.WriteString(SwapFilePath); + writer.WriteInt((int) PageEvictionMode); + writer.WriteDouble(EvictionThreshold); + writer.WriteInt(EmptyPagesPoolSize); + } + + /// <summary> + /// Gets or sets the memory policy name. + /// </summary> + public string Name { get; set; } + + /// <summary> + /// Gets or sets the maximum memory region size defined by this memory policy. + /// If the whole data can not fit into the memory region an out of memory exception will be thrown. + /// </summary> + public long Size { get; set; } + + /// <summary> + /// Gets or sets the the path to the memory-mapped file the memory region defined by this memory policy + /// will be mapped to. Having the path set, allows relying on swapping capabilities of an underlying + /// operating system for the memory region. + /// <para /> + /// Null for no swap. + /// </summary> + public string SwapFilePath { get; set; } + + /// <summary> + /// Gets or sets the page eviction mode. If <see cref="DataPageEvictionMode.Disabled"/> is used (default) + /// then an out of memory exception will be thrown if the memory region usage, + /// defined by this memory policy, goes beyond <see cref="Size"/>. + /// </summary> + public DataPageEvictionMode PageEvictionMode { get; set; } + + /// <summary> + /// Gets or sets the threshold for memory pages eviction initiation. For instance, if the threshold is 0.9 + /// it means that the page memory will start the eviction only after 90% of the memory region + /// (defined by this policy) is occupied. + /// </summary> + [DefaultValue(DefaultEvictionThreshold)] + public double EvictionThreshold { get; set; } + + /// <summary> + /// Gets or sets the minimal number of empty pages to be present in reuse lists for this memory policy. + /// This parameter ensures that Ignite will be able to successfully evict old data entries when the size of + /// (key, value) pair is slightly larger than page size / 2. + /// Increase this parameter if cache can contain very big entries (total size of pages in this pool + /// should be enough to contain largest cache entry). + /// </summary> + [DefaultValue(DefaultEmptyPagesPoolSize)] + public int EmptyPagesPoolSize { get;set; } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs index 85c7548..1f7a184 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfiguration.cs @@ -350,6 +350,16 @@ namespace Apache.Ignite.Core memEventStorage.Write(writer); } + if (MemoryConfiguration != null) + { + writer.WriteBoolean(true); + MemoryConfiguration.Write(writer); + } + else + { + writer.WriteBoolean(false); + } + // Plugins (should be last) if (PluginConfigurations != null) { @@ -482,6 +492,11 @@ namespace Apache.Ignite.Core EventStorageSpi = MemoryEventStorageSpi.Read(r); break; } + + if (r.ReadBoolean()) + { + MemoryConfiguration = new MemoryConfiguration(r); + } } /// <summary> @@ -937,5 +952,11 @@ namespace Apache.Ignite.Core /// <see cref="NoopEventStorageSpi"/>, <see cref="MemoryEventStorageSpi"/>. /// </summary> public IEventStorageSpi EventStorageSpi { get; set; } + + /// <summary> + /// Gets or sets the page memory configuration. + /// <see cref="MemoryConfiguration"/> for more details. + /// </summary> + public MemoryConfiguration MemoryConfiguration { get; set; } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/800fb87a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd ---------------------------------------------------------------------- diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd index 1d2a948..ba2f756 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteConfigurationSection.xsd @@ -78,6 +78,14 @@ </xs:restriction> </xs:simpleType> + <xs:simpleType name="dataPageEvictionMode" final="restriction"> + <xs:restriction base="xs:string"> + <xs:enumeration value="Disabled" /> + <xs:enumeration value="RandomLru" /> + <xs:enumeration value="Random2Lru" /> + </xs:restriction> + </xs:simpleType> + <xs:element name="igniteConfiguration"> <xs:annotation> <xs:documentation>Ignite configuration root.</xs:documentation> @@ -697,6 +705,11 @@ <xs:documentation>Value indicating whether statistics gathering is enabled on a cache. These statistics can be retrieved via ICache.GetMetrics().</xs:documentation> </xs:annotation> </xs:attribute> + <xs:attribute name="memoryPolicyName" type="xs:string"> + <xs:annotation> + <xs:documentation>Name of the MemoryPolicyConfiguration for this cache.</xs:documentation> + </xs:annotation> + </xs:attribute> </xs:complexType> </xs:element> </xs:sequence> @@ -1120,6 +1133,78 @@ </xs:attribute> </xs:complexType> </xs:element> + <xs:element name="memoryConfiguration" minOccurs="0"> + <xs:annotation> + <xs:documentation>Page memory configuration.</xs:documentation> + </xs:annotation> + <xs:complexType> + <xs:sequence> + <xs:element name="memoryPolicies" minOccurs="0" maxOccurs="1"> + <xs:annotation> + <xs:documentation>Memory policies.</xs:documentation> + </xs:annotation> + <xs:complexType> + <xs:sequence> + <xs:element name="memoryPolicyConfiguration" minOccurs="0" maxOccurs="unbounded"> + <xs:complexType> + <xs:attribute name="emptyPagesPoolSize" type="xs:int"> + <xs:annotation> + <xs:documentation>Minimal number of empty pages to be present in reuse lists for this memory policy.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="evictionThreshold" type="xs:double"> + <xs:annotation> + <xs:documentation>Threshold for memory pages eviction initiation. For instance, if the threshold is 0.9 it means that the page memory will start the eviction only after 90% of the memory region (defined by this policy) is occupied.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="name" type="xs:string" use="required"> + <xs:annotation> + <xs:documentation>Memory policy name.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="pageEvictionMode" type="dataPageEvictionMode"> + <xs:annotation> + <xs:documentation>Page eviction mode.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="size" type="xs:int" use="required"> + <xs:annotation> + <xs:documentation>Maximum memory region size defined by this memory policy.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="swapFilePath" type="xs:string"> + <xs:annotation> + <xs:documentation>Path to the memory-mapped file the memory region defined by this memory policy will be mapped to.</xs:documentation> + </xs:annotation> + </xs:attribute> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + <xs:attribute name="concurrencyLevel" type="xs:int"> + <xs:annotation> + <xs:documentation>Number of concurrent segments in Ignite internal page mapping tables.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="defaultMemoryPolicyName" type="xs:string"> + <xs:annotation> + <xs:documentation>Name of the default memory policy in MemoryPolicies.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="pageSize" type="xs:int"> + <xs:annotation> + <xs:documentation>Size of the memory page.</xs:documentation> + </xs:annotation> + </xs:attribute> + <xs:attribute name="systemCacheMemorySize" type="xs:int"> + <xs:annotation> + <xs:documentation>Size of a memory chunk reserved for system cache needs.</xs:documentation> + </xs:annotation> + </xs:attribute> + </xs:complexType> + </xs:element> <xs:element name="pluginConfigurations" minOccurs="0"> <xs:annotation> <xs:documentation>Plugin configurations.</xs:documentation>
