IGNITE-3521: IGFS: Removed "maxSpace" property. This closes #1854.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7c249d7b Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7c249d7b Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7c249d7b Branch: refs/heads/ignite-5009 Commit: 7c249d7bbb3c6f65e2781e0da3cd21c62d2ce078 Parents: 5a43346 Author: tledkov-gridgain <[email protected]> Authored: Mon Apr 24 15:17:49 2017 +0300 Committer: devozerov <[email protected]> Committed: Mon Apr 24 15:17:49 2017 +0300 ---------------------------------------------------------------------- .../configuration/FileSystemConfiguration.java | 25 ---- .../org/apache/ignite/igfs/IgfsMetrics.java | 5 +- .../processors/cache/GridCacheAdapter.java | 14 --- .../processors/cache/GridCacheProcessor.java | 5 +- .../processors/cache/GridCacheProxyImpl.java | 12 -- .../processors/cache/IgniteInternalCache.java | 7 -- .../dht/GridDhtTransactionalCacheAdapter.java | 1 - .../dht/preloader/GridDhtPartitionDemander.java | 12 -- .../distributed/near/GridNearCacheAdapter.java | 5 - .../processors/igfs/IgfsDataManager.java | 36 +----- .../internal/visor/igfs/VisorIgfsMetrics.java | 3 +- .../visor/node/VisorIgfsConfiguration.java | 13 -- .../processors/igfs/IgfsMaxSizeSelfTest.java | 121 ------------------- .../processors/igfs/IgfsSizeSelfTest.java | 50 +++++--- .../ignite/testsuites/IgniteIgfsTestSuite.java | 3 - .../HadoopIgfs20FileSystemAbstractSelfTest.java | 1 - .../cache/hibernate/HibernateCacheProxy.java | 5 - .../cache/hibernate/HibernateCacheProxy.java | 5 - 18 files changed, 47 insertions(+), 276 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java index af5be3c..e36f857 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/FileSystemConfiguration.java @@ -136,9 +136,6 @@ public class FileSystemConfiguration { /** Path modes. */ private Map<String, IgfsMode> pathModes; - /** Maximum space. */ - private long maxSpace; - /** Maximum range length. */ private long maxTaskRangeLen; @@ -187,7 +184,6 @@ public class FileSystemConfiguration { secondaryFs = cfg.getSecondaryFileSystem(); ipcEndpointCfg = cfg.getIpcEndpointConfiguration(); ipcEndpointEnabled = cfg.isIpcEndpointEnabled(); - maxSpace = cfg.getMaxSpaceSize(); maxTaskRangeLen = cfg.getMaximumTaskRangeLength(); metaCacheCfg = cfg.getMetaCacheConfiguration(); mgmtPort = cfg.getManagementPort(); @@ -682,27 +678,6 @@ public class FileSystemConfiguration { } /** - * Get maximum space available for data cache to store file system entries. - * - * @return Maximum space available for data cache. - */ - public long getMaxSpaceSize() { - return maxSpace; - } - - /** - * Set maximum space in bytes available in data cache. - * - * @param maxSpace Maximum space available in data cache. - * @return {@code this} for chaining. - */ - public FileSystemConfiguration setMaxSpaceSize(long maxSpace) { - this.maxSpace = maxSpace; - - return this; - } - - /** * Get maximum default range size of a file being split during IGFS task execution. When IGFS task is about to * be executed, it requests file block locations first. Each location is defined as {@link org.apache.ignite.igfs.mapreduce.IgfsFileRange} which * has length. In case this parameter is set to positive value, then IGFS will split single file range into smaller http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java index 3283eaf..28225fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsMetrics.java @@ -32,9 +32,8 @@ public interface IgfsMetrics { public long localSpaceSize(); /** - * Gets maximum amount of data that can be stored on local node. This metrics is either - * equal to {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaxSpaceSize()}, or, if it is {@code 0}, equal to - * {@code 80%} of maximum heap size allocated for JVM. + * Gets maximum amount of data that can be stored on local node. This metrics is related to + * to the {@link org.apache.ignite.configuration.MemoryPolicyConfiguration#getSize()} of the IGFS data cache. * * @return Maximum IGFS local space size. */ http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index c9f7430..b364df8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -278,9 +278,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V /** Current IGFS data cache size. */ private LongAdder8 igfsDataCacheSize; - /** Max space for IGFS. */ - private long igfsDataSpaceMax; - /** Asynchronous operations limit semaphore. */ private Semaphore asyncOpsSem; @@ -338,12 +335,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V if (!ctx.isNear()) { igfsDataCache = true; igfsDataCacheSize = new LongAdder8(); - - igfsDataSpaceMax = igfsCfg.getMaxSpaceSize(); - - // Do we have limits? - if (igfsDataSpaceMax <= 0) - igfsDataSpaceMax = Long.MAX_VALUE; } break; @@ -4363,11 +4354,6 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V } /** {@inheritDoc} */ - @Override public long igfsDataSpaceMax() { - return igfsDataSpaceMax; - } - - /** {@inheritDoc} */ @Override public boolean isMongoDataCache() { return mongoDataCache; } http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index a555b55..4b79361 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -738,10 +738,7 @@ public class GridCacheProcessor extends GridProcessorAdapter { else cacheType = CacheType.USER; - if (cacheType != CacheType.USER) - cfg.setMemoryPolicyName(sharedCtx.database().systemMemoryPolicyName()); - - if (cacheType != CacheType.USER) + if (cacheType != CacheType.USER && cfg.getMemoryPolicyName() == null) cfg.setMemoryPolicyName(sharedCtx.database().systemMemoryPolicyName()); boolean template = cfg.getName() != null && cfg.getName().endsWith("*"); http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java index 2979a57..837c22a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java @@ -454,18 +454,6 @@ public class GridCacheProxyImpl<K, V> implements IgniteInternalCache<K, V>, Exte } /** {@inheritDoc} */ - @Override public long igfsDataSpaceMax() { - CacheOperationContext prev = gate.enter(opCtx); - - try { - return delegate.igfsDataSpaceMax(); - } - finally { - gate.leave(prev); - } - } - - /** {@inheritDoc} */ @Override public boolean isMongoDataCache() { CacheOperationContext prev = gate.enter(opCtx); http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java index ce65fd2..cc7fd3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java @@ -1719,13 +1719,6 @@ public interface IgniteInternalCache<K, V> extends Iterable<Cache.Entry<K, V>> { public long igfsDataSpaceUsed(); /** - * Get maximum space available for IGFS. - * - * @return Amount of space available for IGFS in bytes. - */ - public long igfsDataSpaceMax(); - - /** * Checks whether this cache is Mongo data cache. * * @return {@code True} if this cache is mongo data cache. http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index 6af1ef7..c91eb7a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -55,7 +55,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSing import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxRemote; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearUnlockRequest; -import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey; import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx; http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java index d71a3ad..75cbd00 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java @@ -56,7 +56,6 @@ import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.CI1; import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.internal.util.typedef.internal.LT; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteInClosure; @@ -744,17 +743,6 @@ public class GridDhtPartitionDemander { if (log.isDebugEnabled()) log.debug("Rebalancing key [key=" + entry.key() + ", part=" + p + ", node=" + pick.id() + ']'); - if (cctx.dht().isIgfsDataCache() && - cctx.dht().igfsDataSpaceUsed() > cctx.dht().igfsDataSpaceMax()) { - LT.error(log, null, "Failed to rebalance IGFS data cache (IGFS space size exceeded maximum " + - "value, will ignore rebalance entries)"); - - if (cached.markObsoleteIfEmpty(null)) - cached.context().cache().removeEntry(cached); - - return true; - } - if (preloadPred == null || preloadPred.apply(entry)) { if (cached.initialValue( entry.value(), http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java index f4ba043..59d986a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheAdapter.java @@ -403,11 +403,6 @@ public abstract class GridNearCacheAdapter<K, V> extends GridDistributedCacheAda } /** {@inheritDoc} */ - @Override public long igfsDataSpaceMax() { - return dht().igfsDataSpaceMax(); - } - - /** {@inheritDoc} */ @Override public void onIgfsDataSizeChanged(long delta) { dht().onIgfsDataSizeChanged(delta); } http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java index acd0a1f..7fa2355 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.managers.communication.GridMessageListener; import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; +import org.apache.ignite.internal.processors.cache.database.MemoryPolicy; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal; import org.apache.ignite.internal.processors.datastreamer.DataStreamerCacheUpdaters; import org.apache.ignite.internal.processors.igfs.data.IgfsDataPutProcessor; @@ -242,7 +243,11 @@ public class IgfsDataManager extends IgfsManager { * @return Maximum number of bytes for IGFS data cache. */ public long maxSpaceSize() { - return (igfsCtx.configuration().getMaxSpaceSize() <= 0) ? 0 : dataCachePrj.igfsDataSpaceMax(); + MemoryPolicy plc = dataCachePrj.context().memoryPolicy(); + + long size = plc != null ? plc.config().getSize() : 0; + + return (size <= 0) ? 0 : size ; } /** @@ -1022,27 +1027,6 @@ public class IgfsDataManager extends IgfsManager { */ private void processPartialBlockWrite(IgniteUuid fileId, IgfsBlockKey colocatedKey, int startOff, byte[] data, int blockSize) throws IgniteCheckedException { - if (dataCachePrj.igfsDataSpaceUsed() >= dataCachePrj.igfsDataSpaceMax()) { - final WriteCompletionFuture completionFut = pendingWrites.get(fileId); - - if (completionFut == null) { - if (log.isDebugEnabled()) - log.debug("Missing completion future for file write request (most likely exception occurred " + - "which will be thrown upon stream close) [fileId=" + fileId + ']'); - - return; - } - - IgfsOutOfSpaceException e = new IgfsOutOfSpaceException("Failed to write data block " + - "(IGFS maximum data size exceeded) [used=" + dataCachePrj.igfsDataSpaceUsed() + - ", allowed=" + dataCachePrj.igfsDataSpaceMax() + ']'); - - completionFut.onDone(new IgniteCheckedException("Failed to write data (not enough space on node): " + - igfsCtx.kernalContext().localNodeId(), e)); - - return; - } - // No affinity key present, just concat and return. if (colocatedKey.affinityKey() == null) { dataCachePrj.invoke(colocatedKey, new UpdateProcessor(startOff, data)); @@ -1097,14 +1081,6 @@ public class IgfsDataManager extends IgfsManager { @SuppressWarnings("unchecked") private IgniteInternalFuture<?> storeBlocksAsync(Map<IgfsBlockKey, byte[]> blocks) { assert !blocks.isEmpty(); - - if (dataCachePrj.igfsDataSpaceUsed() >= dataCachePrj.igfsDataSpaceMax()) { - return new GridFinishedFuture<Object>( - new IgfsOutOfSpaceException("Failed to write data block (IGFS maximum data size " + - "exceeded) [used=" + dataCachePrj.igfsDataSpaceUsed() + - ", allowed=" + dataCachePrj.igfsDataSpaceMax() + ']')); - } - return dataCachePrj.putAllAsync(blocks); } http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsMetrics.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsMetrics.java index 6ce5d80..f90ca89 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsMetrics.java @@ -22,6 +22,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import org.apache.ignite.IgniteFileSystem; import org.apache.ignite.igfs.IgfsMetrics; +import org.apache.ignite.internal.processors.igfs.IgfsEx; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.visor.VisorDataTransferObject; @@ -91,7 +92,7 @@ public class VisorIgfsMetrics extends VisorDataTransferObject { IgfsMetrics m = igfs.metrics(); - totalSpaceSz = igfs.configuration().getMaxSpaceSize(); + totalSpaceSz = ((IgfsEx)igfs).context().data().maxSpaceSize(); usedSpaceSz = m.localSpaceSize(); foldersCnt = m.directoriesCount(); filesCnt = m.filesCount(); http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java index 3b97129..db91982 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java @@ -92,9 +92,6 @@ public class VisorIgfsConfiguration extends VisorDataTransferObject { /** IPC endpoint enabled flag. */ private boolean ipcEndpointEnabled; - /** Maximum space. */ - private long maxSpace; - /** Management port. */ private int mgmtPort; @@ -135,7 +132,6 @@ public class VisorIgfsConfiguration extends VisorDataTransferObject { ipcEndpointCfg = endpointCfg != null ? endpointCfg.toString() : null; ipcEndpointEnabled = igfs.isIpcEndpointEnabled(); - maxSpace = igfs.getMaxSpaceSize(); mgmtPort = igfs.getManagementPort(); seqReadsBeforePrefetch = igfs.getSequentialReadsBeforePrefetch(); } @@ -277,13 +273,6 @@ public class VisorIgfsConfiguration extends VisorDataTransferObject { } /** - * @return Maximum space. - */ - public long getMaxSpace() { - return maxSpace; - } - - /** * @return Management port. */ public int getManagementPort() { @@ -316,7 +305,6 @@ public class VisorIgfsConfiguration extends VisorDataTransferObject { out.writeLong(fragmentizerThrottlingDelay); U.writeString(out, ipcEndpointCfg); out.writeBoolean(ipcEndpointEnabled); - out.writeLong(maxSpace); out.writeInt(mgmtPort); out.writeInt(seqReadsBeforePrefetch); } @@ -340,7 +328,6 @@ public class VisorIgfsConfiguration extends VisorDataTransferObject { fragmentizerThrottlingDelay = in.readLong(); ipcEndpointCfg = U.readString(in); ipcEndpointEnabled = in.readBoolean(); - maxSpace = in.readLong(); mgmtPort = in.readInt(); seqReadsBeforePrefetch = in.readInt(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMaxSizeSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMaxSizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMaxSizeSelfTest.java deleted file mode 100644 index 2e0be3f..0000000 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMaxSizeSelfTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.igfs; - -import org.apache.ignite.cache.CacheAtomicityMode; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.CacheWriteSynchronizationMode; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.FileSystemConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper; -import org.apache.ignite.internal.IgniteEx; - -/** - * Check max size limit. - */ -@SuppressWarnings("ConstantConditions") -public class IgfsMaxSizeSelfTest extends IgfsCommonAbstractTest { - /** Work directory. */ - private static long maxSize; - - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); - - igfsCfg.setName("test"); - - if (maxSize > 0) - igfsCfg.setMaxSpaceSize(maxSize); - - CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); - - dataCacheCfg.setName("dataCache"); - dataCacheCfg.setCacheMode(CacheMode.PARTITIONED); - dataCacheCfg.setNearConfiguration(null); - dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(2)); - dataCacheCfg.setBackups(0); - dataCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - - CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); - - metaCacheCfg.setName("metaCache"); - metaCacheCfg.setNearConfiguration(null); - metaCacheCfg.setCacheMode(CacheMode.REPLICATED); - metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); - metaCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL); - - igfsCfg.setDataCacheConfiguration(dataCacheCfg); - igfsCfg.setMetaCacheConfiguration(metaCacheCfg); - - cfg.setFileSystemConfiguration(igfsCfg); - cfg.setIgniteInstanceName(igniteInstanceName); - - return cfg; - } - - /** - * @throws Exception If failed. - */ - public void testDefaultOrZero() throws Exception { - IgniteEx ig = startGrid(0); - - try { - assertEquals(0, ((IgfsImpl)ig.igfsx("test")).globalSpace().spaceTotal()); - } - finally { - stopAllGrids(); - } - } - - /** - * @throws Exception If failed. - */ - public void testNegative() throws Exception { - maxSize = -1; - - IgniteEx ig = startGrid(0); - - try { - assertEquals(0, ((IgfsImpl)ig.igfsx("test")).globalSpace().spaceTotal()); - } - finally { - stopAllGrids(); - } - } - - /** - * @throws Exception If failed. - */ - public void testPositive() throws Exception { - maxSize = 1 << 20; - - IgniteEx ig = startGrid(0); - - try { - assertEquals(maxSize, ((IgfsImpl)ig.igfsx("test")).globalSpace().spaceTotal()); - } - finally { - stopAllGrids(); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java index 645f341..fbe0872 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java @@ -17,6 +17,8 @@ package org.apache.ignite.internal.processors.igfs; +import java.io.IOException; +import java.util.concurrent.Callable; import org.apache.ignite.Ignite; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CacheWriteSynchronizationMode; @@ -24,17 +26,20 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.MemoryConfiguration; +import org.apache.ignite.configuration.MemoryPolicyConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper; import org.apache.ignite.igfs.IgfsInputStream; -import org.apache.ignite.igfs.IgfsOutOfSpaceException; import org.apache.ignite.igfs.IgfsOutputStream; import org.apache.ignite.igfs.IgfsPath; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.mem.IgniteOutOfMemoryException; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder; @@ -42,14 +47,12 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; import org.apache.ignite.testframework.GridTestUtils; import org.jsr166.ThreadLocalRandom8; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.UUID; -import java.util.concurrent.Callable; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; import static org.apache.ignite.cache.CacheMode.PARTITIONED; @@ -87,14 +90,13 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { /** Whether near cache is enabled (applicable for PARTITIONED cache only). */ private boolean nearEnabled; - /** IGFS maximum space. */ - private long igfsMaxData; + /** Mem policy setter. */ + private IgniteInClosure<IgniteConfiguration> memIgfsdDataPlcSetter; /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { cacheMode = null; nearEnabled = false; - igfsMaxData = 0; mgmtPort = 11400; } @@ -113,7 +115,6 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { igfsCfg.setName(IGFS_NAME); igfsCfg.setBlockSize(BLOCK_SIZE); igfsCfg.setFragmentizerEnabled(false); - igfsCfg.setMaxSpaceSize(igfsMaxData); igfsCfg.setManagementPort(++mgmtPort); CacheConfiguration dataCfg = defaultCacheConfiguration(); @@ -150,6 +151,9 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setFileSystemConfiguration(igfsCfg); + if (memIgfsdDataPlcSetter != null) + memIgfsdDataPlcSetter.apply(cfg); + return cfg; } @@ -221,7 +225,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { cacheMode = PARTITIONED; nearEnabled = false; - check(); + checkOversize(); } /** @@ -232,7 +236,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { public void testReplicatedOversize() throws Exception { cacheMode = REPLICATED; - check(); + checkOversize(); } /** @@ -386,15 +390,32 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ private void checkOversize() throws Exception { - igfsMaxData = BLOCK_SIZE; + final long maxSize = 32 * 1024 * 1024; + + memIgfsdDataPlcSetter = new IgniteInClosure<IgniteConfiguration>() { + @Override public void apply(IgniteConfiguration cfg) { + String memPlcName = "igfsDataMemPlc"; + + cfg.setMemoryConfiguration(new MemoryConfiguration().setMemoryPolicies( + new MemoryPolicyConfiguration().setSize(maxSize).setName(memPlcName))); + + FileSystemConfiguration igfsCfg = cfg.getFileSystemConfiguration()[0]; + + igfsCfg.getDataCacheConfiguration().setMemoryPolicyName(memPlcName); + + cfg.setCacheConfiguration(new CacheConfiguration().setName("QQQ").setMemoryPolicyName(memPlcName)); + } + }; startUp(); final IgfsPath path = new IgfsPath("/file"); + final int writeChunkSize = (int)(maxSize / 1024); + // This write is expected to be successful. IgfsOutputStream os = igfs(0).create(path, false); - os.write(chunk(BLOCK_SIZE - 1)); + os.write(chunk(writeChunkSize)); os.close(); // This write must be successful as well. @@ -408,7 +429,9 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { IgfsOutputStream osErr = igfs(0).append(path, false); try { - osErr.write(chunk(BLOCK_SIZE)); + for (int i = 0; i < maxSize / writeChunkSize * GRID_CNT; ++i) + osErr.write(chunk(writeChunkSize)); + osErr.close(); return null; @@ -425,8 +448,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { U.closeQuiet(osErr); } } - }, IgfsOutOfSpaceException.class, "Failed to write data block (IGFS maximum data size exceeded) [used=" + - igfsMaxData + ", allowed=" + igfsMaxData + ']'); + }, IgniteOutOfMemoryException.class, "Not enough memory allocated"); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java index 7efc7de..b16f7e9 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java @@ -32,7 +32,6 @@ import org.apache.ignite.internal.processors.igfs.IgfsCacheSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsDualAsyncClientSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsDualSyncClientSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsLocalSecondaryFileSystemProxyClientSelfTest; -import org.apache.ignite.internal.processors.igfs.IgfsMaxSizeSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsPrimaryClientSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsDataManagerSelfTest; import org.apache.ignite.internal.processors.igfs.IgfsDualAsyncSelfTest; @@ -150,8 +149,6 @@ public class IgniteIgfsTestSuite extends TestSuite { // TODO: Enable when IGFS failover is fixed. //suite.addTestSuite(IgfsBackupFailoverSelfTest.class); - suite.addTestSuite(IgfsMaxSizeSelfTest.class); - suite.addTestSuite(IgfsProxySelfTest.class); suite.addTestSuite(IgfsLocalSecondaryFileSystemProxySelfTest.class); suite.addTestSuite(IgfsLocalSecondaryFileSystemProxyClientSelfTest.class); http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java index 77e6757..016fadf 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java @@ -305,7 +305,6 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA cfg.setName("igfs"); cfg.setPrefetchBlocks(1); - cfg.setMaxSpaceSize(64 * 1024 * 1024); cfg.setDefaultMode(mode); cfg.setMetaCacheConfiguration(metaCacheConfiguration(igniteInstanceName)); cfg.setDataCacheConfiguration(dataCacheConfiguration(igniteInstanceName)); http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java ---------------------------------------------------------------------- diff --git a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java index 48fc1f8..7204083 100644 --- a/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java +++ b/modules/hibernate/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java @@ -627,11 +627,6 @@ public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> } /** {@inheritDoc} */ - @Override public long igfsDataSpaceMax() { - return delegate.igfsDataSpaceMax(); - } - - /** {@inheritDoc} */ @Override public boolean isMongoDataCache() { return delegate.isMongoDataCache(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/7c249d7b/modules/hibernate5/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java ---------------------------------------------------------------------- diff --git a/modules/hibernate5/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate5/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java index 48fc1f8..7204083 100644 --- a/modules/hibernate5/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java +++ b/modules/hibernate5/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java @@ -627,11 +627,6 @@ public class HibernateCacheProxy implements IgniteInternalCache<Object, Object> } /** {@inheritDoc} */ - @Override public long igfsDataSpaceMax() { - return delegate.igfsDataSpaceMax(); - } - - /** {@inheritDoc} */ @Override public boolean isMongoDataCache() { return delegate.isMongoDataCache(); }
