IGNITE-3258: IGFS: Secondary file system input stream is opened only when it is really needed.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/52549574 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/52549574 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/52549574 Branch: refs/heads/ignite-3038 Commit: 525495742158de7b077078af372280943b52d87d Parents: 672b495 Author: vozerov-gridgain <voze...@gridgain.com> Authored: Mon Jun 6 17:50:58 2016 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Mon Jun 6 17:50:58 2016 +0300 ---------------------------------------------------------------------- ...zySecondaryFileSystemPositionedReadable.java | 77 ++++++++++++++++++++ .../processors/igfs/IgfsMetaManager.java | 18 ++++- .../processors/igfs/IgfsAbstractSelfTest.java | 3 + .../processors/igfs/IgfsModesSelfTest.java | 1 + .../igfs/HadoopFIleSystemFactorySelfTest.java | 1 + 5 files changed, 98 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java new file mode 100644 index 0000000..0a57c34 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.igfs; + +import org.apache.ignite.igfs.IgfsPath; +import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem; +import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable; +import org.apache.ignite.internal.util.typedef.internal.S; + +import java.io.IOException; + +/** + * Lazy readable entity which is opened on demand. + */ +public class IgfsLazySecondaryFileSystemPositionedReadable implements IgfsSecondaryFileSystemPositionedReadable { + /** File system. */ + private final IgfsSecondaryFileSystem fs; + + /** Path. */ + private final IgfsPath path; + + /** Buffer size. */ + private final int bufSize; + + /** Target stream. */ + private IgfsSecondaryFileSystemPositionedReadable target; + + /** + * Constructor. + * + * @param fs File system. + * @param path Path. + * @param bufSize Buffer size. + */ + public IgfsLazySecondaryFileSystemPositionedReadable(IgfsSecondaryFileSystem fs, IgfsPath path, int bufSize) { + assert fs != null; + assert path != null; + + this.fs = fs; + this.path = path; + this.bufSize = bufSize; + } + + /** {@inheritDoc} */ + @Override public int read(long pos, byte[] buf, int off, int len) throws IOException { + if (target == null) + target = fs.open(path, bufSize); + + return target.read(pos, buf, off, len); + } + + /** {@inheritDoc} */ + @Override public void close() throws IOException { + if (target != null) + target.close(); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(IgfsLazySecondaryFileSystemPositionedReadable.class, this); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java index fa748f8..1dd4c53 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java @@ -2144,7 +2144,7 @@ public class IgfsMetaManager extends IgfsManager { throw fsException(new IgfsPathIsDirectoryException("Failed to open file (not a file): " + path)); - return new IgfsSecondaryInputStreamDescriptor(info, fs.open(path, bufSize)); + return new IgfsSecondaryInputStreamDescriptor(info, lazySecondaryReader(fs, path, bufSize)); } // If failed, try synchronize. @@ -2160,7 +2160,8 @@ public class IgfsMetaManager extends IgfsManager { throw fsException(new IgfsPathIsDirectoryException("Failed to open file " + "(not a file): " + path)); - return new IgfsSecondaryInputStreamDescriptor(infos.get(path), fs.open(path, bufSize)); + return new IgfsSecondaryInputStreamDescriptor(infos.get(path), + lazySecondaryReader(fs, path, bufSize)); } @Override public IgfsSecondaryInputStreamDescriptor onFailure(@Nullable Exception err) @@ -2184,6 +2185,19 @@ public class IgfsMetaManager extends IgfsManager { } /** + * Create lazy secondary file system reader. + * + * @param fs File system. + * @param path Path. + * @param bufSize Buffer size. + * @return Lazy reader. + */ + private static IgfsLazySecondaryFileSystemPositionedReadable lazySecondaryReader(IgfsSecondaryFileSystem fs, + IgfsPath path, int bufSize) { + return new IgfsLazySecondaryFileSystemPositionedReadable(fs, path, bufSize); + } + + /** * Synchronizes with secondary file system. * * @param fs File system. http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java index 3fb7b91..76a038d 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java @@ -1082,6 +1082,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { createFile(igfs.asSecondary(), FILE, true, chunk); checkFileContent(igfs, FILE, chunk); + + // Read again when the whole file is in memory. + checkFileContent(igfs, FILE, chunk); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java index df537bc..1e54f8c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java @@ -102,6 +102,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest { igfsCfg.setMetaCacheName("replicated"); igfsCfg.setName("igfs"); igfsCfg.setBlockSize(512 * 1024); + igfsCfg.setInitializeDefaultPathModes(true); if (setNullMode) igfsCfg.setDefaultMode(null); http://git-wip-us.apache.org/repos/asf/ignite/blob/52549574/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java index 1d02f0f..e4c64ff 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFIleSystemFactorySelfTest.java @@ -226,6 +226,7 @@ public class HadoopFIleSystemFactorySelfTest extends IgfsCommonAbstractTest { igfsCfg.setDefaultMode(dfltMode); igfsCfg.setIpcEndpointConfiguration(endpointCfg); igfsCfg.setSecondaryFileSystem(secondaryFs); + igfsCfg.setInitializeDefaultPathModes(true); CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
