This is an automated email from the ASF dual-hosted git repository. ipavlukhin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new de16d6b IGNITE-10913 Reduce heap occupation by o.a.i.i.processors.cache.persistence.file.FilePageStore instances - Fixes #6500. de16d6b is described below commit de16d6b47c9d3ccd6b172a27540a6a7d2d0a9b5b Author: denis-chudov <dchu...@gridgain.com> AuthorDate: Thu Jun 13 19:32:24 2019 +0300 IGNITE-10913 Reduce heap occupation by o.a.i.i.processors.cache.persistence.file.FilePageStore instances - Fixes #6500. Signed-off-by: ipavlukhin <vololo...@gmail.com> --- .../jol/FileStoreHeapUtilizationJolBenchmark.java | 194 +++++++++++++++++++++ .../cache/persistence/file/FilePageStore.java | 100 ++++++++--- .../persistence/file/FilePageStoreFactory.java | 24 ++- .../persistence/file/FilePageStoreManager.java | 15 +- .../cache/persistence/file/FilePageStoreV2.java | 13 +- .../file/FileVersionCheckingFactory.java | 35 ++-- .../persistence/IgnitePdsTaskCancelingTest.java | 2 +- 7 files changed, 327 insertions(+), 56 deletions(-) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java new file mode 100644 index 0000000..82dd5ca --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.benchmarks.jol; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteException; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.openjdk.jol.info.GraphLayout; + +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; + +/** + * + */ +public class FileStoreHeapUtilizationJolBenchmark { + /** */ + private static final String CACHE_NAME = "testCache"; + + /** */ + private static final String HEAP_USAGE = "heap usage"; + + /** */ + private static final String CACHE_WORK_TIME = "cache work time"; + + /** */ + private static final TestResultParameterInfo HEAP_USAGE_PARAM = new TestResultParameterInfo(HEAP_USAGE, false); + + /** */ + private static final TestResultParameterInfo CACHE_WORK_TIME_PARAM = new TestResultParameterInfo(CACHE_WORK_TIME, false); + + /** + * Cleans persistent directory. + * + * @throws Exception if failed. + */ + private void cleanPersistenceDir() throws Exception { + if (!F.isEmpty(G.allGrids())) + throw new IgniteException("Grids are not stopped"); + + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false)); + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false)); + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false)); + U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "binary_meta", false)); + } + + /** */ + private IgniteConfiguration getConfiguration(String igniteInstanceName) { + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setIgniteInstanceName(igniteInstanceName); + + cfg.setDiscoverySpi( + new TcpDiscoverySpi() + .setIpFinder( + new TcpDiscoveryVmIpFinder() + .setAddresses(Collections.singleton("127.0.0.1:47500..47502")) + ) + ); + + CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME); + + ccfg.setAffinity(new RendezvousAffinityFunction(false, CacheConfiguration.MAX_PARTITIONS_COUNT)); + + cfg.setCacheConfiguration(ccfg); + + cfg.setActiveOnStart(false); + + DataStorageConfiguration memCfg = new DataStorageConfiguration() + .setDefaultDataRegionConfiguration( + new DataRegionConfiguration() + .setPersistenceEnabled(true) + .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE) + ) + .setWalMode(WALMode.LOG_ONLY); + + cfg.setDataStorageConfiguration(memCfg); + + return cfg; + } + + /** */ + private Map<TestResultParameterInfo, Comparable> testGrid() { + String name = UUID.randomUUID().toString().substring(0, 8); + + Ignite ignite = Ignition.start(getConfiguration(name)); + + ignite.cluster().active(true); + + long start = System.currentTimeMillis(); + + IgniteCache<Object, Object> cache = ignite.getOrCreateCache(CACHE_NAME); + + for (int i = 0; i < 100; i++) + cache.put(i, new byte[512]); + + for (int i = 50; i < 100; i++) + cache.remove(i); + + for (int i = 50; i < 150; i++) + cache.put(i, new byte[512]); + + long time = System.currentTimeMillis() - start; + + GraphLayout layout = GraphLayout.parseInstance(ignite); + + ignite.cluster().active(false); + + Ignition.stop(name, true); + + return new HashMap<TestResultParameterInfo, Comparable>() {{ + put(HEAP_USAGE_PARAM, layout.totalSize()); + put(CACHE_WORK_TIME_PARAM, time); + }}; + } + + /** */ + private void beforeTest() throws Exception { + cleanPersistenceDir(); + } + + /** */ + private void afterTest() throws Exception { + cleanPersistenceDir(); + } + + /** + * Benchmark body + * + * @throws Exception if failed. + */ + private void benchmark() throws Exception { + beforeTest(); + + Map<TestResultParameterInfo, Comparable> results = testGrid(); + + afterTest(); + + System.out.println("Benchmark results: "); + + results.forEach((k, v) -> System.out.println(k.name + ": " + v)); + } + + /** */ + public static void main(String[] args) throws Exception { + new FileStoreHeapUtilizationJolBenchmark().benchmark(); + } + + /** + * This class contains info about single parameter, which is measured by benchmark (e.g. heap usage, etc.). + */ + private static class TestResultParameterInfo { + /** */ + final String name; + + /** */ + final boolean greaterIsBetter; + + /** */ + TestResultParameterInfo(String name, boolean better) { + this.name = name; + greaterIsBetter = better; + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index b7dacf1..adaf118 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -24,6 +24,7 @@ import java.nio.ByteOrder; import java.nio.channels.ClosedByInterruptException; import java.nio.channels.ClosedChannelException; import java.nio.file.Files; +import java.nio.file.Path; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -39,6 +40,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteOutClosure; import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.READ; @@ -59,7 +61,13 @@ public class FilePageStore implements PageStore { public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/; /** */ - private final File cfgFile; + private final IgniteOutClosure<Path> pathProvider; + + /** + * Caches the existence state of storage file. After it is initialized, it will be not set to null + * during FilePageStore lifecycle. + */ + private volatile Boolean fileExists; /** */ private final byte type; @@ -97,18 +105,16 @@ public class FilePageStore implements PageStore { /** */ private final ReadWriteLock lock = new ReentrantReadWriteLock(); - /** - * @param file File. - */ + /** */ public FilePageStore( byte type, - File file, + IgniteOutClosure<Path> pathProvider, FileIOFactory factory, DataStorageConfiguration cfg, AllocatedPageTracker allocatedTracker ) { this.type = type; - this.cfgFile = file; + this.pathProvider = pathProvider; this.dbCfg = cfg; this.ioFactory = factory; this.allocated = new AtomicLong(); @@ -150,7 +156,22 @@ public class FilePageStore implements PageStore { /** {@inheritDoc} */ @Override public boolean exists() { - return cfgFile.exists() && cfgFile.length() > headerSize(); + if (fileExists == null) { + lock.writeLock().lock(); + + try { + if (fileExists == null) { + File file = pathProvider.apply().toFile(); + + fileExists = file.exists() && file.length() > headerSize(); + } + } + finally { + lock.writeLock().unlock(); + } + } + + return fileExists; } /** @@ -207,7 +228,16 @@ public class FilePageStore implements PageStore { } catch (ClosedByInterruptException e) { // If thread was interrupted written header can be inconsistent. - Files.delete(cfgFile.toPath()); + lock.writeLock().lock(); + + try { + Files.delete(pathProvider.apply()); + + fileExists = false; + } + finally { + lock.writeLock().unlock(); + } throw e; } @@ -219,7 +249,7 @@ public class FilePageStore implements PageStore { * @return Next available position in the file to store a data. * @throws IOException If check has failed. */ - private long checkFile(FileIO fileIO) throws IOException { + private long checkFile(FileIO fileIO, File cfgFile) throws IOException { ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN); fileIO.readFully(hdr); @@ -281,11 +311,14 @@ public class FilePageStore implements PageStore { fileIO = null; - if (delete) - Files.delete(cfgFile.toPath()); + if (delete) { + Files.delete(pathProvider.apply()); + + fileExists = false; + } } catch (IOException e) { - throw new StorageException("Failed to stop serving partition file [file=" + cfgFile.getPath() + throw new StorageException("Failed to stop serving partition file [file=" + getFileAbsolutePath() + ", delete=" + delete + "]", e); } finally { @@ -301,6 +334,8 @@ public class FilePageStore implements PageStore { @Override public void truncate(int tag) throws StorageException { init(); + Path filePath = pathProvider.apply(); + lock.writeLock().lock(); try { @@ -312,10 +347,12 @@ public class FilePageStore implements PageStore { fileIO = null; - Files.delete(cfgFile.toPath()); + Files.delete(filePath); + + fileExists = false; } catch (IOException e) { - throw new StorageException("Failed to truncate partition file [file=" + cfgFile.getPath() + "]", e); + throw new StorageException("Failed to truncate partition file [file=" + filePath.toAbsolutePath() + "]", e); } finally { allocatedTracker.updateTotalAllocatedPages(-1L * allocated.getAndSet(0) / pageSize); @@ -361,7 +398,7 @@ public class FilePageStore implements PageStore { recover = false; } catch (IOException e) { - throw new StorageException("Failed to finish recover partition file [file=" + cfgFile.getAbsolutePath() + "]", e); + throw new StorageException("Failed to finish recover partition file [file=" + getFileAbsolutePath() + "]", e); } finally { lock.writeLock().unlock(); @@ -381,7 +418,7 @@ public class FilePageStore implements PageStore { if (compressedSize < 0 || compressedSize > pageSize) { throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " + - "[id=" + U.hexLong(pageId) + ", file=" + cfgFile.getAbsolutePath() + ", fileSize=" + fileIO.size() + + "[id=" + U.hexLong(pageId) + ", file=" + getFileAbsolutePath() + ", fileSize=" + fileIO.size() + ", page=" + U.toHexString(pageBuf) + "]"); } @@ -400,7 +437,8 @@ public class FilePageStore implements PageStore { assert pageBuf.position() == 0; assert pageBuf.order() == ByteOrder.nativeOrder(); assert off <= allocated.get() : "calculatedOffset=" + off + - ", allocated=" + allocated.get() + ", headerSize=" + headerSize() + ", cfgFile=" + cfgFile; + ", allocated=" + allocated.get() + ", headerSize=" + headerSize() + ", cfgFile=" + + pathProvider.apply().toAbsolutePath(); int n = readWithFailover(pageBuf, off); @@ -423,7 +461,7 @@ public class FilePageStore implements PageStore { if ((savedCrc32 ^ curCrc32) != 0) throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " + "[id=" + U.hexLong(pageId) + ", off=" + (off - pageSize) + - ", file=" + cfgFile.getAbsolutePath() + ", fileSize=" + fileIO.size() + + ", file=" + getFileAbsolutePath() + ", fileSize=" + fileIO.size() + ", savedCrc=" + U.hexInt(savedCrc32) + ", curCrc=" + U.hexInt(curCrc32) + ", page=" + U.toHexString(pageBuf) + "]"); @@ -435,7 +473,7 @@ public class FilePageStore implements PageStore { PageIO.setCrc(pageBuf, savedCrc32); } catch (IOException e) { - throw new StorageException("Failed to read page [file=" + cfgFile.getAbsolutePath() + ", pageId=" + pageId + "]", e); + throw new StorageException("Failed to read page [file=" + getFileAbsolutePath() + ", pageId=" + pageId + "]", e); } } @@ -449,7 +487,7 @@ public class FilePageStore implements PageStore { readWithFailover(buf, 0); } catch (IOException e) { - throw new StorageException("Failed to read header [file=" + cfgFile.getAbsolutePath() + "]", e); + throw new StorageException("Failed to read header [file=" + getFileAbsolutePath() + "]", e); } } @@ -473,9 +511,13 @@ public class FilePageStore implements PageStore { while (true) { try { + File cfgFile = pathProvider.apply().toFile(); + this.fileIO = fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE); - newSize = (cfgFile.length() == 0 ? initFile(fileIO) : checkFile(fileIO)) - headerSize(); + fileExists = true; + + newSize = (cfgFile.length() == 0 ? initFile(fileIO) : checkFile(fileIO, cfgFile)) - headerSize(); if (interrupted) Thread.currentThread().interrupt(); @@ -501,7 +543,7 @@ public class FilePageStore implements PageStore { } catch (IOException e) { err = new StorageException( - "Failed to initialize partition file: " + cfgFile.getAbsolutePath(), e); + "Failed to initialize partition file: " + getFileAbsolutePath(), e); throw err; } @@ -547,9 +589,13 @@ public class FilePageStore implements PageStore { try { fileIO = null; + File cfgFile = pathProvider.apply().toFile(); + fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE); - checkFile(fileIO); + fileExists = true; + + checkFile(fileIO, cfgFile); this.fileIO = fileIO; @@ -602,7 +648,7 @@ public class FilePageStore implements PageStore { assert (off >= 0 && off <= allocated.get()) || recover : "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + - ", pageId=" + U.hexLong(pageId) + ", file=" + cfgFile.getPath(); + ", pageId=" + U.hexLong(pageId) + ", file=" + getFileAbsolutePath(); assert pageBuf.position() == 0; assert pageBuf.order() == ByteOrder.nativeOrder() : "Page buffer order " + pageBuf.order() @@ -659,7 +705,7 @@ public class FilePageStore implements PageStore { } } - throw new StorageException("Failed to write page [file=" + cfgFile.getAbsolutePath() + throw new StorageException("Failed to write page [file=" + getFileAbsolutePath() + ", pageId=" + pageId + ", tag=" + tag + "]", e); } } @@ -698,7 +744,7 @@ public class FilePageStore implements PageStore { fileIO.force(); } catch (IOException e) { - throw new StorageException("Failed to fsync partition file [file=" + cfgFile.getAbsolutePath() + ']', e); + throw new StorageException("Failed to fsync partition file [file=" + getFileAbsolutePath() + ']', e); } finally { lock.writeLock().unlock(); @@ -721,7 +767,7 @@ public class FilePageStore implements PageStore { * @return File absolute path. */ public String getFileAbsolutePath() { - return cfgFile.getAbsolutePath(); + return pathProvider.apply().toAbsolutePath().toString(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java index 2fb1d50..ca8c1c3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java @@ -18,20 +18,40 @@ package org.apache.ignite.internal.processors.cache.persistence.file; import java.io.File; +import java.nio.file.Path; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.pagemem.PageIdAllocator; import org.apache.ignite.internal.pagemem.store.PageStore; import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker; +import org.apache.ignite.lang.IgniteOutClosure; /** * */ public interface FilePageStoreFactory { /** - * Creates instance of FilePageStore based on given file. + * Creates instance of PageStore based on given file. * * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}. * @param file File Page store file. + * @param allocatedTracker metrics updater. + * @return page store + * @throws IgniteCheckedException if failed. */ - PageStore createPageStore(byte type, File file, AllocatedPageTracker allocatedTracker) throws IgniteCheckedException; + default PageStore createPageStore(byte type, File file, AllocatedPageTracker allocatedTracker) + throws IgniteCheckedException { + return createPageStore(type, file::toPath, allocatedTracker); + } + + /** + * Creates instance of PageStore based on file path provider. + * + * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA} + * @param pathProvider File Page store path provider. + * @param allocatedTracker metrics updater + * @return page store + * @throws IgniteCheckedException if failed + */ + PageStore createPageStore(byte type, IgniteOutClosure<Path> pathProvider, AllocatedPageTracker allocatedTracker) + throws IgniteCheckedException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index 8337f2f..a267ebe 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -487,7 +487,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen } /** {@inheritDoc} */ - @Override public void onPartitionCreated(int grpId, int partId) throws IgniteCheckedException { + @Override public void onPartitionCreated(int grpId, int partId) { // No-op. } @@ -621,7 +621,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen * */ public Path getPath(boolean isSharedGroup, String cacheOrGroupName, int partId) { - return getPartitionFile(cacheWorkDir(isSharedGroup, cacheOrGroupName), partId).toPath(); + return getPartitionFilePath(cacheWorkDir(isSharedGroup, cacheOrGroupName), partId); } /** @@ -696,7 +696,8 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory( pageStoreFileIoFactory, pageStoreV1FileIoFactory, - igniteCfg.getDataStorageConfiguration()); + igniteCfg.getDataStorageConfiguration() + ); if (encrypted) { int headerSize = pageStoreFactory.headerSize(pageStoreFactory.latestVersion()); @@ -714,10 +715,12 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen PageStore[] partStores = new PageStore[partitions]; for (int partId = 0; partId < partStores.length; partId++) { + final int p = partId; + PageStore partStore = pageStoreFactory.createPageStore( PageMemory.FLAG_DATA, - getPartitionFile(cacheWorkDir, partId), + () -> getPartitionFilePath(cacheWorkDir, p), allocatedTracker); partStores[partId] = partStore; @@ -737,8 +740,8 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen * @param cacheWorkDir Cache work directory. * @param partId Partition id. */ - @NotNull private File getPartitionFile(File cacheWorkDir, int partId) { - return new File(cacheWorkDir, format(PART_FILE_TEMPLATE, partId)); + @NotNull private Path getPartitionFilePath(File cacheWorkDir, int partId) { + return new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)).toPath(); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java index de078eb..b2c2d87 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java @@ -16,9 +16,10 @@ */ package org.apache.ignite.internal.processors.cache.persistence.file; -import java.io.File; +import java.nio.file.Path; import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker; +import org.apache.ignite.lang.IgniteOutClosure; /** * @@ -31,19 +32,21 @@ public class FilePageStoreV2 extends FilePageStore { private final int hdrSize; /** + * Constructor which initializes file path provider closure, allowing to calculate file path in any time. + * * @param type Type. - * @param file File. + * @param pathProvider file path provider. * @param factory Factory. * @param cfg Config. - * @param allocatedTracker Metrics updater + * @param allocatedTracker Allocated tracker. */ public FilePageStoreV2( byte type, - File file, + IgniteOutClosure<Path> pathProvider, FileIOFactory factory, DataStorageConfiguration cfg, AllocatedPageTracker allocatedTracker) { - super(type, file, factory, cfg, allocatedTracker); + super(type, pathProvider, factory, cfg, allocatedTracker); hdrSize = cfg.getPageSize(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java index af478de..fd290b1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java @@ -17,13 +17,16 @@ package org.apache.ignite.internal.processors.cache.persistence.file; -import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.internal.pagemem.store.PageStore; import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker; +import org.apache.ignite.lang.IgniteOutClosure; /** * Checks version in files if it's present on the disk, creates store with latest version otherwise. @@ -63,18 +66,20 @@ public class FileVersionCheckingFactory implements FilePageStoreFactory { } /** {@inheritDoc} */ - @Override public FilePageStore createPageStore( + @Override public PageStore createPageStore( byte type, - File file, + IgniteOutClosure<Path> pathProvider, AllocatedPageTracker allocatedTracker) throws IgniteCheckedException { - if (!file.exists()) - return createPageStore(type, file, latestVersion(), allocatedTracker); + Path filePath = pathProvider.apply(); - try (FileIO fileIO = fileIOFactoryStoreV1.create(file)) { + if (!Files.exists(filePath)) + return createPageStore(type, pathProvider, latestVersion(), allocatedTracker); + + try (FileIO fileIO = fileIOFactoryStoreV1.create(filePath.toFile())) { int minHdr = FilePageStore.HEADER_SIZE; if (fileIO.size() < minHdr) - return createPageStore(type, file, latestVersion(), allocatedTracker); + return createPageStore(type, pathProvider, latestVersion(), allocatedTracker); ByteBuffer hdr = ByteBuffer.allocate(minHdr).order(ByteOrder.LITTLE_ENDIAN); @@ -86,10 +91,10 @@ public class FileVersionCheckingFactory implements FilePageStoreFactory { int ver = hdr.getInt(); - return createPageStore(type, file, ver, allocatedTracker); + return createPageStore(type, pathProvider, ver, allocatedTracker); } catch (IOException e) { - throw new IgniteCheckedException("Error while creating file page store [file=" + file + "]:", e); + throw new IgniteCheckedException("Error while creating file page store [file=" + filePath.toAbsolutePath() + "]:", e); } } @@ -112,24 +117,24 @@ public class FileVersionCheckingFactory implements FilePageStoreFactory { * Instantiates specific version of FilePageStore. * * @param type Type. - * @param file File. * @param ver Version. * @param allocatedTracker Metrics updater */ - public FilePageStore createPageStore( + private FilePageStore createPageStore( byte type, - File file, + IgniteOutClosure<Path> pathProvider, int ver, AllocatedPageTracker allocatedTracker) { + switch (ver) { case FilePageStore.VERSION: - return new FilePageStore(type, file, fileIOFactoryStoreV1, memCfg, allocatedTracker); + return new FilePageStore(type, pathProvider, fileIOFactoryStoreV1, memCfg, allocatedTracker); case FilePageStoreV2.VERSION: - return new FilePageStoreV2(type, file, fileIOFactory, memCfg, allocatedTracker); + return new FilePageStoreV2(type, pathProvider, fileIOFactory, memCfg, allocatedTracker); default: - throw new IllegalArgumentException("Unknown version of file page store: " + ver + " for file [" + file.getAbsolutePath() + "]"); + throw new IllegalArgumentException("Unknown version of file page store: " + ver + " for file [" + pathProvider.apply().toAbsolutePath() + "]"); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java index 3b20d9b..7e2b7af 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java @@ -198,7 +198,7 @@ public class IgnitePdsTaskCancelingTest extends GridCommonAbstractTest { DataStorageConfiguration dbCfg = getDataStorageConfiguration(); - FilePageStore pageStore = new FilePageStore(PageMemory.FLAG_DATA, file, factory, dbCfg, + FilePageStore pageStore = new FilePageStore(PageMemory.FLAG_DATA, () -> file.toPath(), factory, dbCfg, AllocatedPageTracker.NO_OP); int pageSize = dbCfg.getPageSize();