Repository: ignite Updated Branches: refs/heads/master b5ad72969 -> 3919d8035
IGNITE-5961 - Align pages in LFS partition files to pageSize Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3919d803 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3919d803 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3919d803 Branch: refs/heads/master Commit: 3919d8035fb172e1546f7a3ecea0f29f8e1e2214 Parents: b5ad729 Author: Ivan Rakov <ivan.glu...@gmail.com> Authored: Thu Aug 10 13:31:23 2017 +0300 Committer: Alexey Goncharuk <alexey.goncha...@gmail.com> Committed: Thu Aug 10 13:31:23 2017 +0300 ---------------------------------------------------------------------- .../internal/pagemem/store/PageStore.java | 5 + .../cache/persistence/file/FilePageStore.java | 54 ++++++--- .../persistence/file/FilePageStoreFactory.java | 35 ++++++ .../persistence/file/FilePageStoreManager.java | 17 +-- .../cache/persistence/file/FilePageStoreV2.java | 53 +++++++++ .../file/FileVersionCheckingFactory.java | 116 +++++++++++++++++++ ...gnitePdsRecoveryAfterFileCorruptionTest.java | 2 +- 7 files changed, 251 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java index 4698a6b..f6e577c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java @@ -95,4 +95,9 @@ public interface PageStore { * @throws IgniteCheckedException If sync failed (IO error occurred). */ public void ensure() throws IgniteCheckedException; + + /** + * @return Page store version. + */ + public int version(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index a7ca13c..7fe1ffe 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -45,10 +45,10 @@ public class FilePageStore implements PageStore { private static final long SIGNATURE = 0xF19AC4FE60C530B8L; /** File version. */ - private static final int VERSION = 1; + public static final int VERSION = 1; /** Allocated field offset. */ - public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/; + static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/; /** */ private final File cfgFile; @@ -57,7 +57,7 @@ public class FilePageStore implements PageStore { private final byte type; /** Database configuration. */ - private final MemoryConfiguration dbCfg; + protected final MemoryConfiguration dbCfg; /** Factory to provide I/O interfaces for read/write operations with files */ private final FileIOFactory ioFactory; @@ -103,20 +103,36 @@ public class FilePageStore implements PageStore { /** {@inheritDoc} */ @Override public boolean exists() { - return cfgFile.exists() && cfgFile.length() > HEADER_SIZE; + return cfgFile.exists() && cfgFile.length() > headerSize(); } /** + * Size of page store header. + */ + public int headerSize() { + return HEADER_SIZE; + } + + /** + * Page store version. + */ + public int version() { + return VERSION; + } + + /** + * Creates header for current version file store. Doesn't init the store. + * * @param type Type. * @param pageSize Page size. * @return Byte buffer instance. */ - public static ByteBuffer header(byte type, int pageSize) { - ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + public ByteBuffer header(byte type, int pageSize) { + ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN); hdr.putLong(SIGNATURE); - hdr.putInt(VERSION); + hdr.putInt(version()); hdr.put(type); @@ -142,7 +158,7 @@ public class FilePageStore implements PageStore { } //there is 'super' page in every file - return HEADER_SIZE + dbCfg.getPageSize(); + return headerSize() + dbCfg.getPageSize(); } /** @@ -150,7 +166,7 @@ public class FilePageStore implements PageStore { */ private long checkFile() throws IgniteCheckedException { try { - ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN); while (hdr.remaining() > 0) fileIO.read(hdr); @@ -166,9 +182,9 @@ public class FilePageStore implements PageStore { int ver = hdr.getInt(); - if (VERSION != ver) + if (version() != ver) throw new IgniteCheckedException("Failed to verify store file (invalid file version)" + - " [expectedVersion=" + VERSION + + " [expectedVersion=" + version() + ", fileVersion=" + ver + "]"); byte type = hdr.get(); @@ -187,10 +203,10 @@ public class FilePageStore implements PageStore { long fileSize = cfgFile.length(); - if (fileSize == HEADER_SIZE) // Every file has a special meta page. - fileSize = pageSize + HEADER_SIZE; + if (fileSize == headerSize()) // Every file has a special meta page. + fileSize = pageSize + headerSize(); - if ((fileSize - HEADER_SIZE) % pageSize != 0) + if ((fileSize - headerSize()) % pageSize != 0) throw new IgniteCheckedException("Failed to verify store file (invalid file size)" + " [fileSize=" + U.hexLong(fileSize) + ", pageSize=" + U.hexLong(pageSize) + ']'); @@ -346,9 +362,9 @@ public class FilePageStore implements PageStore { init(); try { - assert buf.remaining() == HEADER_SIZE; + assert buf.remaining() == headerSize(); - int len = HEADER_SIZE; + int len = headerSize(); long off = 0; @@ -425,7 +441,7 @@ public class FilePageStore implements PageStore { long off = pageOffset(pageId); - assert (off >= 0 && off + pageSize <= allocated.get() + HEADER_SIZE) || recover : + assert (off >= 0 && off + pageSize <= allocated.get() + headerSize()) || recover : "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId); assert pageBuf.capacity() == pageSize; @@ -463,7 +479,7 @@ public class FilePageStore implements PageStore { /** {@inheritDoc} */ @Override public long pageOffset(long pageId) { - return (long) PageIdUtils.pageIndex(pageId) * pageSize + HEADER_SIZE; + return (long) PageIdUtils.pageIndex(pageId) * pageSize + headerSize(); } /** {@inheritDoc} */ @@ -494,7 +510,7 @@ public class FilePageStore implements PageStore { long off = allocPage(); - return off / pageSize; + return (off - headerSize()) / pageSize; } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java new file mode 100644 index 0000000..d97ab26 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java @@ -0,0 +1,35 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.File; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdAllocator; + +/** + * + */ +public interface FilePageStoreFactory { + /** + * Creates instance of FilePageStore based on given file. + * + * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}. + * @param file File Page store file. + */ + public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException; +} http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java index e2ad070..0041ea6 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java @@ -365,21 +365,16 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen if (dirExisted && !idxFile.exists()) grpsWithoutIdx.add(grpDesc.groupId()); - FilePageStore idxStore = new FilePageStore( - PageMemory.FLAG_IDX, - idxFile, - pstCfg.getFileIOFactory(), - cctx.kernalContext().config().getMemoryConfiguration()); + FileVersionCheckingFactory pageStoreFactory = new FileVersionCheckingFactory( + pstCfg.getFileIOFactory(), igniteCfg.getMemoryConfiguration()); + + FilePageStore idxStore = pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile); FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()]; for (int partId = 0; partId < partStores.length; partId++) { - FilePageStore partStore = new FilePageStore( - PageMemory.FLAG_DATA, - new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)), - pstCfg.getFileIOFactory(), - cctx.kernalContext().config().getMemoryConfiguration() - ); + FilePageStore partStore = pageStoreFactory.createPageStore( + PageMemory.FLAG_DATA, new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId))); partStores[partId] = partStore; } http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java new file mode 100644 index 0000000..5d044ec --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java @@ -0,0 +1,53 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.File; +import org.apache.ignite.configuration.MemoryConfiguration; + +/** + * + */ +public class FilePageStoreV2 extends FilePageStore { + /** File version. */ + public static final int VERSION = 2; + + /** Header size. */ + private final int hdrSize; + + /** + * @param type Type. + * @param file File. + * @param factory Factory. + * @param cfg Config. + */ + public FilePageStoreV2(byte type, File file, FileIOFactory factory, MemoryConfiguration cfg) { + super(type, file, factory, cfg); + + hdrSize = cfg.getPageSize(); + } + + /** {@inheritDoc} */ + @Override public int headerSize() { + return hdrSize; + } + + /** {@inheritDoc} */ + @Override public int version() { + return VERSION; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java new file mode 100644 index 0000000..53bd802 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java @@ -0,0 +1,116 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.ignite.internal.processors.cache.persistence.file; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.configuration.MemoryConfiguration; + +/** + * Checks version in files if it's present on the disk, creates store with latest version otherwise. + */ +public class FileVersionCheckingFactory implements FilePageStoreFactory { + /** Property to override latest version. Should be used only in tests. */ + public static final String LATEST_VERSION_OVERRIDE_PROPERTY = "file.page.store.latest.version.override"; + + /** Latest page store version. */ + public final static int LATEST_VERSION = 2; + + /** Factory to provide I/O interfaces for read/write operations with files. */ + private final FileIOFactory fileIOFactory; + + /** Memory configuration. */ + private final MemoryConfiguration memCfg; + + /** + * @param fileIOFactory File io factory. + * @param memCfg Memory configuration. + */ + public FileVersionCheckingFactory( + FileIOFactory fileIOFactory, MemoryConfiguration memCfg) { + this.fileIOFactory = fileIOFactory; + this.memCfg = memCfg; + } + + /** {@inheritDoc} */ + @Override public FilePageStore createPageStore(byte type, File file) throws IgniteCheckedException { + if (!file.exists()) + return createPageStore(type, file, latestVersion()); + + try (FileIO fileIO = fileIOFactory.create(file, "r")) { + int minHdr = FilePageStore.HEADER_SIZE; + + if (fileIO.size() < minHdr) + return createPageStore(type, file, latestVersion()); + + ByteBuffer hdr = ByteBuffer.allocate(minHdr).order(ByteOrder.LITTLE_ENDIAN); + + while (hdr.remaining() > 0) + fileIO.read(hdr); + + hdr.rewind(); + + hdr.getLong(); // Read signature + + int ver = hdr.getInt(); + + return createPageStore(type, file, ver); + } + catch (IOException e) { + throw new IgniteCheckedException("Error while creating file page store [file=" + file + "]:", e); + } + } + + /** + * Resolves latest page store version. + */ + public int latestVersion() { + int latestVer = LATEST_VERSION; + + try { + latestVer = Integer.parseInt(System.getProperty(LATEST_VERSION_OVERRIDE_PROPERTY)); + } catch (NumberFormatException e) { + // No override. + } + + return latestVer; + } + + /** + * Instantiates specific version of FilePageStore. + * + * @param type Type. + * @param file File. + * @param ver Version. + */ + public FilePageStore createPageStore(byte type, File file, int ver) throws IgniteCheckedException { + switch (ver) { + case FilePageStore.VERSION: + return new FilePageStore(type, file, fileIOFactory, memCfg); + + case FilePageStoreV2.VERSION: + return new FilePageStoreV2(type, file, fileIOFactory, memCfg); + + default: + throw new IllegalArgumentException("Unknown version of file page store: " + ver); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java index c248c35..11d5eef 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java @@ -194,7 +194,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract long size = fileIO.size(); - fileIO.write(ByteBuffer.allocate((int)size - FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE); + fileIO.write(ByteBuffer.allocate((int)size - filePageStore.headerSize()), filePageStore.headerSize()); fileIO.force(); }
