IGNITE-5961 - Align pages in LFS partition files to pageSize

Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3919d803
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3919d803
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3919d803

Branch: refs/heads/ignite-5872
Commit: 3919d8035fb172e1546f7a3ecea0f29f8e1e2214
Parents: b5ad729
Author: Ivan Rakov <ivan.glu...@gmail.com>
Authored: Thu Aug 10 13:31:23 2017 +0300
Committer: Alexey Goncharuk <alexey.goncha...@gmail.com>
Committed: Thu Aug 10 13:31:23 2017 +0300

----------------------------------------------------------------------
 .../internal/pagemem/store/PageStore.java       |   5 +
 .../cache/persistence/file/FilePageStore.java   |  54 ++++++---
 .../persistence/file/FilePageStoreFactory.java  |  35 ++++++
 .../persistence/file/FilePageStoreManager.java  |  17 +--
 .../cache/persistence/file/FilePageStoreV2.java |  53 +++++++++
 .../file/FileVersionCheckingFactory.java        | 116 +++++++++++++++++++
 ...gnitePdsRecoveryAfterFileCorruptionTest.java |   2 +-
 7 files changed, 251 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
index 4698a6b..f6e577c 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
@@ -95,4 +95,9 @@ public interface PageStore {
      * @throws IgniteCheckedException If sync failed (IO error occurred).
      */
     public void ensure() throws IgniteCheckedException;
+
+    /**
+     * @return Page store version.
+     */
+    public int version();
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index a7ca13c..7fe1ffe 100755
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -45,10 +45,10 @@ public class FilePageStore implements PageStore {
     private static final long SIGNATURE = 0xF19AC4FE60C530B8L;
 
     /** File version. */
-    private static final int VERSION = 1;
+    public static final int VERSION = 1;
 
     /** Allocated field offset. */
-    public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 
1/*type*/ + 4/*page size*/;
+    static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 
4/*page size*/;
 
     /** */
     private final File cfgFile;
@@ -57,7 +57,7 @@ public class FilePageStore implements PageStore {
     private final byte type;
 
     /** Database configuration. */
-    private final MemoryConfiguration dbCfg;
+    protected final MemoryConfiguration dbCfg;
 
     /** Factory to provide I/O interfaces for read/write operations with files 
*/
     private final FileIOFactory ioFactory;
@@ -103,20 +103,36 @@ public class FilePageStore implements PageStore {
 
     /** {@inheritDoc} */
     @Override public boolean exists() {
-        return cfgFile.exists() && cfgFile.length() > HEADER_SIZE;
+        return cfgFile.exists() && cfgFile.length() > headerSize();
     }
 
     /**
+     * Size of page store header.
+     */
+    public int headerSize() {
+        return HEADER_SIZE;
+    }
+
+    /**
+     * Page store version.
+     */
+    public int version() {
+        return VERSION;
+    }
+
+    /**
+     * Creates header for current version file store. Doesn't init the store.
+     *
      * @param type Type.
      * @param pageSize Page size.
      * @return Byte buffer instance.
      */
-    public static ByteBuffer header(byte type, int pageSize) {
-        ByteBuffer hdr = 
ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+    public ByteBuffer header(byte type, int pageSize) {
+        ByteBuffer hdr = 
ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
 
         hdr.putLong(SIGNATURE);
 
-        hdr.putInt(VERSION);
+        hdr.putInt(version());
 
         hdr.put(type);
 
@@ -142,7 +158,7 @@ public class FilePageStore implements PageStore {
         }
 
         //there is 'super' page in every file
-        return HEADER_SIZE + dbCfg.getPageSize();
+        return headerSize() + dbCfg.getPageSize();
     }
 
     /**
@@ -150,7 +166,7 @@ public class FilePageStore implements PageStore {
      */
     private long checkFile() throws IgniteCheckedException {
         try {
-            ByteBuffer hdr = 
ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+            ByteBuffer hdr = 
ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
 
             while (hdr.remaining() > 0)
                 fileIO.read(hdr);
@@ -166,9 +182,9 @@ public class FilePageStore implements PageStore {
 
             int ver = hdr.getInt();
 
-            if (VERSION != ver)
+            if (version() != ver)
                 throw new IgniteCheckedException("Failed to verify store file 
(invalid file version)" +
-                    " [expectedVersion=" + VERSION +
+                    " [expectedVersion=" + version() +
                     ", fileVersion=" + ver + "]");
 
             byte type = hdr.get();
@@ -187,10 +203,10 @@ public class FilePageStore implements PageStore {
 
             long fileSize = cfgFile.length();
 
-            if (fileSize == HEADER_SIZE) // Every file has a special meta page.
-                fileSize = pageSize + HEADER_SIZE;
+            if (fileSize == headerSize()) // Every file has a special meta 
page.
+                fileSize = pageSize + headerSize();
 
-            if ((fileSize - HEADER_SIZE) % pageSize != 0)
+            if ((fileSize - headerSize()) % pageSize != 0)
                 throw new IgniteCheckedException("Failed to verify store file 
(invalid file size)" +
                     " [fileSize=" + U.hexLong(fileSize) +
                     ", pageSize=" + U.hexLong(pageSize) + ']');
@@ -346,9 +362,9 @@ public class FilePageStore implements PageStore {
         init();
 
         try {
-            assert buf.remaining() == HEADER_SIZE;
+            assert buf.remaining() == headerSize();
 
-            int len = HEADER_SIZE;
+            int len = headerSize();
 
             long off = 0;
 
@@ -425,7 +441,7 @@ public class FilePageStore implements PageStore {
 
             long off = pageOffset(pageId);
 
-            assert (off >= 0 && off + pageSize <= allocated.get() + 
HEADER_SIZE) || recover :
+            assert (off >= 0 && off + pageSize <= allocated.get() + 
headerSize()) || recover :
                 "off=" + U.hexLong(off) + ", allocated=" + 
U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
 
             assert pageBuf.capacity() == pageSize;
@@ -463,7 +479,7 @@ public class FilePageStore implements PageStore {
 
     /** {@inheritDoc} */
     @Override public long pageOffset(long pageId) {
-        return (long) PageIdUtils.pageIndex(pageId) * pageSize + HEADER_SIZE;
+        return (long) PageIdUtils.pageIndex(pageId) * pageSize + headerSize();
     }
 
     /** {@inheritDoc} */
@@ -494,7 +510,7 @@ public class FilePageStore implements PageStore {
 
         long off = allocPage();
 
-        return off / pageSize;
+        return (off - headerSize()) / pageSize;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
new file mode 100644
index 0000000..d97ab26
--- /dev/null
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreFactory.java
@@ -0,0 +1,35 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+
+/**
+ *
+ */
+public interface FilePageStoreFactory {
+    /**
+     * Creates instance of FilePageStore based on given file.
+     *
+     * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or 
{@link PageIdAllocator#FLAG_DATA}.
+     * @param file File Page store file.
+     */
+    public FilePageStore createPageStore(byte type, File file) throws 
IgniteCheckedException;
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index e2ad070..0041ea6 100755
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -365,21 +365,16 @@ public class FilePageStoreManager extends 
GridCacheSharedManagerAdapter implemen
         if (dirExisted && !idxFile.exists())
             grpsWithoutIdx.add(grpDesc.groupId());
 
-        FilePageStore idxStore = new FilePageStore(
-            PageMemory.FLAG_IDX,
-            idxFile,
-            pstCfg.getFileIOFactory(),
-            cctx.kernalContext().config().getMemoryConfiguration());
+        FileVersionCheckingFactory pageStoreFactory = new 
FileVersionCheckingFactory(
+            pstCfg.getFileIOFactory(), igniteCfg.getMemoryConfiguration());
+
+        FilePageStore idxStore = 
pageStoreFactory.createPageStore(PageMemory.FLAG_IDX, idxFile);
 
         FilePageStore[] partStores = new 
FilePageStore[grpDesc.config().getAffinity().partitions()];
 
         for (int partId = 0; partId < partStores.length; partId++) {
-            FilePageStore partStore = new FilePageStore(
-                PageMemory.FLAG_DATA,
-                new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, 
partId)),
-                pstCfg.getFileIOFactory(),
-                cctx.kernalContext().config().getMemoryConfiguration()
-            );
+            FilePageStore partStore = pageStoreFactory.createPageStore(
+                PageMemory.FLAG_DATA, new File(cacheWorkDir, 
String.format(PART_FILE_TEMPLATE, partId)));
 
             partStores[partId] = partStore;
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
new file mode 100644
index 0000000..5d044ec
--- /dev/null
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreV2.java
@@ -0,0 +1,53 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class FilePageStoreV2 extends FilePageStore {
+    /** File version. */
+    public static final int VERSION = 2;
+
+    /** Header size. */
+    private final int hdrSize;
+
+    /**
+     * @param type Type.
+     * @param file File.
+     * @param factory Factory.
+     * @param cfg Config.
+     */
+    public FilePageStoreV2(byte type, File file, FileIOFactory factory, 
MemoryConfiguration cfg) {
+        super(type, file, factory, cfg);
+
+        hdrSize = cfg.getPageSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int headerSize() {
+        return hdrSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int version() {
+        return VERSION;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
new file mode 100644
index 0000000..53bd802
--- /dev/null
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -0,0 +1,116 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ * Checks version in files if it's present on the disk, creates store with 
latest version otherwise.
+ */
+public class FileVersionCheckingFactory implements FilePageStoreFactory {
+    /** Property to override latest version. Should be used only in tests. */
+    public static final String LATEST_VERSION_OVERRIDE_PROPERTY = 
"file.page.store.latest.version.override";
+
+    /** Latest page store version. */
+    public final static int LATEST_VERSION = 2;
+
+    /** Factory to provide I/O interfaces for read/write operations with 
files. */
+    private final FileIOFactory fileIOFactory;
+
+    /** Memory configuration. */
+    private final MemoryConfiguration memCfg;
+
+    /**
+     * @param fileIOFactory File io factory.
+     * @param memCfg Memory configuration.
+     */
+    public FileVersionCheckingFactory(
+        FileIOFactory fileIOFactory, MemoryConfiguration memCfg) {
+        this.fileIOFactory = fileIOFactory;
+        this.memCfg = memCfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override public FilePageStore createPageStore(byte type, File file) 
throws IgniteCheckedException {
+        if (!file.exists())
+            return createPageStore(type, file, latestVersion());
+
+        try (FileIO fileIO = fileIOFactory.create(file, "r")) {
+            int minHdr = FilePageStore.HEADER_SIZE;
+
+            if (fileIO.size() < minHdr)
+                return createPageStore(type, file, latestVersion());
+
+            ByteBuffer hdr = 
ByteBuffer.allocate(minHdr).order(ByteOrder.LITTLE_ENDIAN);
+
+            while (hdr.remaining() > 0)
+                fileIO.read(hdr);
+
+            hdr.rewind();
+
+            hdr.getLong(); // Read signature
+
+            int ver = hdr.getInt();
+
+            return createPageStore(type, file, ver);
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Error while creating file page 
store [file=" + file + "]:", e);
+        }
+    }
+
+    /**
+     * Resolves latest page store version.
+     */
+    public int latestVersion() {
+        int latestVer = LATEST_VERSION;
+
+        try {
+            latestVer = 
Integer.parseInt(System.getProperty(LATEST_VERSION_OVERRIDE_PROPERTY));
+        } catch (NumberFormatException e) {
+            // No override.
+        }
+
+        return latestVer;
+    }
+
+    /**
+     * Instantiates specific version of FilePageStore.
+     *
+     * @param type Type.
+     * @param file File.
+     * @param ver Version.
+     */
+    public FilePageStore createPageStore(byte type, File file, int ver) throws 
IgniteCheckedException {
+        switch (ver) {
+            case FilePageStore.VERSION:
+                return new FilePageStore(type, file, fileIOFactory, memCfg);
+
+            case FilePageStoreV2.VERSION:
+                return new FilePageStoreV2(type, file, fileIOFactory, memCfg);
+
+            default:
+                throw new IllegalArgumentException("Unknown version of file 
page store: " + ver);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/3919d803/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
index c248c35..11d5eef 100644
--- 
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ 
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -194,7 +194,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest 
extends GridCommonAbstract
 
         long size = fileIO.size();
 
-        fileIO.write(ByteBuffer.allocate((int)size - 
FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE);
+        fileIO.write(ByteBuffer.allocate((int)size - 
filePageStore.headerSize()), filePageStore.headerSize());
 
         fileIO.force();
     }

Reply via email to