Author: chetanm
Date: Wed Nov 16 08:45:17 2016
New Revision: 1769940
URL: http://svn.apache.org/viewvc?rev=1769940&view=rev
Log:
OAK-4114 - Cached lucene index gets corrupted in case of unclean shutdown and
journal rollback in SegmentNodeStore
Added a sanity check logic which upon first index access checks for local index
file sanity. In case of mismatch it would purge the local index content
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
(with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityCheckerTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java?rev=1769940&r1=1769939&r2=1769940&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
Wed Nov 16 08:45:17 2016
@@ -46,11 +46,11 @@ import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.CopyOnReadDirectory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.CopyOnWriteDirectory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexSanityChecker;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryUtils;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexRootDirectory;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LocalIndexDir;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LocalIndexFile;
-import org.apache.jackrabbit.oak.util.PerfLogger;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.FilterDirectory;
@@ -102,6 +102,7 @@ public class IndexCopier implements Copy
private final boolean prefetchEnabled;
private volatile boolean closed;
private final IndexRootDirectory indexRootDirectory;
+ private final Set<String> validatedIndexPaths =
Sets.newConcurrentHashSet();
public IndexCopier(Executor executor, File indexRootDir) throws
IOException {
this(executor, indexRootDir, false);
@@ -117,12 +118,14 @@ public class IndexCopier implements Copy
public Directory wrapForRead(String indexPath, IndexDefinition definition,
Directory remote, String dirName) throws
IOException {
Directory local = createLocalDirForIndexReader(indexPath, definition,
dirName);
+ checkIntegrity(indexPath, local, remote);
return new CopyOnReadDirectory(this, remote, local, prefetchEnabled,
indexPath, executor);
}
public Directory wrapForWrite(IndexDefinition definition, Directory
remote, boolean reindexMode, String dirName) throws IOException {
Directory local = createLocalDirForIndexWriter(definition, dirName);
String indexPath = definition.getIndexPathFromConfig();
+ checkIntegrity(indexPath, local, remote);
return new CopyOnWriteDirectory(this, remote, local, reindexMode,
indexPath, executor);
}
@@ -233,6 +236,22 @@ public class IndexCopier implements Copy
return sharedSet;
}
+ private void checkIntegrity(String indexPath, Directory local, Directory
remote) throws IOException {
+ if (validatedIndexPaths.contains(indexPath)){
+ return;
+ }
+
+ //The integrity check needs to be done for the very first time at
startup when
+ //a directory gets created as at that time it can be ensured that
there is no
+ //work in progress files, no memory mapping issue etc
+ //Also at this time its required that state in local dir should
exactly same as
+ //one in remote dir
+ synchronized (validatedIndexPaths){
+ new IndexSanityChecker(indexPath, local, remote).check();
+ validatedIndexPaths.add(indexPath);
+ }
+ }
+
/**
* Creates the workDir. If it exists then it is cleaned
*
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java?rev=1769940&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
Wed Nov 16 08:45:17 2016
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.lucene.store.Directory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Checks that all files in local which are present in remote have same file
length.
+ * If there is a size mismatch in any one of the file then whole of local
index content
+ * would be purged
+ */
+public class IndexSanityChecker {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final Directory local;
+ private final Directory remote;
+ private final String indexPath;
+ private int localFileCount;
+ private int remoteFileCount;
+ private long localDirSize;
+ private long remoteDirSize;
+
+ public IndexSanityChecker(String indexPath, Directory local, Directory
remote) {
+ this.local = local;
+ this.remote = remote;
+ this.indexPath = indexPath;
+ }
+
+ public boolean check() throws IOException {
+ boolean allFine = true;
+ //TODO Add support for checksum based checks
+ if (isThereASizeMismatch()){
+ //In case of any mismatch just purge all local files
+ deleteAllFiles(local);
+ allFine = false;
+ } else {
+ //Remove local files which are not found in remote
+ for (String fileName : local.listAll()) {
+ if (!remote.fileExists(fileName)) {
+ local.deleteFile(fileName);
+ }
+ }
+ }
+
+ if (allFine) {
+ log.info("Local index directory content found to be valid for
index [{}]. " +
+ "Stats Local: {} files ({}), Remote: {} files ({})",
indexPath,
+ localFileCount,
IOUtils.humanReadableByteCount(localDirSize),
+ remoteFileCount,
IOUtils.humanReadableByteCount(remoteDirSize));
+ } else {
+ log.warn("Local index directory content were not found to be in
sync with remote for index [{}]. " +
+ "Local directory content has been purged and would be
synced again from remote", indexPath);
+ }
+ return allFine;
+ }
+
+ private boolean isThereASizeMismatch() throws IOException {
+ for (String fileName : remote.listAll()){
+ long localLength = DirectoryUtils.getFileLength(local, fileName);
+ long remoteLength = remote.fileLength(fileName);
+
+ //This is a weak check based on length.
+ if (localLength > 0 && localLength != remoteLength){
+ log.warn("[{}] Found local copy for {} in {} but size of local
{} differs from remote {}. ",
+ indexPath, fileName, local, localLength, remoteLength);
+ return true;
+ }
+
+ if (localLength > 0) {
+ localDirSize += localLength;
+ localFileCount++;
+ }
+
+ remoteDirSize += remoteLength;
+ remoteFileCount++;
+ }
+ return false;
+ }
+
+ private static void deleteAllFiles(Directory dir) throws IOException {
+ for (String fileName : dir.listAll()){
+ dir.deleteFile(fileName);
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityChecker.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java?rev=1769940&r1=1769939&r2=1769940&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
Wed Nov 16 08:45:17 2016
@@ -1022,7 +1022,6 @@ public class IndexCopierTest {
executorService.shutdown();
}
- @Ignore("OAK-4114")
@Test
public void directoryContentMismatch_COR() throws Exception{
Directory baseDir = new CloseSafeDir();
Added:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityCheckerTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityCheckerTest.java?rev=1769940&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityCheckerTest.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityCheckerTest.java
Wed Nov 16 08:45:17 2016
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMDirectory;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class IndexSanityCheckerTest {
+ private Random rnd = new Random();
+
+ private Directory local = new RAMDirectory();
+ private Directory remote = new RAMDirectory();
+
+ @Test
+ public void validDirs() throws Exception{
+ byte[] t1 = writeFile(local, "t1", 100);
+ writeFile(remote, "t1", t1);
+
+ assertTrue(new IndexSanityChecker("/foo", local, remote).check());
+
+ assertTrue(local.fileExists("t1"));
+ assertTrue(remote.fileExists("t1"));
+ }
+
+ @Test
+ public void sizeMismatch() throws Exception{
+ byte[] t1L = writeFile(local, "t1", 100);
+ byte[] t1R = writeFile(remote, "t1", 110);
+ byte[] t2R = writeFile(remote, "t2", 120);
+ byte[] t3R = writeFile(remote, "t3", 140);
+ writeFile(local, "t3", t3R);
+
+ assertFalse(new IndexSanityChecker("/foo", local, remote).check());
+
+ assertTrue(remote.fileExists("t3"));
+
+ //In case of size mismatch all local files would be removed
+ assertFalse(local.fileExists("t1"));
+ assertFalse(local.fileExists("t3"));
+ }
+
+ @Test
+ public void extraLocalFiles() throws Exception{
+ byte[] t1L = writeFile(local, "t1", 100);
+ byte[] t3R = writeFile(remote, "t3", 140);
+ writeFile(local, "t3", t3R);
+
+ new IndexSanityChecker("/foo", local, remote).check();
+
+ //t1 exist in local but not in remote
+ //it must be removed
+ assertFalse(local.fileExists("t1"));
+
+ //t3 should remain present
+ assertTrue(remote.fileExists("t3"));
+ }
+
+ private byte[] writeFile(Directory dir, String name, int size) throws
IOException {
+ byte[] data = randomBytes(rnd.nextInt(size) + 1);
+ writeFile(dir, name, data);
+ return data;
+ }
+
+ private void writeFile(Directory dir, String name, byte[] data) throws
IOException {
+ IndexOutput o = dir.createOutput(name, IOContext.DEFAULT);
+ o.writeBytes(data, data.length);
+ o.close();
+ }
+
+ private byte[] randomBytes(int size) {
+ byte[] data = new byte[size];
+ rnd.nextBytes(data);
+ return data;
+ }
+}
\ No newline at end of file
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexSanityCheckerTest.java
------------------------------------------------------------------------------
svn:eol-style = native
