Author: chetanm
Date: Thu Sep 15 07:15:23 2016
New Revision: 1760837
URL: http://svn.apache.org/viewvc?rev=1760837&view=rev
Log:
OAK-4412 - Lucene hybrid index
Garbage collect NRTIndex created directory on startup which may have been left
due to unclean shutdown
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectory.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectoryTest.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectory.java?rev=1760837&r1=1760836&r2=1760837&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectory.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectory.java
Thu Sep 15 07:15:23 2016
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.File;
import java.io.FileFilter;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -42,6 +43,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndex;
import org.apache.jackrabbit.oak.stats.Clock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -251,6 +253,10 @@ public class IndexRootDirectory {
"Deletion would be retried later again.", dir);
}
}
+
+ if (!dirs.isEmpty()) {
+ totalDeletedSize += gcNRTIndexDirs(dirs.get(0));
+ }
totalDeletedSize += deleteOldFormatDir(dirs.get(0).getJcrPath());
}
@@ -260,6 +266,30 @@ public class IndexRootDirectory {
}
}
+ /**
+ * Removes all directory created by NRTIndex which have
+ * nrt prefix
+ */
+ private long gcNRTIndexDirs(LocalIndexDir idxDir) {
+ final String prefix = getFSSafeName(NRTIndex.NRT_DIR_PREFIX);
+ File[] nrtDirs = idxDir.dir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.startsWith(prefix);
+ }
+ });
+
+ long size = 0;
+ if (nrtDirs != null) {
+ for (File f : nrtDirs){
+ size += FileUtils.sizeOf(f);
+ FileUtils.deleteQuietly(f);
+ }
+ }
+
+ return size;
+ }
+
@CheckForNull
private LocalIndexDir findMatchingIndexDir(File dir) throws IOException {
//Resolve to canonical file so that equals can work reliable
@@ -294,7 +324,8 @@ public class IndexRootDirectory {
return new File(indexRootDir, subDir);
}
- private static String getFSSafeName(String e) {
+ static String getFSSafeName(String e) {
+ //TODO Exclude -_ like chars via [^\W_]
return e.replaceAll("\\W", "");
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java?rev=1760837&r1=1760836&r2=1760837&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
Thu Sep 15 07:15:23 2016
@@ -57,7 +57,7 @@ public class NRTIndex implements Closeab
/**
* Prefix used for naming the directory created for NRT indexes
*/
- private static final String NRT_DIR_PREFIX = "nrt-";
+ public static final String NRT_DIR_PREFIX = "nrt-";
private final IndexDefinition definition;
private final IndexCopier indexCopier;
@@ -151,8 +151,7 @@ public class NRTIndex implements Closeab
}
private synchronized NRTIndexWriter createWriter() throws IOException {
- long uniqueCount = System.currentTimeMillis() +
COUNTER.incrementAndGet();
- String dirName = NRT_DIR_PREFIX + uniqueCount;
+ String dirName = generateDirName();
indexDir = indexCopier.getIndexDir(definition,
definition.getIndexPathFromConfig(), dirName);
Directory fsdir = FSDirectory.open(indexDir);
//TODO make these configurable
@@ -162,6 +161,11 @@ public class NRTIndex implements Closeab
return new NRTIndexWriter(indexWriter);
}
+ public static String generateDirName() {
+ long uniqueCount = System.currentTimeMillis() +
COUNTER.incrementAndGet();
+ return NRT_DIR_PREFIX + uniqueCount;
+ }
+
private static class NRTReader implements LuceneIndexReader {
private final IndexReader indexReader;
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectoryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectoryTest.java?rev=1760837&r1=1760836&r2=1760837&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectoryTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexRootDirectoryTest.java
Thu Sep 15 07:15:23 2016
@@ -28,6 +28,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorContext;
+import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndex;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.junit.Before;
@@ -195,6 +196,21 @@ public class IndexRootDirectoryTest {
assertEquals(1, dir.getLocalIndexes("/a").size());
}
+ @Test
+ public void gcNRTDirsOnStart() throws Exception{
+ configureUniqueId();
+
+ File fa0 = dir.getIndexDir(getDefn(), "/a", "default");
+ File nrt0 = dir.getIndexDir(getDefn(), "/a",
NRTIndex.generateDirName());
+ File nrt1 = dir.getIndexDir(getDefn(), "/a",
NRTIndex.generateDirName());
+
+ //Now reinitialize
+ dir = new IndexRootDirectory(temporaryFolder.getRoot());
+ assertFalse(nrt0.exists());
+ assertFalse(nrt1.exists());
+ assertTrue(fa0.exists());
+ }
+
private NodeBuilder resetBuilder() {
builder = EMPTY_NODE.builder();
return builder;
