Author: mkataria Date: Wed Mar 25 12:46:05 2020 New Revision: 1875638 URL: http://svn.apache.org/viewvc?rev=1875638&view=rev Log: OAK-8898: On querying, IndexReader failed with AlreadyClosedException
Added: jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java (with props) jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java (with props) jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java (with props) Modified: jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Added: jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java?rev=1875638&view=auto ============================================================================== --- jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java (added) +++ jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java Wed Mar 25 12:46:05 2020 @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FacetTestHelper { + private static final Logger LOG = LoggerFactory.getLogger(FacetTestHelper.class); + + public static void sleep(int timeInMillis) { + try { + if (timeInMillis > 0) { + LOG.info("Sleep time set to:" + timeInMillis + " ms"); + Thread.sleep(timeInMillis); + } + } catch (InterruptedException e) { + LOG.error("Exception while thread sleep", e); + throw new RuntimeException(e); + } + } + +} Propchange: jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java?rev=1875638&r1=1875637&r2=1875638&view=diff ============================================================================== --- jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java (original) +++ jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java Wed Mar 25 12:46:05 2020 @@ -64,6 +64,8 @@ public class LuceneIndexNodeManager { private static final PerfLogger PERF_LOGGER = new PerfLogger(LoggerFactory.getLogger(LuceneIndexNodeManager.class.getName() + ".perf")); + public final static String OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME = "oak.lucene.oldFacetProviderTestFailSleepInstrument"; + private final static int OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_VALUE = Integer.getInteger(OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, 0); static LuceneIndexNodeManager open(String indexPath, NodeState root, NodeState defnNodeState, LuceneIndexReaderFactory readerFactory, @Nullable NRTIndexFactory nrtFactory) @@ -113,10 +115,12 @@ public class LuceneIndexNodeManager { private final Runnable refreshCallback = new Runnable() { @Override public void run() { + //Index reader gets closed when searching facets. This gets triggered in a multi threaded environment. Refer OAK-8898 + FacetTestHelper.sleep(OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_VALUE); if (refreshLock.tryAcquire()) { try { refreshReaders(); - }finally { + } finally { refreshLock.release(); } } Modified: jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1875638&r1=1875637&r2=1875638&view=diff ============================================================================== --- jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original) +++ jackrabbit/oak/branches/1.22/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Wed Mar 25 12:46:05 2020 @@ -52,6 +52,7 @@ import org.apache.jackrabbit.oak.plugins import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration; import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexLookup; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader; import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; @@ -193,6 +194,8 @@ public class LucenePropertyIndex extends private static boolean NON_LAZY = Boolean.parseBoolean(System.getProperty("oak.lucene.nonLazyIndex","true")); + public final static String OLD_FACET_PROVIDER_CONFIG_NAME = "oak.lucene.oldFacetProvider"; + private final static boolean OLD_FACET_PROVIDER = Boolean.getBoolean(OLD_FACET_PROVIDER_CONFIG_NAME); private static double MIN_COST = 2.1; @@ -259,7 +262,7 @@ public class LucenePropertyIndex extends private boolean noDocs = false; private IndexSearcher indexSearcher; private int indexNodeId = -1; - private LuceneFacetProvider facetProvider = null; + private FacetProvider facetProvider; private int rewoundCount = 0; @Override @@ -277,7 +280,7 @@ public class LucenePropertyIndex extends } private FulltextResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, Map<String, String> excerpts, - LuceneFacetProvider facetProvider, + FacetProvider facetProvider, String explanation) throws IOException { IndexReader reader = searcher.getIndexReader(); //TODO Look into usage of field cache for retrieving the path @@ -362,11 +365,18 @@ public class LucenePropertyIndex extends PERF_LOGGER.end(start, -1, "{} ...", docs.scoreDocs.length); nextBatchSize = (int) Math.min(nextBatchSize * 2L, 100000); - long f = PERF_LOGGER.start(); if (facetProvider == null) { - facetProvider = new LuceneFacetProvider( - FacetHelper.getFacets(searcher, query, plan, indexNode.getDefinition().getSecureFacetConfiguration()) - ); + long f = PERF_LOGGER.start(); + if (OLD_FACET_PROVIDER) { + // here the current searcher gets referenced for later + // but the searcher might get closed in the meantime + facetProvider = new LuceneFacetProvider( + FacetHelper.getFacets(searcher, query, plan, indexNode.getDefinition().getSecureFacetConfiguration()) + ); + } else { + // a new searcher is opened and closed when needed + facetProvider = new DelayedLuceneFacetProvider(LucenePropertyIndex.this, query, plan, indexNode.getDefinition().getSecureFacetConfiguration()); + } PERF_LOGGER.end(f, -1, "facets retrieved"); } @@ -1564,6 +1574,45 @@ public class LucenePropertyIndex extends return Iterators.concat(propIndex.iterator(), itr); } + static class DelayedLuceneFacetProvider implements FacetProvider { + private final LucenePropertyIndex index; + private final Query query; + private final IndexPlan plan; + private final SecureFacetConfiguration config; + + DelayedLuceneFacetProvider(LucenePropertyIndex index, Query query, IndexPlan plan, SecureFacetConfiguration config) { + this.index = index; + this.query = query; + this.plan = plan; + this.config = config; + } + + @Override + public List<Facet> getFacets(int numberOfFacets, String columnName) throws IOException { + LuceneIndexNode indexNode = index.acquireIndexNode(plan); + try { + IndexSearcher searcher = indexNode.getSearcher(); + String facetFieldName = FulltextIndex.parseFacetField(columnName); + Facets facets = FacetHelper.getFacets(searcher, query, plan, config); + if (facets != null) { + ImmutableList.Builder<Facet> res = new ImmutableList.Builder<>(); + FacetResult topChildren = facets.getTopChildren(numberOfFacets, facetFieldName); + if (topChildren != null) { + for (LabelAndValue lav : topChildren.labelValues) { + res.add(new Facet( + lav.label, lav.value.intValue() + )); + } + return res.build(); + } + } + return null; + } finally { + indexNode.release(); + } + } + } + static class LuceneFacetProvider implements FacetProvider { private final Facets facets; Added: jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java?rev=1875638&view=auto ============================================================================== --- jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java (added) +++ jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java Wed Mar 25 12:46:05 2020 @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid; + +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.jcr.Jcr; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexNodeManager; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex; +import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil; +import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.OptionalEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.query.AbstractQueryTest; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import javax.jcr.GuestCredentials; +import javax.jcr.Repository; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.Value; +import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; +import javax.jcr.query.QueryResult; +import javax.jcr.query.RowIterator; +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; +import static org.apache.jackrabbit.oak.spi.mount.Mounts.defaultMountInfoProvider; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +public class DelayedFacetReadTest extends AbstractQueryTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + private static final int NUM_LABELS = 4; + private static final int NUM_LEAF_NODES = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; + private static final String FACET_PROP = "facets"; + private static final long REFRESH_DELTA = TimeUnit.SECONDS.toMillis(1); + + private ExecutorService executorService = Executors.newFixedThreadPool(2); + private OptionalEditorProvider optionalEditorProvider = new OptionalEditorProvider(); + private NRTIndexFactory nrtIndexFactory; + private LuceneIndexProvider luceneIndexProvider; + private NodeStore nodeStore; + private DocumentQueue queue; + private Clock clock = new Clock.Virtual(); + private Whiteboard wb; + private QueryManager qm; + private Repository jcrRepo; + private Jcr jcr; + private Oak oak; + private Thread thread; + // backup original system properties i.e. before test started + private final Properties backupProperties = (Properties) System.getProperties().clone(); + + @After + public void tearDown() throws IOException { + luceneIndexProvider.close(); + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + new ExecutorCloser(executorService).close(); + nrtIndexFactory.close(); + // restore original system properties i.e. before test started + System.setProperties(backupProperties); + } + + @Override + protected ContentRepository createRepository() { + IndexCopier copier; + try { + copier = new IndexCopier(executorService, temporaryFolder.getRoot()); + } catch (IOException e) { + throw new RuntimeException(e); + } + MountInfoProvider mip = defaultMountInfoProvider(); + nrtIndexFactory = new NRTIndexFactory(copier, clock, TimeUnit.MILLISECONDS.toSeconds(REFRESH_DELTA), StatisticsProvider.NOOP); + nrtIndexFactory.setAssertAllResourcesClosed(true); + LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier); + IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory); + luceneIndexProvider = new LuceneIndexProvider(tracker); + queue = new DocumentQueue(100, tracker, sameThreadExecutor()); + LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier, + tracker, + null, + null, + mip); + editorProvider.setIndexingQueue(queue); + LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, StatisticsProvider.NOOP); + nodeStore = new MemoryNodeStore(); + oak = new Oak(nodeStore) + .with(new InitialContent()) + .with(new OpenSecurityProvider()) + .with((QueryIndexProvider) luceneIndexProvider) + .with((Observer) luceneIndexProvider) + .with(localIndexObserver) + .with(editorProvider) + .with(new PropertyIndexEditorProvider()) + .with(new NodeTypeIndexProvider()) + .with(optionalEditorProvider) + .with(new NodeCounterEditorProvider()) + //Effectively disable async indexing auto run + //such that we can control run timing as per test requirement + .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1)); + + wb = oak.getWhiteboard(); + ContentRepository repo = oak.createContentRepository(); + return repo; + } + + private void createSmallDataset(int k) throws RepositoryException { + Random rGen = new Random(42); + Tree par = createPath("/parent" + k); + par.setProperty("foo", "bar"); + for (int i = 0; i < NUM_LABELS * 2; i++) { + Tree subPar = par.addChild("par" + i); + for (int j = 0; j < NUM_LEAF_NODES / (2 * NUM_LABELS); j++) { + Tree child = subPar.addChild("c" + j); + child.setProperty("cons", "val"); + int labelNum = rGen.nextInt(NUM_LABELS); + child.setProperty("foo", "l" + labelNum); + } + } + } + + /* + This test's counter part is MultithreadedOldLuceneFacetProviderReadFailureTest which test old implementation having race condition on index reader opening + and closing. The test in MultithreadedOldLuceneFacetProviderReadFailureTest is marked as ignored but can be tested on demand. + */ + @Test + public void facet() throws Exception { + // Explicitly setting following configs to run DelayedLuceneFacetProvider and a thread sleep of 50 ms in refresh readers. Refer: OAK-8898 + System.setProperty(LucenePropertyIndex.OLD_FACET_PROVIDER_CONFIG_NAME, "false"); + // The variable is static final so once set it remains same for all tests and which will lead to slow execution + // of other tests as this add a sleep of specified milliseconds in refresh reader method in LuceneIndexNodeManager. + // System.setProperty(LuceneIndexNodeManager.OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, "40"); + Thread.currentThread().setName("main"); + String idxName = "hybridtest"; + Tree idx = createIndex(root.getTree("/"), idxName); + TestUtil.enableIndexingMode(idx, FulltextIndexConstants.IndexingMode.NRT); + setTraversalEnabled(false); + root.commit(); + jcr = new Jcr(oak); + jcrRepo = jcr.createRepository(); + createSmallDataset(0); + clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1); + root.commit(); + runAsyncIndex(); + createSmallDataset(2); + clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1); + root.commit(); + Session anonSession = jcrRepo.login(new GuestCredentials()); + qm = anonSession.getWorkspace().getQueryManager(); + Query q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2); + QueryResult qr = q.execute(); + + thread = new Thread(new Runnable() { + public void run() { + Thread.currentThread().setName("branch"); + Query q = null; + try { + clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1); + q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2); + QueryResult qr = q.execute(); + } catch (RepositoryException | InterruptedException e) { + e.printStackTrace(); + } + } + }); + thread.start(); + try { + RowIterator it = qr.getRows(); + String firstColumnName = qr.getColumnNames()[0]; + if (it.hasNext()) { + Value v = it.nextRow().getValue(firstColumnName); + } + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + private void runAsyncIndex() { + AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, Runnable.class, new Predicate<Runnable>() { + @Override + public boolean test(@Nullable Runnable input) { + return input instanceof AsyncIndexUpdate; + } + }); + assertNotNull(async); + async.run(); + if (async.isFailing()) { + fail("AsyncIndexUpdate failed"); + } + root.refresh(); + } + + private Tree createPath(String path) { + Tree base = root.getTree("/"); + for (String e : PathUtils.elements(path)) { + base = base.addChild(e); + } + return base; + } + + private Tree createIndex(Tree index, String name) throws RepositoryException { + IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder(); + idxBuilder.noAsync() + .indexRule("nt:base") + .property("cons").propertyIndex() + .property("foo").propertyIndex() + .getBuilderTree().setProperty(PROP_FACETS, true); + Tree facetConfig = idxBuilder.getBuilderTree().addChild(FACET_PROP); + facetConfig.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + facetConfig.setProperty("secure", "statistical"); + facetConfig.setProperty("topChildren", "100"); + Tree idxTree = index.getChild("oak:index").addChild(name); + idxBuilder.build(idxTree); + return idxTree; + } +} Propchange: jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java ------------------------------------------------------------------------------ svn:eol-style = native Added: jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java?rev=1875638&view=auto ============================================================================== --- jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java (added) +++ jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java Wed Mar 25 12:46:05 2020 @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid; + +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.jcr.Jcr; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexNodeManager; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex; +import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil; +import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.OptionalEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.query.AbstractQueryTest; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import javax.jcr.GuestCredentials; +import javax.jcr.Repository; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.Value; +import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; +import javax.jcr.query.QueryResult; +import javax.jcr.query.RowIterator; +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; +import static org.apache.jackrabbit.oak.spi.mount.Mounts.defaultMountInfoProvider; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +public class MultithreadedOldLuceneFacetProviderReadFailureTest extends AbstractQueryTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + private static final int NUM_LABELS = 4; + private static final int NUM_LEAF_NODES = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; + private static final String FACET_PROP = "facets"; + private static final long REFRESH_DELTA = TimeUnit.SECONDS.toMillis(1); + + private ExecutorService executorService = Executors.newFixedThreadPool(2); + private OptionalEditorProvider optionalEditorProvider = new OptionalEditorProvider(); + private NRTIndexFactory nrtIndexFactory; + private LuceneIndexProvider luceneIndexProvider; + private NodeStore nodeStore; + private DocumentQueue queue; + private Clock clock = new Clock.Virtual(); + private Whiteboard wb; + private QueryManager qm; + private Repository jcrRepo; + private Jcr jcr; + private Oak oak; + private Thread thread; + // backup original system properties i.e. before test started + private final Properties backupProperties = (Properties) System.getProperties().clone(); + + @After + public void tearDown() throws IOException { + luceneIndexProvider.close(); + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + new ExecutorCloser(executorService).close(); + nrtIndexFactory.close(); + // restore original system properties i.e. before test started + System.setProperties(backupProperties); + } + + @Override + protected ContentRepository createRepository() { + IndexCopier copier; + try { + copier = new IndexCopier(executorService, temporaryFolder.getRoot()); + } catch (IOException e) { + throw new RuntimeException(e); + } + MountInfoProvider mip = defaultMountInfoProvider(); + nrtIndexFactory = new NRTIndexFactory(copier, clock, TimeUnit.MILLISECONDS.toSeconds(REFRESH_DELTA), StatisticsProvider.NOOP); + nrtIndexFactory.setAssertAllResourcesClosed(true); + LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier); + IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory); + luceneIndexProvider = new LuceneIndexProvider(tracker); + queue = new DocumentQueue(100, tracker, sameThreadExecutor()); + LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier, + tracker, + null, + null, + mip); + editorProvider.setIndexingQueue(queue); + LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, StatisticsProvider.NOOP); + nodeStore = new MemoryNodeStore(); + oak = new Oak(nodeStore) + .with(new InitialContent()) + .with(new OpenSecurityProvider()) + .with((QueryIndexProvider) luceneIndexProvider) + .with((Observer) luceneIndexProvider) + .with(localIndexObserver) + .with(editorProvider) + .with(new PropertyIndexEditorProvider()) + .with(new NodeTypeIndexProvider()) + .with(optionalEditorProvider) + .with(new NodeCounterEditorProvider()) + //Effectively disable async indexing auto run + //such that we can control run timing as per test requirement + .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1)); + + wb = oak.getWhiteboard(); + ContentRepository repo = oak.createContentRepository(); + return repo; + } + + private void createSmallDataset(int k) throws RepositoryException { + Random rGen = new Random(42); + Tree par = createPath("/parent" + k); + par.setProperty("foo", "bar"); + for (int i = 0; i < NUM_LABELS * 2; i++) { + Tree subPar = par.addChild("par" + i); + for (int j = 0; j < NUM_LEAF_NODES / (2 * NUM_LABELS); j++) { + Tree child = subPar.addChild("c" + j); + child.setProperty("cons", "val"); + int labelNum = rGen.nextInt(NUM_LABELS); + child.setProperty("foo", "l" + labelNum); + } + } + } + + /* + The facet test delays threads intentionally so that index reader closes before reading facets. But sometimes it may fail + because of load on a system. You can change the system property below between 30-50 and see if this test passes on + your system. + The following test is ignored as it is not a consistent test and result may vary based on system load etc. + This test is a race condition and can be test for failure by using different values for OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME + The probable value where this will fail is between 30ms to 60ms. + This test's counter part is DelayedFacetReadTest which test new implementation solving the race condition on index reader opening + and closing. + */ + @Ignore + @Test(expected = RuntimeException.class) + public void facet() throws Exception { + // Explicitly setting following configs to run LuceneFacetProvider (old) and a thread sleep of 50 ms in refresh readers. Refer: OAK-8898 + System.setProperty(LucenePropertyIndex.OLD_FACET_PROVIDER_CONFIG_NAME, "true"); + System.setProperty(LuceneIndexNodeManager.OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, "40"); + Thread.currentThread().setName("main"); + String idxName = "hybridtest"; + Tree idx = createIndex(root.getTree("/"), idxName); + TestUtil.enableIndexingMode(idx, FulltextIndexConstants.IndexingMode.NRT); + setTraversalEnabled(false); + root.commit(); + jcr = new Jcr(oak); + jcrRepo = jcr.createRepository(); + createSmallDataset(0); + clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1); + root.commit(); + runAsyncIndex(); + createSmallDataset(2); + clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1); + root.commit(); + Session anonSession = jcrRepo.login(new GuestCredentials()); + qm = anonSession.getWorkspace().getQueryManager(); + Query q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2); + QueryResult qr = q.execute(); + + thread = new Thread(new Runnable() { + public void run() { + Thread.currentThread().setName("branch"); + Query q = null; + try { + clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1); + q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2); + QueryResult qr = q.execute(); + } catch (RepositoryException | InterruptedException e) { + e.printStackTrace(); + } + } + }); + thread.start(); + RowIterator it; + try { + it = qr.getRows(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("org.apache.lucene.store.AlreadyClosedException: this IndexReader is closed")); + e.printStackTrace(System.out); + throw e; + } + String firstColumnName = qr.getColumnNames()[0]; + if (it.hasNext()) { + Value v = it.nextRow().getValue(firstColumnName); + } + } + + private void runAsyncIndex() { + AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, Runnable.class, new Predicate<Runnable>() { + @Override + public boolean test(@Nullable Runnable input) { + return input instanceof AsyncIndexUpdate; + } + }); + assertNotNull(async); + async.run(); + if (async.isFailing()) { + fail("AsyncIndexUpdate failed"); + } + root.refresh(); + } + + private Tree createPath(String path) { + Tree base = root.getTree("/"); + for (String e : PathUtils.elements(path)) { + base = base.addChild(e); + } + return base; + } + + private Tree createIndex(Tree index, String name) throws RepositoryException { + IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder(); + idxBuilder.noAsync() + .indexRule("nt:base") + .property("cons").propertyIndex() + .property("foo").propertyIndex() + .getBuilderTree().setProperty(PROP_FACETS, true); + Tree facetConfig = idxBuilder.getBuilderTree().addChild(FACET_PROP); + facetConfig.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + facetConfig.setProperty("secure", "statistical"); + facetConfig.setProperty("topChildren", "100"); + Tree idxTree = index.getChild("oak:index").addChild(name); + idxBuilder.build(idxTree); + return idxTree; + } + +} Propchange: jackrabbit/oak/branches/1.22/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java ------------------------------------------------------------------------------ svn:eol-style = native