Repository: jena Updated Branches: refs/heads/master d1b5e21df -> 4ee8d9e37
JENA-848 : Fix concurrency control for in-process Lucene index Project: http://git-wip-us.apache.org/repos/asf/jena/repo Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/4ee8d9e3 Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/4ee8d9e3 Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/4ee8d9e3 Branch: refs/heads/master Commit: 4ee8d9e37885595710dc3a3f4d28d4aedc39f51a Parents: d1b5e21 Author: Stephen Allen <[email protected]> Authored: Tue Jan 13 19:34:37 2015 -0500 Committer: Stephen Allen <[email protected]> Committed: Tue Jan 13 19:34:37 2015 -0500 ---------------------------------------------------------------------- jena-text/ReleaseNotes.txt | 8 + .../jena/query/text/DatasetGraphText.java | 17 ++ .../jena/query/text/TextDatasetFactory.java | 27 ++- .../apache/jena/query/text/TextIndexLucene.java | 70 ++++--- .../text/assembler/TextDatasetAssembler.java | 2 +- .../AbstractTestDatasetWithGraphTextIndex.java | 24 ++- ...ractTestDatasetWithLuceneGraphTextIndex.java | 22 ++- .../AbstractTestDatasetWithLuceneTextIndex.java | 103 ---------- .../text/AbstractTestDatasetWithTextIndex.java | 6 +- .../AbstractTestDatasetWithTextIndexBase.java | 10 +- .../org/apache/jena/query/text/TS_Text.java | 2 +- .../TestDatasetWithEmbeddedSolrTextIndex.java | 15 +- .../text/TestDatasetWithKeywordAnalyzer.java | 53 +++-- ...TestDatasetWithLowerCaseKeywordAnalyzer.java | 18 +- .../TestDatasetWithLuceneGraphTextIndex.java | 5 +- .../text/TestDatasetWithLuceneTextIndex.java | 100 +++++++++- .../text/TestDatasetWithSimpleAnalyzer.java | 45 +++-- .../text/TestDatasetWithStandardAnalyzer.java | 45 +++-- .../text/TestLuceneWithMultipleThreads.java | 193 +++++++++++++++++++ ...BeforeWriteOnDatasetWithLuceneTextIndex.java | 49 ----- .../assembler/AbstractTestTextAssembler.java | 70 ++++++- .../assembler/TestTextDatasetAssembler.java | 4 +- .../assembler/TestTextIndexLuceneAssembler.java | 52 +++-- 23 files changed, 593 insertions(+), 347 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/ReleaseNotes.txt ---------------------------------------------------------------------- diff --git a/jena-text/ReleaseNotes.txt b/jena-text/ReleaseNotes.txt new file mode 100644 index 0000000..5645f55 --- /dev/null +++ b/jena-text/ReleaseNotes.txt @@ -0,0 +1,8 @@ +ChangeLog for jena-text +======================= + +==== jena-text 1.1.2 + ++ JENA-848 : Fix concurrency control for in-process Lucene index + + http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/main/java/org/apache/jena/query/text/DatasetGraphText.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/DatasetGraphText.java b/jena-text/src/main/java/org/apache/jena/query/text/DatasetGraphText.java index e20e68b..1681323 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/DatasetGraphText.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/DatasetGraphText.java @@ -37,8 +37,14 @@ public class DatasetGraphText extends DatasetGraphMonitor implements Transaction private final TextIndex textIndex ; private final Transactional dsgtxn ; private final Graph dftGraph ; + private final boolean closeIndexOnClose; public DatasetGraphText(DatasetGraph dsg, TextIndex index, TextDocProducer producer) + { + this(dsg, index, producer, false); + } + + public DatasetGraphText(DatasetGraph dsg, TextIndex index, TextDocProducer producer, boolean closeIndexOnClose) { super(dsg, producer) ; this.textIndex = index ; @@ -47,6 +53,7 @@ public class DatasetGraphText extends DatasetGraphMonitor implements Transaction else dsgtxn = new DatasetGraphWithLock(dsg) ; dftGraph = GraphView.createDefaultGraph(this) ; + this.closeIndexOnClose = closeIndexOnClose; } // ---- Intecept these and force the use of views. @@ -122,6 +129,7 @@ public class DatasetGraphText extends DatasetGraphMonitor implements Transaction catch (Throwable ex) { log.warn("Exception in commit: " + ex.getMessage(), ex) ; dsgtxn.abort() ; + throw ex; } } @@ -152,4 +160,13 @@ public class DatasetGraphText extends DatasetGraphMonitor implements Transaction } catch (Throwable ex) { log.warn("Exception in end: " + ex.getMessage(), ex) ; } } + + @Override + public void close() { + super.close(); + if (closeIndexOnClose) { + textIndex.close(); + } + } + } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/main/java/org/apache/jena/query/text/TextDatasetFactory.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextDatasetFactory.java b/jena-text/src/main/java/org/apache/jena/query/text/TextDatasetFactory.java index 18be497..aad5e93 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/TextDatasetFactory.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/TextDatasetFactory.java @@ -41,23 +41,34 @@ public class TextDatasetFactory /** Create a text-indexed dataset */ public static Dataset create(Dataset base, TextIndex textIndex) { + return create(base, textIndex, false); + } + + /** Create a text-indexed dataset, optionally allowing the text index to be closed if the Dataset is */ + public static Dataset create(Dataset base, TextIndex textIndex, boolean closeIndexOnDSGClose) + { DatasetGraph dsg = base.asDatasetGraph() ; - dsg = create(dsg, textIndex) ; + dsg = create(dsg, textIndex, closeIndexOnDSGClose) ; return DatasetFactory.create(dsg) ; } - /** Create a text-indexed dataset */ + /** Create a text-indexed DatasetGraph */ public static DatasetGraph create(DatasetGraph dsg, TextIndex textIndex) { + return create(dsg, textIndex, false); + } + + /** Create a text-indexed DatasetGraph, optionally allowing the text index to be closed if the DatasetGraph is */ + public static DatasetGraph create(DatasetGraph dsg, TextIndex textIndex, boolean closeIndexOnDSGClose) + { TextDocProducer producer = new TextDocProducerTriples(textIndex.getDocDef(), textIndex) ; - DatasetGraph dsgt = new DatasetGraphText(dsg, textIndex, producer) ; + DatasetGraph dsgt = new DatasetGraphText(dsg, textIndex, producer, closeIndexOnDSGClose) ; // Also set on dsg Context c = dsgt.getContext() ; dsgt.getContext().set(TextQuery.textIndex, textIndex) ; return dsgt ; - } /** Create a Lucene TextIndex */ @@ -71,14 +82,14 @@ public class TextDatasetFactory public static Dataset createLucene(Dataset base, Directory directory, EntityDefinition entMap) { TextIndex index = createLuceneIndex(directory, entMap) ; - return create(base, index) ; + return create(base, index, true) ; } /** Create a text-indexed dataset, using Lucene */ public static DatasetGraph createLucene(DatasetGraph base, Directory directory, EntityDefinition entMap) { TextIndex index = createLuceneIndex(directory, entMap) ; - return create(base, index) ; + return create(base, index, true) ; } /** Create a Solr TextIndex */ @@ -92,14 +103,14 @@ public class TextDatasetFactory public static Dataset createSolrIndex(Dataset base, SolrServer server, EntityDefinition entMap) { TextIndex index = createSolrIndex(server, entMap) ; - return create(base, index) ; + return create(base, index, true) ; } /** Create a text-indexed dataset, using Solr */ public static DatasetGraph createSolrIndex(DatasetGraph base, SolrServer server, EntityDefinition entMap) { TextIndex index = createSolrIndex(server, entMap) ; - return create(base, index) ; + return create(base, index, true) ; } } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java index 3e7879f..47a804d 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java @@ -19,14 +19,21 @@ package org.apache.jena.query.text ; import java.io.IOException ; -import java.util.* ; +import java.util.ArrayList ; +import java.util.HashMap ; +import java.util.List ; +import java.util.Map ; import java.util.Map.Entry ; import org.apache.lucene.analysis.Analyzer ; import org.apache.lucene.analysis.core.KeywordAnalyzer ; import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper ; import org.apache.lucene.analysis.standard.StandardAnalyzer ; -import org.apache.lucene.document.* ; +import org.apache.lucene.document.Document ; +import org.apache.lucene.document.Field ; +import org.apache.lucene.document.FieldType ; +import org.apache.lucene.document.StringField ; +import org.apache.lucene.document.TextField ; import org.apache.lucene.index.DirectoryReader ; import org.apache.lucene.index.IndexReader ; import org.apache.lucene.index.IndexWriter ; @@ -67,9 +74,9 @@ public class TextIndexLucene implements TextIndex { private final EntityDefinition docDef ; private final Directory directory ; - private IndexWriter indexWriter ; - private Analyzer analyzer ; - + private final IndexWriter indexWriter ; + private final Analyzer analyzer ; + public TextIndexLucene(Directory directory, EntityDefinition def) { this.directory = directory ; this.docDef = def ; @@ -90,11 +97,15 @@ public class TextIndexLucene implements TextIndex { this.analyzer = new PerFieldAnalyzerWrapper(new StandardAnalyzer(VER), analyzerPerField) ; - // force creation of the index if it don't exist - // otherwise if we get a search before data is written we get an - // exception - startIndexing() ; - finishIndexing() ; + IndexWriterConfig wConfig = new IndexWriterConfig(VER, analyzer) ; + try + { + indexWriter = new IndexWriter(directory, wConfig) ; + } + catch (IOException e) + { + throw new TextIndexException(e) ; + } } public Directory getDirectory() { @@ -104,24 +115,18 @@ public class TextIndexLucene implements TextIndex { public Analyzer getAnalyzer() { return analyzer ; } + + public IndexWriter getIndexWriter() { + return indexWriter; + } @Override - public void startIndexing() { - try { - IndexWriterConfig wConfig = new IndexWriterConfig(VER, analyzer) ; - indexWriter = new IndexWriter(directory, wConfig) ; - } - catch (IOException e) { - exception(e) ; - } - } + public void startIndexing() { } @Override public void finishIndexing() { try { indexWriter.commit() ; - indexWriter.close() ; - indexWriter = null ; } catch (IOException e) { exception(e) ; @@ -140,13 +145,12 @@ public class TextIndexLucene implements TextIndex { @Override public void close() { - if ( indexWriter != null ) - try { - indexWriter.close() ; - } - catch (IOException ex) { - exception(ex) ; - } + try { + indexWriter.close() ; + } + catch (IOException ex) { + exception(ex) ; + } } @Override @@ -154,14 +158,8 @@ public class TextIndexLucene implements TextIndex { if ( log.isDebugEnabled() ) log.debug("Add entity: " + entity) ; try { - boolean autoBatch = (indexWriter == null) ; - Document doc = doc(entity) ; - if ( autoBatch ) - startIndexing() ; indexWriter.addDocument(doc) ; - if ( autoBatch ) - finishIndexing() ; } catch (IOException e) { exception(e) ; @@ -189,7 +187,7 @@ public class TextIndexLucene implements TextIndex { @Override public Map<String, Node> get(String uri) { try { - IndexReader indexReader = DirectoryReader.open(directory) ; + IndexReader indexReader = DirectoryReader.open(indexWriter, true); List<Map<String, Node>> x = get$(indexReader, uri) ; if ( x.size() == 0 ) return null ; @@ -250,7 +248,7 @@ public class TextIndexLucene implements TextIndex { @Override public List<Node> query(String qs, int limit) { //** score - try(IndexReader indexReader = DirectoryReader.open(directory)) { + try (IndexReader indexReader = DirectoryReader.open(indexWriter, true)) { return query$(indexReader, qs, limit) ; } catch (Exception ex) { http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextDatasetAssembler.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextDatasetAssembler.java b/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextDatasetAssembler.java index a379da7..c343824 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextDatasetAssembler.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextDatasetAssembler.java @@ -55,7 +55,7 @@ public class TextDatasetAssembler extends AssemblerBase implements Assembler Dataset ds = (Dataset)a.open(dataset) ; TextIndex textIndex = (TextIndex)a.open(index) ; - Dataset dst = TextDatasetFactory.create(ds, textIndex) ; + Dataset dst = TextDatasetFactory.create(ds, textIndex, true) ; return dst ; } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithGraphTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithGraphTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithGraphTextIndex.java index d08fe62..b6d7ae0 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithGraphTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithGraphTextIndex.java @@ -35,14 +35,19 @@ import com.hp.hpl.jena.rdf.model.Model ; /** * This abstract class defines tests of the graph-specific indexing. */ -public class AbstractTestDatasetWithGraphTextIndex extends AbstractTestDatasetWithTextIndex { +public abstract class AbstractTestDatasetWithGraphTextIndex extends AbstractTestDatasetWithTextIndex { private void putTurtleInModel(String turtle, String modelName) { Model model = modelName != null ? dataset.getNamedModel(modelName) : dataset.getDefaultModel() ; Reader reader = new StringReader(turtle) ; dataset.begin(ReadWrite.WRITE) ; - model.read(reader, "", "TURTLE") ; - dataset.commit() ; + try { + model.read(reader, "", "TURTLE") ; + dataset.commit() ; + } + finally { + dataset.end(); + } } @Test @@ -165,10 +170,15 @@ public class AbstractTestDatasetWithGraphTextIndex extends AbstractTestDatasetWi " rdfs:label 'bar testResult barfoo foo' .", "}" ); - StringReader reader = new StringReader(trig); - dataset.begin(ReadWrite.WRITE) ; - RDFDataMgr.read(dataset.asDatasetGraph(), reader, "", Lang.TRIG); - dataset.commit(); + StringReader reader = new StringReader(trig); + dataset.begin(ReadWrite.WRITE) ; + try { + RDFDataMgr.read(dataset.asDatasetGraph(), reader, "", Lang.TRIG); + dataset.commit(); + } + finally { + dataset.end(); + } String queryString = StrUtils.strjoinNL( QUERY_PROLOG, http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneGraphTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneGraphTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneGraphTextIndex.java index c237db6..7d78fba 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneGraphTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneGraphTextIndex.java @@ -20,6 +20,8 @@ package org.apache.jena.query.text; import org.apache.lucene.store.Directory ; import org.apache.lucene.store.RAMDirectory ; +import org.junit.After ; +import org.junit.Before ; import com.hp.hpl.jena.query.Dataset ; import com.hp.hpl.jena.tdb.TDBFactory ; @@ -31,13 +33,19 @@ import com.hp.hpl.jena.vocabulary.RDFS ; */ public class AbstractTestDatasetWithLuceneGraphTextIndex extends AbstractTestDatasetWithGraphTextIndex { - public static void init() { - Dataset ds1 = TDBFactory.createDataset() ; - Directory dir = new RAMDirectory() ; - EntityDefinition eDef = new EntityDefinition("iri", "text", "graph", RDFS.label.asNode()) ; - eDef.set("comment", RDFS.comment.asNode()) ; // some tests require indexing rdfs:comment - TextIndex tidx = new TextIndexLucene(dir, eDef) ; - dataset = TextDatasetFactory.create(ds1, tidx) ; + @Before + public void init() { + Dataset ds1 = TDBFactory.createDataset() ; + Directory dir = new RAMDirectory() ; + EntityDefinition eDef = new EntityDefinition("iri", "text", "graph", RDFS.label.asNode()) ; + eDef.set("comment", RDFS.comment.asNode()) ; // some tests require indexing rdfs:comment + TextIndex tidx = new TextIndexLucene(dir, eDef) ; + dataset = TextDatasetFactory.create(ds1, tidx) ; } + + @After + public void teardown() { + dataset.close(); + } } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneTextIndex.java deleted file mode 100644 index 6af0e2f..0000000 --- a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithLuceneTextIndex.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.query.text; - -import java.io.File; -import java.io.Reader; -import java.io.StringReader; - -import org.apache.jena.atlas.lib.StrUtils; -import org.apache.jena.query.text.assembler.TextAssembler; - -import com.hp.hpl.jena.assembler.Assembler; -import com.hp.hpl.jena.query.Dataset; -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.Resource; - -/** - * This abstract class defines a setup configuration for a dataset with a Lucene index. - */ -public class AbstractTestDatasetWithLuceneTextIndex extends AbstractTestDatasetWithTextIndex { - private static final String INDEX_PATH = "target/test/TestDatasetWithLuceneIndex"; - private static final File indexDir = new File(INDEX_PATH); - - private static final String SPEC_BASE = "http://example.org/spec#";; - private static final String SPEC_ROOT_LOCAL = "lucene_text_dataset"; - private static final String SPEC_ROOT_URI = SPEC_BASE + SPEC_ROOT_LOCAL; - private static final String SPEC; - static { - SPEC = StrUtils.strjoinNL( - "prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> ", - "prefix ja: <http://jena.hpl.hp.com/2005/11/Assembler#> ", - "prefix tdb: <http://jena.hpl.hp.com/2008/tdb#>", - "prefix text: <http://jena.apache.org/text#>", - "prefix : <" + SPEC_BASE + ">", - "", - "[] ja:loadClass \"org.apache.jena.query.text.TextQuery\" .", - "text:TextDataset rdfs:subClassOf ja:RDFDataset .", - "text:TextIndexLucene rdfs:subClassOf text:TextIndex .", - - ":" + SPEC_ROOT_LOCAL, - " a text:TextDataset ;", - " text:dataset :dataset ;", - " text:index :indexLucene ;", - " .", - "", - ":dataset", - " a ja:RDFDataset ;", - " ja:defaultGraph :graph ;", - ".", - ":graph", - " a ja:MemoryModel ;", - ".", - "", - ":indexLucene", - " a text:TextIndexLucene ;", - " text:directory <file:" + INDEX_PATH + "> ;", - " text:entityMap :entMap ;", - " .", - "", - ":entMap", - " a text:EntityMap ;", - " text:entityField \"uri\" ;", - " text:defaultField \"label\" ;", - " text:map (", - " [ text:field \"label\" ; text:predicate rdfs:label ]", - " [ text:field \"comment\" ; text:predicate rdfs:comment ]", - " ) ." - ); - } - - public static void init() { - Reader reader = new StringReader(SPEC); - Model specModel = ModelFactory.createDefaultModel(); - specModel.read(reader, "", "TURTLE"); - TextAssembler.init(); - deleteOldFiles(); - indexDir.mkdirs(); - Resource root = specModel.getResource(SPEC_ROOT_URI); - dataset = (Dataset) Assembler.general.open(root); - } - - - public static void deleteOldFiles() { - if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); - } -} http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndex.java index 67068da..3bcd5ff 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndex.java @@ -53,9 +53,9 @@ public abstract class AbstractTestDatasetWithTextIndex extends AbstractTestDatas doTestSearch(turtle, queryString, expectedURIs); } - static String R_S1 = RESOURCE_BASE + "s1" ; - static String R_S2 = RESOURCE_BASE + "s2" ; - static String PF_DATA = StrUtils.strjoinNL( + static final String R_S1 = RESOURCE_BASE + "s1" ; + static final String R_S2 = RESOURCE_BASE + "s2" ; + static final String PF_DATA = StrUtils.strjoinNL( TURTLE_PROLOG, "<" + R_S1 + "> rdfs:label 'text' .", "<" + R_S2 + "> rdfs:label 'fuzz' ." http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java index 88aa029..e0c7a06 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java @@ -44,7 +44,6 @@ import com.hp.hpl.jena.rdf.model.Model; */ public abstract class AbstractTestDatasetWithTextIndexBase { protected static final String RESOURCE_BASE = "http://example.org/data/resource/";; - protected static Dataset dataset; protected static final String QUERY_PROLOG = StrUtils.strjoinNL( "PREFIX text: <http://jena.apache.org/text#>", @@ -57,6 +56,8 @@ public abstract class AbstractTestDatasetWithTextIndexBase { "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> ." ); + protected Dataset dataset; + protected void doTestSearch(String turtle, String queryString, Set<String> expectedEntityURIs) { doTestSearch("", turtle, queryString, expectedEntityURIs); } @@ -76,8 +77,8 @@ public abstract class AbstractTestDatasetWithTextIndexBase { public static void doTestQuery(Dataset dataset, String label, String queryString, Set<String> expectedEntityURIs, int expectedNumResults) { Query query = QueryFactory.create(queryString) ; + dataset.begin(ReadWrite.READ); try(QueryExecution qexec = QueryExecutionFactory.create(query, dataset)) { - dataset.begin(ReadWrite.READ); ResultSet results = qexec.execSelect() ; assertEquals(label, expectedNumResults > 0, results.hasNext()); @@ -87,6 +88,9 @@ public abstract class AbstractTestDatasetWithTextIndexBase { assertTrue(label + ": unexpected result: " + entityURI, expectedEntityURIs.contains(entityURI)); } assertEquals(label, expectedNumResults, count); - } finally { dataset.end() ; } + } + finally { + dataset.end() ; + } } } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java b/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java index eca58e5..115b493 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java @@ -35,7 +35,6 @@ import org.junit.runners.Suite.SuiteClasses ; // Embedded solr not supported //, TestDatasetWithEmbeddedSolrTextIndex.class - , TestSearchBeforeWriteOnDatasetWithLuceneTextIndex.class , TestEntityMapAssembler.class , TestTextDatasetAssembler.class , TestTextIndexLuceneAssembler.class @@ -45,6 +44,7 @@ import org.junit.runners.Suite.SuiteClasses ; , TestDatasetWithStandardAnalyzer.class , TestDatasetWithKeywordAnalyzer.class , TestDatasetWithLowerCaseKeywordAnalyzer.class + , TestLuceneWithMultipleThreads.class }) public class TS_Text http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithEmbeddedSolrTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithEmbeddedSolrTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithEmbeddedSolrTextIndex.java index dac0f51..3e9502a 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithEmbeddedSolrTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithEmbeddedSolrTextIndex.java @@ -20,8 +20,8 @@ package org.apache.jena.query.text; import java.io.File ; -import org.junit.AfterClass ; -import org.junit.BeforeClass ; +import org.junit.After ; +import org.junit.Before ; public class TestDatasetWithEmbeddedSolrTextIndex extends AbstractTestDatasetWithTextIndex { @@ -31,21 +31,24 @@ public class TestDatasetWithEmbeddedSolrTextIndex extends AbstractTestDatasetWit private static final File INDEX_DIR = new File(INDEX_PATH); private static final String TEST_ASSEM = "testing/TextQuery/text-solr-config.ttl" ; - @BeforeClass public static void beforeClass() { - deleteOldFiles(); + @Before + public void before() { +// deleteOldFiles(); INDEX_DIR.mkdirs(); TextQuery.init() ; TextSearchUtil.createEmptyIndex(INDEX_DIR); dataset = TextDatasetFactory.create(TEST_ASSEM) ; } - @AfterClass public static void afterClass() { + @After + public void after() { TextIndexSolr index = (TextIndexSolr) dataset.getContext().get(TextQuery.textIndex) ; index.getServer().shutdown(); deleteOldFiles(); } - public static void deleteOldFiles() { + public void deleteOldFiles() { + dataset.close(); if (DATA_DIR.exists()) TextSearchUtil.emptyAndDeleteDirectory(DATA_DIR); } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithKeywordAnalyzer.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithKeywordAnalyzer.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithKeywordAnalyzer.java index f812695..bfb7fb6 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithKeywordAnalyzer.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithKeywordAnalyzer.java @@ -18,31 +18,28 @@ package org.apache.jena.query.text; -import java.io.File; -import java.io.Reader; -import java.io.StringReader; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.io.Reader ; +import java.io.StringReader ; +import java.util.Arrays ; +import java.util.HashSet ; +import java.util.Set ; -import org.apache.jena.atlas.lib.StrUtils; -import org.apache.jena.query.text.assembler.TextAssembler; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.jena.atlas.lib.StrUtils ; +import org.apache.jena.query.text.assembler.TextAssembler ; +import org.junit.After ; +import org.junit.Before ; +import org.junit.Test ; -import com.hp.hpl.jena.assembler.Assembler; -import com.hp.hpl.jena.query.Dataset; -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.Resource; +import com.hp.hpl.jena.assembler.Assembler ; +import com.hp.hpl.jena.query.Dataset ; +import com.hp.hpl.jena.rdf.model.Model ; +import com.hp.hpl.jena.rdf.model.ModelFactory ; +import com.hp.hpl.jena.rdf.model.Resource ; /** * This class defines a setup configuration for a dataset that uses a keyword analyzer with a Lucene index. */ public class TestDatasetWithKeywordAnalyzer extends AbstractTestDatasetWithTextIndexBase { - private static final String INDEX_PATH = "target/test/TestDatasetWithLuceneIndex"; - private static final File indexDir = new File(INDEX_PATH); private static final String SPEC_BASE = "http://example.org/spec#";; private static final String SPEC_ROOT_LOCAL = "lucene_text_dataset"; @@ -76,7 +73,7 @@ public class TestDatasetWithKeywordAnalyzer extends AbstractTestDatasetWithTextI "", ":indexLucene", " a text:TextIndexLucene ;", - " text:directory <file:" + INDEX_PATH + "> ;", + " text:directory \"mem\" ;", " text:entityMap :entMap ;", " .", "", @@ -94,28 +91,24 @@ public class TestDatasetWithKeywordAnalyzer extends AbstractTestDatasetWithTextI ); } - public static void init(String analyzer) { + public void init(String analyzer) { Reader reader = new StringReader(makeSpec(analyzer)); Model specModel = ModelFactory.createDefaultModel(); specModel.read(reader, "", "TURTLE"); TextAssembler.init(); - deleteOldFiles(); - indexDir.mkdirs(); Resource root = specModel.getResource(SPEC_ROOT_URI); dataset = (Dataset) Assembler.general.open(root); } - public static void deleteOldFiles() { - if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); - } - - @BeforeClass public static void beforeClass() { + @Before + public void before() { init("text:KeywordAnalyzer"); - } + } - @AfterClass public static void afterClass() { - deleteOldFiles(); + @After + public void after() { + dataset.close(); } @Test http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLowerCaseKeywordAnalyzer.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLowerCaseKeywordAnalyzer.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLowerCaseKeywordAnalyzer.java index 4409670..56a8916 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLowerCaseKeywordAnalyzer.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLowerCaseKeywordAnalyzer.java @@ -18,23 +18,23 @@ package org.apache.jena.query.text; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.Arrays ; +import java.util.HashSet ; +import java.util.Set ; -import org.apache.jena.atlas.lib.StrUtils; - -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.jena.atlas.lib.StrUtils ; +import org.junit.Before ; +import org.junit.Test ; /** * This class defines a setup configuration for a dataset that uses a lowercase keyword analyzer with a Lucene index. */ public class TestDatasetWithLowerCaseKeywordAnalyzer extends TestDatasetWithKeywordAnalyzer { - @BeforeClass public static void beforeClass() { + @Before + public void before() { init("text:LowerCaseKeywordAnalyzer"); } - + @Test public void testLowerCaseKeywordAnalyzerIsCaseInsensitive() { final String testName = "testLowerCaseKeywordAnalyzerIsCaseInsensitive"; http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneGraphTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneGraphTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneGraphTextIndex.java index b8d3fe9..47e0eec 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneGraphTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneGraphTextIndex.java @@ -18,11 +18,12 @@ package org.apache.jena.query.text; -import org.junit.BeforeClass ; +import org.junit.Before ; public class TestDatasetWithLuceneGraphTextIndex extends AbstractTestDatasetWithLuceneGraphTextIndex { - @BeforeClass public static void beforeClass() { + @Before + public void before() { init(); } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndex.java index a3bd6df..cc55550 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndex.java @@ -18,16 +18,96 @@ package org.apache.jena.query.text; -import org.junit.AfterClass ; -import org.junit.BeforeClass ; +import java.io.Reader ; +import java.io.StringReader ; +import java.util.HashSet ; -public class TestDatasetWithLuceneTextIndex extends AbstractTestDatasetWithLuceneTextIndex { - - @BeforeClass public static void beforeClass() { - init(); - } +import org.apache.jena.atlas.lib.StrUtils ; +import org.apache.jena.query.text.assembler.TextAssembler ; +import org.junit.After ; +import org.junit.Before ; +import org.junit.Test ; + +import com.hp.hpl.jena.assembler.Assembler ; +import com.hp.hpl.jena.query.Dataset ; +import com.hp.hpl.jena.rdf.model.Model ; +import com.hp.hpl.jena.rdf.model.ModelFactory ; +import com.hp.hpl.jena.rdf.model.Resource ; + +public class TestDatasetWithLuceneTextIndex extends AbstractTestDatasetWithTextIndex { - @AfterClass public static void afterClass() { - deleteOldFiles(); - } + private static final String SPEC_BASE = "http://example.org/spec#";; + private static final String SPEC_ROOT_LOCAL = "lucene_text_dataset"; + private static final String SPEC_ROOT_URI = SPEC_BASE + SPEC_ROOT_LOCAL; + private static final String SPEC; + static { + SPEC = StrUtils.strjoinNL( + "prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> ", + "prefix ja: <http://jena.hpl.hp.com/2005/11/Assembler#> ", + "prefix tdb: <http://jena.hpl.hp.com/2008/tdb#>", + "prefix text: <http://jena.apache.org/text#>", + "prefix : <" + SPEC_BASE + ">", + "", + "[] ja:loadClass \"org.apache.jena.query.text.TextQuery\" .", + "text:TextDataset rdfs:subClassOf ja:RDFDataset .", + "text:TextIndexLucene rdfs:subClassOf text:TextIndex .", + + ":" + SPEC_ROOT_LOCAL, + " a text:TextDataset ;", + " text:dataset :dataset ;", + " text:index :indexLucene ;", + " .", + "", + ":dataset", + " a ja:RDFDataset ;", + " ja:defaultGraph :graph ;", + ".", + ":graph", + " a ja:MemoryModel ;", + ".", + "", + ":indexLucene", + " a text:TextIndexLucene ;", + " text:directory \"mem\" ;", + " text:entityMap :entMap ;", + " .", + "", + ":entMap", + " a text:EntityMap ;", + " text:entityField \"uri\" ;", + " text:defaultField \"label\" ;", + " text:map (", + " [ text:field \"label\" ; text:predicate rdfs:label ]", + " [ text:field \"comment\" ; text:predicate rdfs:comment ]", + " ) ." + ); + } + + @Before + public void before() { + Reader reader = new StringReader(SPEC); + Model specModel = ModelFactory.createDefaultModel(); + specModel.read(reader, "", "TURTLE"); + TextAssembler.init(); + Resource root = specModel.getResource(SPEC_ROOT_URI); + dataset = (Dataset) Assembler.general.open(root); + } + + @After + public void after() { + dataset.close(); + } + + @Test + public void testNoResultsOnFirstCreateIndex(){ + String turtle = ""; + String queryString = StrUtils.strjoinNL( + QUERY_PROLOG, + "SELECT ?s", + "WHERE {", + " ?s text:query ( rdfs:label \"a* OR b* OR c* OR d* OR e* OR f* OR g* OR h* OR i* OR j* OR k* OR l* OR m* OR n* OR o* OR p* OR q* OR r* OR s* OR t* OR u* OR v* OR w* OR x* OR y* OR z*\" 10 ) .", + "}" + ); + doTestSearch(turtle, queryString, new HashSet<String>()); + } } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithSimpleAnalyzer.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithSimpleAnalyzer.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithSimpleAnalyzer.java index 37bae49..7701713 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithSimpleAnalyzer.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithSimpleAnalyzer.java @@ -18,24 +18,24 @@ package org.apache.jena.query.text; -import java.io.File; -import java.io.Reader; -import java.io.StringReader; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.io.File ; +import java.io.Reader ; +import java.io.StringReader ; +import java.util.Arrays ; +import java.util.HashSet ; +import java.util.Set ; -import org.apache.jena.atlas.lib.StrUtils; -import org.apache.jena.query.text.assembler.TextAssembler; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.jena.atlas.lib.StrUtils ; +import org.apache.jena.query.text.assembler.TextAssembler ; +import org.junit.After ; +import org.junit.Before ; +import org.junit.Test ; -import com.hp.hpl.jena.assembler.Assembler; -import com.hp.hpl.jena.query.Dataset; -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.Resource; +import com.hp.hpl.jena.assembler.Assembler ; +import com.hp.hpl.jena.query.Dataset ; +import com.hp.hpl.jena.rdf.model.Model ; +import com.hp.hpl.jena.rdf.model.ModelFactory ; +import com.hp.hpl.jena.rdf.model.Resource ; /** * This class defines a setup configuration for a dataset that uses a simple analyzer with a Lucene index. @@ -94,27 +94,30 @@ public class TestDatasetWithSimpleAnalyzer extends AbstractTestDatasetWithTextIn ); } - public static void init() { + public void init() { Reader reader = new StringReader(SPEC); Model specModel = ModelFactory.createDefaultModel(); specModel.read(reader, "", "TURTLE"); TextAssembler.init(); - deleteOldFiles(); +// deleteOldFiles(); indexDir.mkdirs(); Resource root = specModel.getResource(SPEC_ROOT_URI); dataset = (Dataset) Assembler.general.open(root); } - public static void deleteOldFiles() { + public void deleteOldFiles() { + dataset.close(); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); } - @BeforeClass public static void beforeClass() { + @Before + public void beforeClass() { init(); } - @AfterClass public static void afterClass() { + @After + public void afterClass() { deleteOldFiles(); } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithStandardAnalyzer.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithStandardAnalyzer.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithStandardAnalyzer.java index 812bf44..d873171 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithStandardAnalyzer.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithStandardAnalyzer.java @@ -18,24 +18,24 @@ package org.apache.jena.query.text; -import java.io.File; -import java.io.Reader; -import java.io.StringReader; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.io.File ; +import java.io.Reader ; +import java.io.StringReader ; +import java.util.Arrays ; +import java.util.HashSet ; +import java.util.Set ; -import org.apache.jena.atlas.lib.StrUtils; -import org.apache.jena.query.text.assembler.TextAssembler; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.jena.atlas.lib.StrUtils ; +import org.apache.jena.query.text.assembler.TextAssembler ; +import org.junit.After ; +import org.junit.Before ; +import org.junit.Test ; -import com.hp.hpl.jena.assembler.Assembler; -import com.hp.hpl.jena.query.Dataset; -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.Resource; +import com.hp.hpl.jena.assembler.Assembler ; +import com.hp.hpl.jena.query.Dataset ; +import com.hp.hpl.jena.rdf.model.Model ; +import com.hp.hpl.jena.rdf.model.ModelFactory ; +import com.hp.hpl.jena.rdf.model.Resource ; /** * This class defines a setup configuration for a dataset that uses a standard analyzer with a Lucene index. @@ -94,27 +94,30 @@ public class TestDatasetWithStandardAnalyzer extends AbstractTestDatasetWithText ); } - public static void init() { + public void init() { Reader reader = new StringReader(SPEC); Model specModel = ModelFactory.createDefaultModel(); specModel.read(reader, "", "TURTLE"); TextAssembler.init(); - deleteOldFiles(); +// deleteOldFiles(); indexDir.mkdirs(); Resource root = specModel.getResource(SPEC_ROOT_URI); dataset = (Dataset) Assembler.general.open(root); } - public static void deleteOldFiles() { + public void deleteOldFiles() { + dataset.close(); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); } - @BeforeClass public static void beforeClass() { + @Before + public void beforeClass() { init(); } - @AfterClass public static void afterClass() { + @After + public void afterClass() { deleteOldFiles(); } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestLuceneWithMultipleThreads.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestLuceneWithMultipleThreads.java b/jena-text/src/test/java/org/apache/jena/query/text/TestLuceneWithMultipleThreads.java new file mode 100644 index 0000000..4788596 --- /dev/null +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestLuceneWithMultipleThreads.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.query.text; + +import java.util.concurrent.ExecutionException ; +import java.util.concurrent.ExecutorService ; +import java.util.concurrent.Executors ; +import java.util.concurrent.Future ; +import java.util.concurrent.TimeUnit ; + +import org.apache.lucene.analysis.standard.StandardAnalyzer ; +import org.apache.lucene.store.RAMDirectory ; +import org.apache.lucene.util.Version ; +import org.junit.Before ; +import org.junit.Test ; + +import com.hp.hpl.jena.query.Dataset ; +import com.hp.hpl.jena.query.DatasetFactory ; +import com.hp.hpl.jena.query.QueryExecution ; +import com.hp.hpl.jena.query.QueryExecutionFactory ; +import com.hp.hpl.jena.query.ReadWrite ; +import com.hp.hpl.jena.query.ResultSet ; +import com.hp.hpl.jena.rdf.model.Model ; +import com.hp.hpl.jena.rdf.model.ResourceFactory ; +import com.hp.hpl.jena.sparql.core.DatasetGraph ; +import com.hp.hpl.jena.sparql.core.Transactional ; +import com.hp.hpl.jena.sparql.modify.GraphStoreNullTransactional ; +import com.hp.hpl.jena.vocabulary.RDFS ; + +import static org.junit.Assert.* ; + +/** + * Spin up multiple threads against a multiple-reader/single-writer Dataset to test that the Lucene index handles concurrency properly. + */ +public class TestLuceneWithMultipleThreads +{ + private static final EntityDefinition entDef; + + static { + entDef = new EntityDefinition("uri", "label", "graph", RDFS.label.asNode()); + StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_46); + entDef.setAnalyzer("label", analyzer); + } + + private DatasetGraph dsg; + private Transactional tx; + + @Before + public void setup() + { + dsg = TextDatasetFactory.createLucene(new GraphStoreNullTransactional(), new RAMDirectory(), entDef); + tx = (Transactional)dsg; + } + + + @Test + public void testReadInMiddleOfWrite() throws InterruptedException, ExecutionException + { + final Dataset ds = DatasetFactory.create(dsg); + final ExecutorService execService = Executors.newSingleThreadExecutor(); + final Future<?> f = execService.submit(new Runnable() + { + @Override + public void run() + { + // Hammer the dataset with a series of read queries + while (!Thread.interrupted()) + { + tx.begin(ReadWrite.READ); + try + { + QueryExecution qExec = QueryExecutionFactory.create("select * where { ?s ?p ?o }", ds); + ResultSet rs = qExec.execSelect(); + while (rs.hasNext()) + { + rs.next(); + } + tx.commit(); + } + finally + { + tx.end(); + } + } + } + }); + + tx.begin(ReadWrite.WRITE); + try + { + Model m = ds.getDefaultModel(); + m.add(ResourceFactory.createResource("http://example.org/";), RDFS.label, "entity"); + // Sleep for a bit so that the reader thread can get in between these two writes + Thread.sleep(100); + m.add(ResourceFactory.createResource("http://example.org/";), RDFS.comment, "comment"); + + tx.commit(); + } + finally + { + tx.end(); + } + + execService.shutdownNow(); + execService.awaitTermination(1000, TimeUnit.MILLISECONDS); + + // If there was an exception in the read thread then Future.get() will throw an ExecutionException + assertTrue(f.get() == null); + } + + @Test + public void testWriteInMiddleOfRead() throws InterruptedException, ExecutionException + { + final int numReads = 10; + final Dataset ds = DatasetFactory.create(dsg); + final ExecutorService execService = Executors.newFixedThreadPool(10); //.newSingleThreadExecutor(); + final Future<?> f = execService.submit(new Runnable() + { + @Override + public void run() + { + while (!Thread.interrupted()) + { + tx.begin(ReadWrite.WRITE); + try + { + Model m = ds.getDefaultModel(); + m.add(ResourceFactory.createResource("http://example.org/";), RDFS.label, "entity"); + // Sleep for a bit so that the reader thread can get in between these two writes + try + { + Thread.sleep(100); + } + catch (InterruptedException e) + { + break; + } + m.add(ResourceFactory.createResource("http://example.org/";), RDFS.comment, "comment"); + + tx.commit(); + } + finally + { + tx.end(); + } + } + } + }); + + for (int i=0; i<numReads; i++) + { + tx.begin(ReadWrite.READ); + try + { + QueryExecution qExec = QueryExecutionFactory.create("select * where { ?s ?p ?o }", ds); + ResultSet rs = qExec.execSelect(); + while (rs.hasNext()) + { + rs.next(); + } + // Sleep for a bit so that the writer thread can get in between the reads + Thread.sleep(100); + tx.commit(); + } + finally + { + tx.end(); + } + } + + execService.shutdownNow(); + execService.awaitTermination(1000, TimeUnit.MILLISECONDS); + + // If there was an exception in the write thread then Future.get() will throw an ExecutionException + assertTrue(f.get() == null); + } +} http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/TestSearchBeforeWriteOnDatasetWithLuceneTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestSearchBeforeWriteOnDatasetWithLuceneTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/TestSearchBeforeWriteOnDatasetWithLuceneTextIndex.java deleted file mode 100644 index df02b1b..0000000 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestSearchBeforeWriteOnDatasetWithLuceneTextIndex.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.jena.query.text; - -import java.util.HashSet; - -import org.apache.jena.atlas.lib.StrUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestSearchBeforeWriteOnDatasetWithLuceneTextIndex extends AbstractTestDatasetWithLuceneTextIndex { - - @BeforeClass public static void beforeClass() { - init(); - } - - @AfterClass public static void afterClass() { - deleteOldFiles(); - } - - @Test public void testNoResultsOnFirstCreateIndex(){ - String turtle = ""; - String queryString = StrUtils.strjoinNL( - QUERY_PROLOG, - "SELECT ?s", - "WHERE {", - " ?s text:query ( rdfs:label \"a* OR b* OR c* OR d* OR e* OR f* OR g* OR h* OR i* OR j* OR k* OR l* OR m* OR n* OR o* OR p* OR q* OR r* OR s* OR t* OR u* OR v* OR w* OR x* OR y* OR z*\" 10 ) .", - "}" - ); - doTestSearch(turtle, queryString, new HashSet<String>()); - } -} http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/assembler/AbstractTestTextAssembler.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/assembler/AbstractTestTextAssembler.java b/jena-text/src/test/java/org/apache/jena/query/text/assembler/AbstractTestTextAssembler.java index e48f4ba..49d029e 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/assembler/AbstractTestTextAssembler.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/assembler/AbstractTestTextAssembler.java @@ -18,19 +18,29 @@ package org.apache.jena.query.text.assembler; -import com.hp.hpl.jena.rdf.model.Model; -import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.RDFNode; -import com.hp.hpl.jena.rdf.model.Resource; -import com.hp.hpl.jena.tdb.assembler.VocabTDB; -import com.hp.hpl.jena.vocabulary.RDF; -import com.hp.hpl.jena.vocabulary.RDFS; +import java.io.File ; + +import org.apache.jena.query.text.TextSearchUtil ; +import org.junit.After ; +import org.junit.Before ; + +import com.hp.hpl.jena.rdf.model.Model ; +import com.hp.hpl.jena.rdf.model.ModelFactory ; +import com.hp.hpl.jena.rdf.model.RDFNode ; +import com.hp.hpl.jena.rdf.model.Resource ; +import com.hp.hpl.jena.tdb.assembler.VocabTDB ; +import com.hp.hpl.jena.vocabulary.RDF ; +import com.hp.hpl.jena.vocabulary.RDFS ; public abstract class AbstractTestTextAssembler { + protected static final Model model = ModelFactory.createDefaultModel(); private static final String TESTBASE = "http://example.org/abstractTestTextAssembler/";; protected static final Resource SIMPLE_DATASET_SPEC; protected static final Resource SIMPLE_INDEX_SPEC; + protected static final Resource SIMPLE_INDEX_SPEC2; + protected static final Resource SIMPLE_INDEX_SPEC3; + protected static final Resource SIMPLE_INDEX_SPEC4; protected static final Resource SIMPLE_ENTITY_MAP_SPEC; protected static final Resource SIMPLE_INDEX_SPEC_LITERAL_DIR; protected static final Resource SIMPLE_INDEX_SPEC_MEM_DIR; @@ -52,18 +62,36 @@ public abstract class AbstractTestTextAssembler { SIMPLE_DATASET_SPEC = model.createResource(TESTBASE + "simpleDatasetSpec") .addProperty(RDF.type, VocabTDB.tDatasetTDB) - .addProperty(VocabTDB.pLocation, "target/test/DB"); + .addProperty(VocabTDB.pLocation, "target/test/testasm/DB"); SIMPLE_INDEX_SPEC = model.createResource(TESTBASE + "simpleIndexSpec") .addProperty(RDF.type, TextVocab.textIndexLucene) - .addProperty(TextVocab.pDirectory, model.createResource("file:target/test/simpleLuceneIndex")) + .addProperty(TextVocab.pDirectory, model.createResource("file:target/test/testasm/simpleIndexSpec")) .addProperty(TextVocab.pEntityMap, SIMPLE_ENTITY_MAP_SPEC); + + SIMPLE_INDEX_SPEC2 = + model.createResource(TESTBASE + "simpleIndexSpec2") + .addProperty(RDF.type, TextVocab.textIndexLucene) + .addProperty(TextVocab.pDirectory, model.createResource("file:target/test/testasm/simpleIndexSpec2")) + .addProperty(TextVocab.pEntityMap, SIMPLE_ENTITY_MAP_SPEC); + + SIMPLE_INDEX_SPEC4 = + model.createResource(TESTBASE + "simpleIndexSpec3") + .addProperty(RDF.type, TextVocab.textIndexLucene) + .addProperty(TextVocab.pDirectory, model.createResource("file:target/test/testasm/simpleIndexSpec3")) + .addProperty(TextVocab.pEntityMap, SIMPLE_ENTITY_MAP_SPEC); + + SIMPLE_INDEX_SPEC3 = + model.createResource(TESTBASE + "simpleIndexSpec4") + .addProperty(RDF.type, TextVocab.textIndexLucene) + .addProperty(TextVocab.pDirectory, model.createResource("file:target/test/testasm/simpleIndexSpec4")) + .addProperty(TextVocab.pEntityMap, SIMPLE_ENTITY_MAP_SPEC); SIMPLE_INDEX_SPEC_LITERAL_DIR = model.createResource(TESTBASE + "simpleIndexLiteralDirSpec") .addProperty(RDF.type, TextVocab.textIndexLucene) - .addProperty(TextVocab.pDirectory, model.createLiteral("target/test/simpleLuceneIndex")) + .addProperty(TextVocab.pDirectory, model.createLiteral("target/test/testasm/simpleIndexLiteralDir")) .addProperty(TextVocab.pEntityMap, SIMPLE_ENTITY_MAP_SPEC); SIMPLE_INDEX_SPEC_MEM_DIR = @@ -72,5 +100,27 @@ public abstract class AbstractTestTextAssembler { .addProperty(TextVocab.pDirectory, model.createLiteral("mem")) .addProperty(TextVocab.pEntityMap, SIMPLE_ENTITY_MAP_SPEC); } + + protected void deleteFiles() { + File indexDir; + indexDir = new File("target/test/testasm/DB"); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); + indexDir = new File("target/test/testasm/simpleIndexSpec"); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); + indexDir = new File("target/test/testasm/simpleIndexSpec2"); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); + indexDir = new File("target/test/testasm/simpleIndexSpec3"); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); + indexDir = new File("target/test/testasm/simpleIndexSpec4"); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); + indexDir = new File("target/test/testasm/simpleIndexLiteralDir"); if (indexDir.exists()) TextSearchUtil.emptyAndDeleteDirectory(indexDir); + } + + @Before + public void before() { + deleteFiles(); + + TextSearchUtil.createEmptyIndex(new File("target/test/testasm/DB")); + } + + @After + public void after() { + deleteFiles(); + } } http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextDatasetAssembler.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextDatasetAssembler.java b/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextDatasetAssembler.java index 51d5085..cdc9f54 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextDatasetAssembler.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextDatasetAssembler.java @@ -68,11 +68,11 @@ public class TestTextDatasetAssembler extends AbstractTestTextAssembler { model.createResource(TESTBASE + "spec1") .addProperty(RDF.type, TextVocab.textDataset) .addProperty(TextVocab.pDataset, SIMPLE_DATASET_SPEC) - .addProperty(TextVocab.pIndex, SIMPLE_INDEX_SPEC); + .addProperty(TextVocab.pIndex, SIMPLE_INDEX_SPEC3); noDatasetPropertySpec = model.createResource(TESTBASE + "noDatasetPropertySpec") .addProperty(RDF.type, TextVocab.textDataset) - .addProperty(TextVocab.pIndex, SIMPLE_INDEX_SPEC); + .addProperty(TextVocab.pIndex, SIMPLE_INDEX_SPEC4); noIndexPropertySpec = model.createResource(TESTBASE + "noIndexPropertySpec") .addProperty(RDF.type, TextVocab.textDataset) http://git-wip-us.apache.org/repos/asf/jena/blob/4ee8d9e3/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextIndexLuceneAssembler.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextIndexLuceneAssembler.java b/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextIndexLuceneAssembler.java index 1ef7cea..ffaf202 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextIndexLuceneAssembler.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/assembler/TestTextIndexLuceneAssembler.java @@ -18,25 +18,26 @@ package org.apache.jena.query.text.assembler; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import org.apache.jena.query.text.TextIndexLucene ; +import org.apache.lucene.store.RAMDirectory ; +import org.junit.Test ; -import org.apache.jena.query.text.TextIndex; -import org.apache.jena.query.text.TextIndexLucene; -import org.apache.lucene.store.RAMDirectory; -import org.junit.Test; +import com.hp.hpl.jena.assembler.Assembler ; +import com.hp.hpl.jena.rdf.model.Resource ; +import com.hp.hpl.jena.vocabulary.RDFS ; -import com.hp.hpl.jena.assembler.Assembler; -import com.hp.hpl.jena.rdf.model.Resource; -import com.hp.hpl.jena.vocabulary.RDFS; +import static org.junit.Assert.* ; public class TestTextIndexLuceneAssembler extends AbstractTestTextAssembler { @Test public void testIndexHasEntityMap() { - TextIndexLucene indexLucene = (TextIndexLucene) Assembler.general.open(SIMPLE_INDEX_SPEC); - assertEquals(RDFS.label.asNode(), indexLucene.getDocDef().getPrimaryPredicate()); + TextIndexLucene index = (TextIndexLucene) Assembler.general.open(SIMPLE_INDEX_SPEC); + try { + assertEquals(RDFS.label.asNode(), index.getDocDef().getPrimaryPredicate()); + } + finally { + index.close(); + } } @Test public void testLiteralDirectory() { @@ -46,19 +47,29 @@ public class TestTextIndexLuceneAssembler extends AbstractTestTextAssembler { Assembler a = Assembler.general; // the open method is not supposed to throw exceptions when the directory is // a literal - TextIndex index = assembler.open(a, root, /*mode*/ null); - assertNotNull(index); + TextIndexLucene index = (TextIndexLucene)assembler.open(a, root, /*mode*/ null); + try { + assertNotNull(index); + } + finally { + index.close(); + } } @Test public void testResourceDirectory() { TextIndexLuceneAssembler assembler = new TextIndexLuceneAssembler(); - Resource root = SIMPLE_INDEX_SPEC; + Resource root = SIMPLE_INDEX_SPEC2; Assembler a = Assembler.general; // the open method is not supposed to throw exceptions when the directory is // a resource TextIndexLucene index = (TextIndexLucene) assembler.open(a, root, /*mode*/ null); - assertFalse(index.getDirectory() instanceof RAMDirectory); + try { + assertFalse(index.getDirectory() instanceof RAMDirectory); + } + finally { + index.close(); + } } @Test public void testMemDirectory() { @@ -69,7 +80,12 @@ public class TestTextIndexLuceneAssembler extends AbstractTestTextAssembler { // the open method is not supposed to throw exceptions when the directory is // a iri resource TextIndexLucene index = (TextIndexLucene) assembler.open(a, root, /*mode*/ null); - assertTrue(index.getDirectory() instanceof RAMDirectory); + try { + assertTrue(index.getDirectory() instanceof RAMDirectory); + } + finally { + index.close(); + } } static {
