fix head conflict + adding tests
Project: http://git-wip-us.apache.org/repos/asf/jena/repo Commit: http://git-wip-us.apache.org/repos/asf/jena/commit/8e9babe7 Tree: http://git-wip-us.apache.org/repos/asf/jena/tree/8e9babe7 Diff: http://git-wip-us.apache.org/repos/asf/jena/diff/8e9babe7 Branch: refs/heads/master Commit: 8e9babe75ccfa9024035bb19f8257ad0dbd5018e Parents: c7d099d Author: Alexis Miara <[email protected]> Authored: Mon Jun 1 09:51:27 2015 -0400 Committer: Alexis Miara <[email protected]> Committed: Mon Jun 1 09:51:27 2015 -0400 ---------------------------------------------------------------------- .../jena/query/text/EntityDefinition.java | 9 ++ .../jena/query/text/TextDocProducerTriples.java | 24 +++- .../org/apache/jena/query/text/TextIndex.java | 1 + .../apache/jena/query/text/TextIndexLucene.java | 28 +++- .../apache/jena/query/text/TextIndexSolr.java | 8 +- .../assembler/EntityDefinitionAssembler.java | 5 + .../jena/query/text/assembler/TextVocab.java | 1 + .../AbstractTestDatasetWithTextIndexBase.java | 43 ++++++- .../org/apache/jena/query/text/TS_Text.java | 1 + ...tDatasetWithLuceneMultilingualTextIndex.java | 2 +- ...DatasetWithLuceneTextIndexWithLangField.java | 2 +- ...tDatasetWithLuceneTextIndexWithUidField.java | 129 +++++++++++++++++++ 12 files changed, 241 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java b/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java index 2a68247..69cca8c 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/EntityDefinition.java @@ -43,6 +43,7 @@ public class EntityDefinition { private final String primaryField ; private String graphField = null ; private String langField ; + private String uidField ; /** * @param entityField @@ -171,6 +172,14 @@ public class EntityDefinition { this.langField = langField; } + public String getUidField() { + return uidField; + } + + public void setUidField(String uidField) { + this.uidField = uidField; + } + public Collection<String> fields() { return fields ; } http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/TextDocProducerTriples.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextDocProducerTriples.java b/jena-text/src/main/java/org/apache/jena/query/text/TextDocProducerTriples.java index c0bcabd..51f5480 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/TextDocProducerTriples.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/TextDocProducerTriples.java @@ -56,17 +56,29 @@ public class TextDocProducerTriples implements TextDocProducer { public void change(QuadAction qaction, Node g, Node s, Node p, Node o) { // One document per triple/quad - if ( qaction != QuadAction.ADD ) + if ( qaction != QuadAction.ADD && + qaction != QuadAction.DELETE ) return ; + Entity entity = TextQueryFuncs.entityFromQuad(defn, g, s, p, o) ; // Null means does not match defn if ( entity != null ) { - indexer.addEntity(entity) ; - - // Auto commit the entity if we aren't in a transaction - if (!inTransaction.get()) { - indexer.commit() ; + if (qaction == QuadAction.ADD) { + indexer.addEntity(entity); + + // Auto commit the entity if we aren't in a transaction + if (!inTransaction.get()) { + indexer.commit(); + } + } + else if (qaction == QuadAction.DELETE) { + indexer.deleteEntity(entity); + + // Auto commit the entity if we aren't in a transaction + if (!inTransaction.get()) { + indexer.commit(); + } } } } http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/TextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextIndex.java b/jena-text/src/main/java/org/apache/jena/query/text/TextIndex.java index fe19b7b..bb67cfd 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/TextIndex.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/TextIndex.java @@ -36,6 +36,7 @@ public interface TextIndex extends Closeable //, Transactional // Update operations void addEntity(Entity entity) ; void updateEntity(Entity entity) ; + void deleteEntity(Entity entity) ; // read operations http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java index 9c14c27..36d9e86 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexLucene.java @@ -22,6 +22,7 @@ import java.io.IOException ; import java.util.* ; import java.util.Map.Entry ; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.jena.graph.Node ; import org.apache.jena.graph.NodeFactory ; import org.apache.jena.sparql.util.NodeFactoryExtra ; @@ -211,6 +212,26 @@ public class TextIndexLucene implements TextIndex { indexWriter.addDocument(doc) ; } + @Override + public void deleteEntity(Entity entity) { + if (docDef.getUidField() == null) + return; + + if ( log.isDebugEnabled() ) + log.debug("Delete entity: "+entity) ; + try { + Map<String, Object> map = entity.getMap(); + String property = map.keySet().iterator().next(); + String value = (String)map.get(property); + String key = entity.getGraph() + "-" + entity.getId() + "-" + property + "-" + value + "-" + entity.getLanguage(); + Term uid = new Term(docDef.getUidField(), DigestUtils.sha256Hex(key)); + indexWriter.deleteDocuments(uid); + + } catch (Exception e) { + throw new TextIndexException(e) ; + } + } + protected Document doc(Entity entity) { Document doc = new Document() ; Field entField = new Field(docDef.getEntityField(), entity.getId(), ftIRI) ; @@ -223,13 +244,18 @@ public class TextIndexLucene implements TextIndex { } String langField = docDef.getLangField() ; + String uidField = docDef.getUidField() ; for ( Entry<String, Object> e : entity.getMap().entrySet() ) { doc.add( new Field(e.getKey(), (String) e.getValue(), ftText) ); if (langField != null) { String lang = entity.getLanguage(); if (lang != null && !"".equals(lang)) - doc.add(new Field(docDef.getLangField(), lang, StringField.TYPE_STORED)); + doc.add(new Field(langField, lang, StringField.TYPE_STORED)); + } + if (uidField != null) { + String key = entity.getGraph() + "-" + entity.getId() + "-" + e.getKey() + "-" + e.getValue() + "-" + entity.getLanguage(); + doc.add(new Field(uidField, DigestUtils.sha256Hex(key), StringField.TYPE_STORED)); } } return doc ; http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/TextIndexSolr.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexSolr.java b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexSolr.java index 718d4c7..de011bf 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/TextIndexSolr.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/TextIndexSolr.java @@ -104,7 +104,13 @@ public class TextIndexSolr implements TextIndex } } - private SolrInputDocument solrDoc(Entity entity) { + @Override + public void deleteEntity(Entity entity) { + //to be implemented + } + + private SolrInputDocument solrDoc(Entity entity) + { SolrInputDocument doc = new SolrInputDocument() ; doc.addField(docDef.getEntityField(), entity.getId()) ; http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java b/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java index 5d9f97c..48bd94d 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/assembler/EntityDefinitionAssembler.java @@ -69,6 +69,9 @@ public class EntityDefinitionAssembler extends AssemblerBase implements Assemble " OPTIONAL {" , " ?eMap :langField ?langField" , " }", + " OPTIONAL {" , + " ?eMap :uidField ?uidField" , + " }", "}") ; ParameterizedSparqlString pss = new ParameterizedSparqlString(qs1) ; pss.setIri("eMap", root.getURI()) ; @@ -92,6 +95,7 @@ public class EntityDefinitionAssembler extends AssemblerBase implements Assemble String graphField = qsol1.contains("graphField") ? qsol1.getLiteral("graphField").getLexicalForm() : null; String langField = qsol1.contains("langField") ? qsol1.getLiteral("langField").getLexicalForm() : null; String defaultField = qsol1.contains("dftField") ? qsol1.getLiteral("dftField").getLexicalForm() : null ; + String uniqueIdField = qsol1.contains("uidField") ? qsol1.getLiteral("uidField").getLexicalForm() : null; Multimap<String, Node> mapDefs = HashMultimap.create() ; Map<String, Analyzer> analyzerDefs = new HashMap<>(); @@ -162,6 +166,7 @@ public class EntityDefinitionAssembler extends AssemblerBase implements Assemble EntityDefinition docDef = new EntityDefinition(entityField, defaultField); docDef.setGraphField(graphField); docDef.setLangField(langField); + docDef.setUidField(uniqueIdField); for ( String f : mapDefs.keys() ) { for ( Node p : mapDefs.get(f)) docDef.set(f, p) ; http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java ---------------------------------------------------------------------- diff --git a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java b/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java index 802990d..743d773 100644 --- a/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java +++ b/jena-text/src/main/java/org/apache/jena/query/text/assembler/TextVocab.java @@ -49,6 +49,7 @@ public class TextVocab public static final Property pDefaultField = Vocab.property(NS, "defaultField") ; public static final Property pGraphField = Vocab.property(NS, "graphField") ; public static final Property pLangField = Vocab.property(NS, "langField") ; + public static final Property pUidField = Vocab.property(NS, "uidField") ; public static final Property pMap = Vocab.property(NS, "map") ; public static final Property pField = Vocab.property(NS, "field") ; public static final Property pPredicate = Vocab.property(NS, "predicate") ; http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java index b51651f..94f947c 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/AbstractTestDatasetWithTextIndexBase.java @@ -19,6 +19,7 @@ package org.apache.jena.query.text; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.Reader; @@ -28,6 +29,10 @@ import java.util.Set; import org.apache.jena.atlas.lib.StrUtils; import org.apache.jena.query.* ; import org.apache.jena.rdf.model.Model ; +import org.apache.jena.update.UpdateExecutionFactory; +import org.apache.jena.update.UpdateFactory; +import org.apache.jena.update.UpdateProcessor; +import org.apache.jena.update.UpdateRequest; /* * This abstract class defines a collection of test methods for testing @@ -60,12 +65,25 @@ public abstract class AbstractTestDatasetWithTextIndexBase { } protected void doTestSearch(String label, String turtle, String queryString, Set<String> expectedEntityURIs, int expectedNumResults) { + loadData(turtle); + doTestQuery(dataset, label, queryString, expectedEntityURIs, expectedNumResults); + } + + protected void doTestSearchNoResult(String turtle, String queryString) { + doTestSearchNoResult("", turtle, queryString); + } + + protected void doTestSearchNoResult(String label, String turtle, String queryString) { + loadData(turtle); + doTestNoResult(dataset, label, queryString); + } + + protected void loadData(String turtle) { Model model = dataset.getDefaultModel(); Reader reader = new StringReader(turtle); dataset.begin(ReadWrite.WRITE); model.read(reader, "", "TURTLE"); dataset.commit(); - doTestQuery(dataset, label, queryString, expectedEntityURIs, expectedNumResults); } public static void doTestQuery(Dataset dataset, String label, String queryString, Set<String> expectedEntityURIs, int expectedNumResults) { @@ -73,7 +91,7 @@ public abstract class AbstractTestDatasetWithTextIndexBase { dataset.begin(ReadWrite.READ); try(QueryExecution qexec = QueryExecutionFactory.create(query, dataset)) { ResultSet results = qexec.execSelect() ; - + assertEquals(label, expectedNumResults > 0, results.hasNext()); int count; for (count=0; results.hasNext(); count++) { @@ -86,4 +104,25 @@ public abstract class AbstractTestDatasetWithTextIndexBase { dataset.end() ; } } + + public static void doTestNoResult(Dataset dataset, String label, String queryString) { + Query query = QueryFactory.create(queryString) ; + dataset.begin(ReadWrite.READ); + try(QueryExecution qexec = QueryExecutionFactory.create(query, dataset)) { + ResultSet results = qexec.execSelect() ; + assertFalse(label, results.hasNext()); + } + finally { + dataset.end() ; + } + } + + protected void doUpdate(String updateString) { + dataset.begin(ReadWrite.WRITE); + UpdateRequest request = UpdateFactory.create(updateString); + UpdateProcessor proc = UpdateExecutionFactory.create(request, dataset); + proc.execute(); + dataset.commit(); + } + } http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java b/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java index 6d1cb25..471b129 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TS_Text.java @@ -32,6 +32,7 @@ import org.junit.runners.Suite.SuiteClasses ; , TestDatasetWithLuceneTextIndex.class , TestDatasetWithLuceneMultilingualTextIndex.class , TestDatasetWithLuceneTextIndexWithLangField.class + , TestDatasetWithLuceneTextIndexWithUidField.class , TestDatasetWithLuceneGraphTextIndex.class // Embedded solr not supported http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneMultilingualTextIndex.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneMultilingualTextIndex.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneMultilingualTextIndex.java index 53e2426..18acda8 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneMultilingualTextIndex.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneMultilingualTextIndex.java @@ -37,7 +37,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; -public class TestDatasetWithLuceneMultilingualTextIndex extends AbstractTestDatasetWithTextIndex { +public class TestDatasetWithLuceneMultilingualTextIndex extends AbstractTestDatasetWithTextIndexBase { private static final String SPEC_BASE = "http://example.org/spec#";; private static final String SPEC_ROOT_LOCAL = "lucene_text_dataset"; http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithLangField.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithLangField.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithLangField.java index 9d99a29..f1e7e08 100644 --- a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithLangField.java +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithLangField.java @@ -35,7 +35,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Set; -public class TestDatasetWithLuceneTextIndexWithLangField extends AbstractTestDatasetWithTextIndex { +public class TestDatasetWithLuceneTextIndexWithLangField extends AbstractTestDatasetWithTextIndexBase { private static final String SPEC_BASE = "http://example.org/spec#";; private static final String SPEC_ROOT_LOCAL = "lucene_text_dataset"; http://git-wip-us.apache.org/repos/asf/jena/blob/8e9babe7/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithUidField.java ---------------------------------------------------------------------- diff --git a/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithUidField.java b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithUidField.java new file mode 100644 index 0000000..9431449 --- /dev/null +++ b/jena-text/src/test/java/org/apache/jena/query/text/TestDatasetWithLuceneTextIndexWithUidField.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.query.text; + +import org.apache.jena.assembler.Assembler; +import org.apache.jena.atlas.lib.StrUtils; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.text.assembler.TextAssembler; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Resource; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.Reader; +import java.io.StringReader; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class TestDatasetWithLuceneTextIndexWithUidField extends AbstractTestDatasetWithTextIndexBase { + + private static final String SPEC_BASE = "http://example.org/spec#";; + private static final String SPEC_ROOT_LOCAL = "lucene_text_dataset"; + private static final String SPEC_ROOT_URI = SPEC_BASE + SPEC_ROOT_LOCAL; + private static final String SPEC; + static { + SPEC = StrUtils.strjoinNL( + "prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> ", + "prefix ja: <http://jena.hpl.hp.com/2005/11/Assembler#> ", + "prefix tdb: <http://jena.hpl.hp.com/2008/tdb#>", + "prefix text: <http://jena.apache.org/text#>", + "prefix : <" + SPEC_BASE + ">", + "", + "[] ja:loadClass \"org.apache.jena.query.text.TextQuery\" .", + "text:TextDataset rdfs:subClassOf ja:RDFDataset .", + "text:TextIndexLucene rdfs:subClassOf text:TextIndex .", + + ":" + SPEC_ROOT_LOCAL, + " a text:TextDataset ;", + " text:dataset :dataset ;", + " text:index :indexLucene ;", + " .", + "", + ":dataset", + " a ja:RDFDataset ;", + " ja:defaultGraph :graph ;", + ".", + ":graph", + " a ja:MemoryModel ;", + ".", + "", + ":indexLucene", + " a text:TextIndexLucene ;", + " text:directory \"mem\" ;", + " text:entityMap :entMap ;", + " .", + "", + ":entMap", + " a text:EntityMap ;", + " text:entityField \"uri\" ;", + " text:defaultField \"label\" ;", + " text:uidField \"uuid\" ;", + " text:map (", + " [ text:field \"label\" ; text:predicate rdfs:label ]", + " [ text:field \"comment\" ; text:predicate rdfs:comment ]", + " ) ." + ); + } + + @Before + public void before() { + Reader reader = new StringReader(SPEC); + Model specModel = ModelFactory.createDefaultModel(); + specModel.read(reader, "", "TURTLE"); + TextAssembler.init(); + Resource root = specModel.getResource(SPEC_ROOT_URI); + dataset = (Dataset) Assembler.general.open(root); + } + + @After + public void after() { + dataset.close(); + } + + @Test + public void testDeleteTriple(){ + final String turtle = StrUtils.strjoinNL( + TURTLE_PROLOG, + "<" + RESOURCE_BASE + "myResource>", + " rdfs:label 'My first resource'", + "." + ); + loadData(turtle); + String updateString = StrUtils.strjoinNL( + QUERY_PROLOG, + "DELETE DATA { ", + " <" + RESOURCE_BASE + "myResource> rdfs:label 'My first resource'", + "}" + ); + doUpdate(updateString); + + String queryString = StrUtils.strjoinNL( + QUERY_PROLOG, + "SELECT ?s", + "WHERE {", + " ?s text:query ( rdfs:label 'first' ) .", + "}" + ); + doTestNoResult(dataset, "", queryString); + } +}
