Smalyshev has uploaded a new change for review.
https://gerrit.wikimedia.org/r/204225
Change subject: WIP: change how values/refs are handled
......................................................................
WIP: change how values/refs are handled
Change-Id: I0397dc10fb7fbaf0d11433c31728ed8e28f58d84
---
M tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
A tools/src/main/java/org/wikidata/query/rdf/tool/rdf/CleanUnused.sync.sparql
A tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetRefs.sync.sparql
A tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetValues.sync.sparql
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.sync.sparql
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
8 files changed, 185 insertions(+), 89 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/25/204225/1
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
index 012ebab..5f82268 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
@@ -251,9 +251,15 @@
log.debug("RDF repository already has this revision, skipping.");
return;
}
- Collection<Statement> statements =
wikibase.fetchRdfForEntity(change.entityId());
- munger.munge(change.entityId(), statements);
- rdfRepository.sync(change.entityId(), statements);
+ String entityId = change.entityId();
+ Collection<Statement> statements =
wikibase.fetchRdfForEntity(entityId);
+ Collection<String> values = rdfRepository.getValues(entityId);
+ Collection<String> refs = rdfRepository.getRefs(entityId);
+ munger.munge(entityId, statements, values, refs);
+ rdfRepository.sync(entityId, statements);
+ List<String> cleanupList = new ArrayList<String>(values);
+ cleanupList.addAll(refs);
+ rdfRepository.cleanUnused(cleanupList);
updateMeter.mark();
}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/CleanUnused.sync.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/CleanUnused.sync.sparql
new file mode 100644
index 0000000..edecc15
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/CleanUnused.sync.sparql
@@ -0,0 +1,7 @@
+DELETE { ?s ?p ?o } WHERE {
+ VALUES ?s { %values% }
+ # Since values are shared we can only clear the values on them when they are
no longer used
+ # anywhere else.
+ FILTER NOT EXISTS { ?someEntity ?someStatementPred ?s . }
+ ?s ?p ?o .
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetRefs.sync.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetRefs.sync.sparql
new file mode 100644
index 0000000..cfb03b6
--- /dev/null
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetRefs.sync.sparql
@@ -0,0 +1,6 @@
+SELECT DISTINCT ?s
+WHERE {
+ %entity:id% ?statementPred ?statement .
+ FILTER( STRSTARTS(STR(?statement), "%uris.statement%") ) .
+ ?statement %prov:wasDerivedFrom% ?s .
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetValues.sync.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetValues.sync.sparql
new file mode 100644
index 0000000..48ea9f9
--- /dev/null
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/GetValues.sync.sparql
@@ -0,0 +1,13 @@
+SELECT DISTINCT ?s
+WHERE {
+ <http://www.wikidata.org/entity/Q30> ?statementPred ?statement .
+ FILTER( STRSTARTS(STR(?statement),
"http://www.wikidata.org/entity/statement/";) ) .
+ ?statement <http://www.w3.org/ns/prov#wasDerivedFrom> ?s .
+ # Since references are shared we can only clear the values on them when they
are no longer used
+ # anywhere else.
+ FILTER NOT EXISTS {
+ ?otherStatement <http://www.w3.org/ns/prov#wasDerivedFrom> ?s .
+ ?otherEntity ?otherStatementPred ?otherStatement .
+ FILTER ( <http://www.wikidata.org/entity/Q23> != ?otherEntity ) .
+ }
+ ?s ?p ?o .
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
index fcf573d..5f9a32a 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
@@ -49,6 +49,14 @@
* True if we should remove site links or false if we shouldn't.
*/
private final boolean removeSiteLinks;
+ /**
+ * List of values connected to this entity
+ */
+ private Collection<String> valuesList;
+ /**
+ * List of refs connected to this entity
+ */
+ private Collection<String> refsList;
public Munger(WikibaseUris uris) {
this(uris, null, null, false);
@@ -112,15 +120,41 @@
* RDF exports into a more queryable form.
*
* @param statements statements to munge
+ * @param existingValues Existing value statements
+ * @param existingRefs Existing reference statements
*/
- public void munge(String entityId, Collection<Statement> statements)
throws ContainedException {
+ public void munge(String entityId, Collection<Statement> statements,
+ Collection<String> existingValues, Collection<String> existingRefs)
throws ContainedException {
+ if(existingValues == null) {
+ existingValues = new HashSet<String>();
+ }
+ if(existingRefs == null) {
+ existingRefs = new HashSet<String>();
+ }
+
if (statements.isEmpty()) {
// Empty collection is a delete.
return;
}
- MungeOperation op = new MungeOperation(entityId, statements);
+ MungeOperation op = new MungeOperation(entityId, statements,
existingValues, existingRefs);
op.munge();
+ // remove all values that we have seen as they are used by statements
+ existingValues.removeAll(op.extraValidSubjects);
+ existingRefs.removeAll(op.extraValidSubjects);
return;
+ }
+
+ public void munge(String entityId, Collection<Statement> statements) {
+ munge(entityId, statements, null, null);
+ }
+
+
+ public Collection<String> getValuesList() {
+ return valuesList;
+ }
+
+ public Collection<String> getRefsList() {
+ return refsList;
}
/**
@@ -174,7 +208,10 @@
private String subject;
private String predicate;
- public MungeOperation(String entityId, Collection<Statement>
statements) {
+ private final Collection<String> existingValues;
+ private final Collection<String> existingRefs;
+
+ public MungeOperation(String entityId, Collection<Statement>
statements, Collection<String> existingValues, Collection<String> existingRefs)
{
this.statements = statements;
entityUri = uris.entity() + entityId;
entityUriImpl = new URIImpl(entityUri);
@@ -185,6 +222,8 @@
singleLabelModeWorkForLabel = null;
singleLabelModeWorkForDescription = null;
}
+ this.existingValues = existingValues;
+ this.existingRefs = existingRefs;
}
public void munge() throws ContainedException {
@@ -348,17 +387,23 @@
* @return true to keep the statement, false to remove it
*/
private boolean entityReferenceStatement() throws ContainedException {
+ if(existingRefs.contains(subject)) {
+ /* We already have this ref, so no need to import it again
+ * Since refs are IDed by content, we know it is the same
+ */
+ return false;
+ }
switch (predicate) {
- case RDF.TYPE:
- /*
- * We don't need r:<uuid> a ontology:Reference because its
super
- * common and not super interesting.
- */
- if
(statement.getObject().stringValue().equals(Ontology.REFERENCE)) {
- return false;
- }
- break;
- default:
+ case RDF.TYPE:
+ /*
+ * We don't need r:<uuid> a ontology:Reference because its
super
+ * common and not super interesting.
+ */
+ if
(statement.getObject().stringValue().equals(Ontology.REFERENCE)) {
+ return false;
+ }
+ break;
+ default:
}
if (!extraValidSubjects.contains(subject)) {
/*
@@ -383,6 +428,12 @@
* @return true to keep the statement, false to remove it
*/
private boolean entityValueStatement() throws ContainedException {
+ if(existingValues.contains(subject)) {
+ /* We already have this value, so no need to import it again
+ * Since values are IDed by content, we know it is the same
+ */
+ return false;
+ }
switch (predicate) {
case RDF.TYPE:
/*
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
index ab85823..dd1b31d 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
@@ -11,6 +11,7 @@
import java.util.Collection;
import java.util.Date;
import java.util.GregorianCalendar;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
@@ -59,17 +60,86 @@
.build();
private final URI uri;
private final WikibaseUris uris;
+ // SPARQL queries
private final String syncBody;
+ private final String getValues;
+ private final String getRefs;
+ private final String cleanUnused;
+
+ protected String loadSparql(String filename) {
+ URL bodyUrl = getResource(RdfRepository.class, filename);
+ String body = null;
+ try {
+ body = Resources.toString(bodyUrl, Charsets.UTF_8);
+ } catch (IOException e) {
+ throw new FatalException("Can't load " + bodyUrl);
+ }
+ return body;
+ }
public RdfRepository(URI uri, WikibaseUris uris) {
this.uri = uri;
this.uris = uris;
- URL syncBodyUrl = getResource(RdfRepository.class,
"RdfRepository.sync.sparql");
+ syncBody = loadSparql("RdfRepository.sync.sparql");
+ getRefs = loadSparql("GetRefs.sync.sparql");
+ getValues = loadSparql("GetValues.sync.sparql");
+ cleanUnused = loadSparql("CleanUnused.sync.sparql");
+ }
+
+ /**
+ * Collect results of the query into string set
+ * @param TupleQueryResult result Result object
+ * @param String binding Binding name to collect
+ * @return Collection<String>
+ */
+ private Collection<String> resultToCollection(TupleQueryResult result,
String binding) {
+ Collection<String> values = new HashSet<String>();
try {
- syncBody = Resources.toString(syncBodyUrl, Charsets.UTF_8);
- } catch (IOException e) {
- throw new FatalException("Can't load " + syncBodyUrl);
+ while(result.hasNext()) {
+ Binding value = result.next().getBinding(binding);
+ if (value == null) {
+ continue;
+ }
+ values.add(value.getValue().stringValue());
+ }
+ } catch(QueryEvaluationException e) {
+ return values;
}
+ return values;
+ }
+
+ public Collection<String> getValues(String entityId) {
+ UpdateBuilder b = new UpdateBuilder(getValues);
+ b.bindUri("entity:id", uris.entity() + entityId);
+ b.bind("uris.value", uris.value());
+ b.bind("uris.statement", uris.statement());
+ b.bindUri("prov:wasDerivedFrom", Provenance.WAS_DERIVED_FROM);
+
+ return resultToCollection(query(b.toString()), "s");
+ }
+
+ public Collection<String> getRefs(String entityId) {
+ UpdateBuilder b = new UpdateBuilder(getRefs);
+ b.bindUri("entity:id", uris.entity() + entityId);
+ b.bind("uris.statement", uris.statement());
+ b.bindUri("prov:wasDerivedFrom", Provenance.WAS_DERIVED_FROM);
+
+ return resultToCollection(query(b.toString()), "s");
+ }
+
+ /**
+ * Clean subjects if they are not used anymore
+ * @param valueList
+ */
+ public void cleanUnused(Collection<String> valueList) {
+ if(valueList.isEmpty()) {
+ return;
+ }
+ long start = System.currentTimeMillis();
+ UpdateBuilder b = new UpdateBuilder(cleanUnused);
+ b.bindUris("values", valueList);
+ int modified = execute("update", UPDATE_COUNT_RESPONSE, b.toString());
+ log.debug("Cleanup {} millis and modified {} statements",
System.currentTimeMillis() - start, modified);
}
/**
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.sync.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.sync.sparql
index c30cb0b..a94d8c6 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.sync.sparql
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.sync.sparql
@@ -13,73 +13,7 @@
}
}
};
-# Clear statements on expanded values in referenes that are no longer used
-DELETE {
- ?s ?p ?o .
-}
-WHERE {
- %entity:id% ?statementPred ?statement .
- FILTER( STRSTARTS(STR(?statement), "%uris.statement%") ) .
- ?statement %prov:wasDerivedFrom% ?ref .
- # Since references are shared we can only clear the values on them when they
are no longer used
- # anywhere else.
- FILTER NOT EXISTS {
- ?otherStatement %prov:wasDerivedFrom% ?ref .
- ?otherEntity ?otherStatementPred ?otherStatement .
- FILTER ( %entity:id% != ?otherEntity ) .
- }
- ?ref ?expandedValuePred ?s .
- # Without this filter we'd try to delete stuff from entities. For example
that pattern above matches
- # ref:_ v:P143 entity:Q328
- # so we'd try to clear everything from Q328 (enwiki). So we filter where ?s
is in the value prefix.
- FILTER( STRSTARTS(STR(?s), "%uris.value%") ) .
- ?s ?p ?o .
- FILTER NOT EXISTS {
- VALUES ( ?s ?p ?o ) {
- %valueStatements%
- }
- }
-};
-# Clear statements about references that are no longer used
-DELETE {
- ?s ?p ?o .
-}
-WHERE {
- %entity:id% ?statementPred ?statement .
- FILTER( STRSTARTS(STR(?statement), "%uris.statement%") ) .
- ?statement %prov:wasDerivedFrom% ?s .
- # Since references are shared we can only clear the values on them when they
are no longer used
- # anywhere else.
- FILTER NOT EXISTS {
- ?otherStatement %prov:wasDerivedFrom% ?s .
- ?otherEntity ?otherStatementPred ?otherStatement .
- FILTER ( %entity:id% != ?otherEntity ) .
- }
- ?s ?p ?o .
- FILTER NOT EXISTS {
- VALUES ( ?s ?p ?o ) {
- %valueStatements%
- }
- }
-};
-# Clear out of date expanded values on statements about the entity
-DELETE {
- ?s ?p ?o .
-}
-WHERE {
- %entity:id% ?statementPred ?statement .
- FILTER( STRSTARTS(STR(?statement), "%uris.statement%") ) .
- ?statement ?expandedValuePred ?s .
- # Without this filter we'd clear all kinds of things. Only try and clear
value nodes.
- FILTER( STRSTARTS(STR(?s), "%uris.value%") ) .
- ?s ?p ?o .
- FILTER NOT EXISTS {
- VALUES ( ?s ?p ?o ) {
- %valueStatements%
- }
- }
-};
-# Clear out of date statements about statements
+# Clear out of date statements about statements
DELETE {
?s ?p ?o .
}
@@ -107,6 +41,5 @@
};
INSERT {
%insertStatements%
-}
-WHERE {};
+} WHERE {}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
index 345c211..a52e639 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
@@ -49,6 +49,16 @@
return this;
}
+ public UpdateBuilder bindUris(String from, Collection<String> uris) {
+ StringBuilder b = new StringBuilder(uris.size() * 80);
+
+ for (String s : uris) {
+ b.append('<').append(s).append("> ");
+ }
+ bind(from, b.toString().trim());
+ return this;
+ }
+
@Override
public String toString() {
return template;
--
To view, visit https://gerrit.wikimedia.org/r/204225
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0397dc10fb7fbaf0d11433c31728ed8e28f58d84
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
