jenkins-bot has submitted this change and it was merged.
Change subject: Batch SPARQL queries
......................................................................
Batch SPARQL queries
Bug: T110708
Change-Id: I9c0727b8275e58db7559c51332af07b3a1b1adc5
---
M
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
M
blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/constraints/WikibaseDateBOp.java
M tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
M
tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.CleanUnused.sparql
M
tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRefs.sparql
A
tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRevisions.sparql
M
tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetValues.sparql
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
M
tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
M
tools/src/test/java/org/wikidata/query/rdf/tool/AbstractUpdateIntegrationTestBase.java
11 files changed, 234 insertions(+), 56 deletions(-)
Approvals:
Smalyshev: Looks good to me, approved
jenkins-bot: Verified
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
index d212716..c070075 100644
---
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseContextListener.java
@@ -31,7 +31,7 @@
*/
public class WikibaseContextListener extends BigdataRDFServletContextListener {
- private static final transient Logger log =
LoggerFactory.getLogger(WikibaseContextListener.class);
+ private static final Logger log =
LoggerFactory.getLogger(WikibaseContextListener.class);
/**
* Replaces the default Blazegraph services with ones that do not allow
diff --git
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/constraints/WikibaseDateBOp.java
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/constraints/WikibaseDateBOp.java
index 7ac76d7..8a5c0b2 100644
---
a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/constraints/WikibaseDateBOp.java
+++
b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/constraints/WikibaseDateBOp.java
@@ -31,8 +31,8 @@
public class WikibaseDateBOp extends IVValueExpression<IV> implements
INeedsMaterialization{
/**
- *
- */
+ *
+ */
private static final long serialVersionUID = 9136864442064392445L;
/**
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
index 9e72129..83b60d7 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/Update.java
@@ -11,9 +11,13 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -42,6 +46,7 @@
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.collect.Multimap;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.lexicalscope.jewel.cli.Option;
@@ -79,7 +84,7 @@
@Option(shortName = "t", defaultValue = "10", description = "Thread
count")
int threadCount();
- @Option(shortName = "b", defaultValue = "10", description = "Number of
recent changes fetched at a time.")
+ @Option(shortName = "b", defaultValue = "100", description = "Number
of recent changes fetched at a time.")
int batchSize();
}
@@ -109,7 +114,7 @@
new LinkedBlockingQueue<Runnable>(), threadFactory.build());
Munger munger = mungerFromOptions(options);
- new Update<>(changeSource, wikibaseRepository, rdfRepository, munger,
executor, options.pollDelay()).run();
+ new Update<>(changeSource, wikibaseRepository, rdfRepository, munger,
executor, options.pollDelay(), uris).run();
}
/**
@@ -228,15 +233,28 @@
* particular this will happen if the RecentChangesPoller finds no changes.
*/
private final int pollDelay;
+ /**
+ * Uris for wikibase.
+ */
+ private final WikibaseUris uris;
+ /**
+ * Map entity->values list from repository.
+ */
+ private Multimap<String, String> repoValues;
+ /**
+ * Map entity->references list from repository.
+ */
+ private Multimap<String, String> repoRefs;
public Update(Change.Source<B> changeSource, WikibaseRepository wikibase,
RdfRepository rdfRepository,
- Munger munger, ExecutorService executor, int pollDelay) {
+ Munger munger, ExecutorService executor, int pollDelay,
WikibaseUris uris) {
this.changeSource = changeSource;
this.wikibase = wikibase;
this.rdfRepository = rdfRepository;
this.munger = munger;
this.executor = executor;
this.pollDelay = pollDelay;
+ this.uris = uris;
reporter.start();
}
@@ -290,28 +308,76 @@
* changes
*/
private void handleChanges(Change.Batch batch) throws
InterruptedException, ExecutionException {
- List<Future<?>> tasks = new ArrayList<>();
- for (final Change change : batch.changes()) {
- tasks.add(executor.submit(new Runnable() {
+ List<Future<String>> tasks = new ArrayList<>();
+ Set<Change> trueChanges = getRevisionUpdates(batch);
+ for (final Change change : trueChanges) {
+ tasks.add(executor.submit(new Callable<String>() {
@Override
- public void run() {
+ public String call() {
while (true) {
try {
- handleChange(change);
- return;
+ return handleChange(change);
} catch (RetryableException e) {
log.warn("Retryable error syncing. Retrying.", e);
} catch (ContainedException e) {
log.warn("Contained error syncing. Giving up on "
+ change.entityId(), e);
- return;
+ return null;
}
}
}
}));
}
- for (Future<?> task : tasks) {
- task.get();
+ StringBuilder bigQuery = new StringBuilder();
+ for (Future<String> task : tasks) {
+ String query = task.get();
+ if (query != null) {
+ bigQuery.append(query);
+ }
}
+ if (bigQuery.length() > 0) {
+ rdfRepository.syncQuery(bigQuery.toString());
+ }
+ updateMeter.mark(trueChanges.size());
+ }
+
+ /**
+ * Filter change by revisions.
+ * The revisions that have the same or superior revision in the DB will be
removed.
+ * @param batch
+ * @return A set of changes that need to be entered into the repository.
+ */
+ private Set<Change> getRevisionUpdates(Change.Batch batch) {
+ // List of changes that indeed need update
+ Set<Change> trueChanges = new HashSet<>();
+ // List of entity URIs that were changed
+ Set<String> changeIds = new HashSet<>();
+ Map<String, Change> candidateChanges = new HashMap<>();
+ for (final Change change : batch.changes()) {
+ if (change.revision() >= 0) {
+ Change c = candidateChanges.get(change.entityId());
+ if (c == null || c.revision() < change.revision()) {
+ candidateChanges.put(change.entityId(), change);
+ }
+ } else {
+ trueChanges.add(change);
+ changeIds.add(uris.entity() + change.entityId());
+ }
+ }
+ if (candidateChanges.size() > 0) {
+ for (String entityId:
rdfRepository.hasRevisions(candidateChanges.values())) {
+ // Cut off the entity prefix from the resulting URI
+ changeIds.add(entityId);
+
trueChanges.add(candidateChanges.get(entityId.substring(uris.entity().length())));
+ }
+ }
+ log.debug("Filtered batch contains {} changes", trueChanges.size());
+
+ repoValues = rdfRepository.getValues(changeIds);
+ log.debug("Fetched {} values", repoValues.size());
+ repoRefs = rdfRepository.getRefs(changeIds);
+ log.debug("Fetched {} refs", repoRefs.size());
+
+ return trueChanges;
}
/**
@@ -350,21 +416,16 @@
* @throws RetryableException if there is a retryable error updating the
rdf
* store
*/
- private void handleChange(Change change) throws RetryableException {
- log.debug("Received revision information {}", change);
- if (change.revision() >= 0 &&
rdfRepository.hasRevision(change.entityId(), change.revision())) {
- log.debug("RDF repository already has this revision, skipping.");
- return;
- }
+ private String handleChange(Change change) throws RetryableException {
+ log.debug("Processing data for {}", change);
Collection<Statement> statements =
wikibase.fetchRdfForEntity(change.entityId());
- Set<String> values = rdfRepository.getValues(change.entityId());
- Set<String> refs = rdfRepository.getRefs(change.entityId());
+ Set<String> values = new HashSet<>(repoValues.get(change.entityId()));
+ Set<String> refs = new HashSet<>(repoRefs.get(change.entityId()));
munger.munge(change.entityId(), statements, values, refs, change);
List<String> cleanupList = new ArrayList<>();
cleanupList.addAll(values);
cleanupList.addAll(refs);
- rdfRepository.sync(change.entityId(), statements, cleanupList);
- updateMeter.mark();
+ return rdfRepository.getSyncQuery(change.entityId(), statements,
cleanupList);
}
/**
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.CleanUnused.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.CleanUnused.sparql
index edecc15..e43b56b 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.CleanUnused.sparql
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.CleanUnused.sparql
@@ -4,4 +4,4 @@
# anywhere else.
FILTER NOT EXISTS { ?someEntity ?someStatementPred ?s . }
?s ?p ?o .
-}
+};
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRefs.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRefs.sparql
index cfb03b6..905fbbf 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRefs.sparql
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRefs.sparql
@@ -1,6 +1,9 @@
-SELECT DISTINCT ?s
+SELECT DISTINCT ?entity ?s
WHERE {
- %entity:id% ?statementPred ?statement .
+ VALUES ?entity {
+ %entityList%
+ }
+ ?entity ?statementPred ?statement .
FILTER( STRSTARTS(STR(?statement), "%uris.statement%") ) .
?statement %prov:wasDerivedFrom% ?s .
}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRevisions.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRevisions.sparql
new file mode 100644
index 0000000..8343246
--- /dev/null
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetRevisions.sparql
@@ -0,0 +1,9 @@
+SELECT DISTINCT ?s WHERE {
+ VALUES (?s ?rev) {
+ %values%
+ }
+ OPTIONAL {
+ ?s %schema:about% ?repoRev
+ }
+ FILTER (!bound(?repoRev) || ?repoRev < ?rev)
+}
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetValues.sparql
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetValues.sparql
index 55cac7b..b52d34d 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetValues.sparql
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.GetValues.sparql
@@ -1,6 +1,9 @@
-SELECT DISTINCT ?s
+SELECT DISTINCT ?entity ?s
WHERE {
- %entity:id% ?statementPred ?statement .
+ VALUES ?entity {
+ %entityList%
+ }
+ ?entity ?statementPred ?statement .
FILTER( STRSTARTS(STR(?statement), "%uris.statement%") ) .
{ ?statement %prov:wasDerivedFrom% ?ref .
?ref ?expandedValuePred ?s .
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
index e2574da..f158298 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
@@ -40,6 +40,7 @@
import org.openrdf.model.Literal;
import org.openrdf.model.Statement;
import org.openrdf.query.Binding;
+import org.openrdf.query.BindingSet;
import org.openrdf.query.QueryEvaluationException;
import org.openrdf.query.QueryResultHandlerException;
import org.openrdf.query.TupleQueryResult;
@@ -52,10 +53,13 @@
import org.wikidata.query.rdf.common.uri.Provenance;
import org.wikidata.query.rdf.common.uri.SchemaDotOrg;
import org.wikidata.query.rdf.common.uri.WikibaseUris;
+import org.wikidata.query.rdf.tool.change.Change;
import org.wikidata.query.rdf.tool.exception.ContainedException;
import org.wikidata.query.rdf.tool.exception.FatalException;
import com.google.common.base.Charsets;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
import com.google.common.io.CharStreams;
import com.google.common.io.Resources;
@@ -105,6 +109,11 @@
* Sparql to sync the left off time.
*/
private final String updateLeftOffTimeBody;
+ /**
+ * Sparql to filter entities for newer revisions.
+ */
+ private final String getRevisions;
+
/**
* How many times we retry a failed HTTP call.
@@ -131,6 +140,7 @@
getValues = loadBody("GetValues");
getRefs = loadBody("GetRefs");
cleanUnused = loadBody("CleanUnused");
+ getRevisions = loadBody("GetRevisions");
}
/**
@@ -210,54 +220,74 @@
}
/**
+ * Collect results of the query into a multimap by first parameter.
+ *
+ * @param result Result object
+ * @param keyBinding Binding name to serve as key
+ * @param valueBinding Binding name to serve as values
+ * @return Collection of strings resulting from the query.
+ */
+ private Multimap<String, String> resultToMap(TupleQueryResult result,
String keyBinding, String valueBinding) {
+ Multimap<String, String> values = HashMultimap.create();
+ try {
+ while (result.hasNext()) {
+ BindingSet bindings = result.next();
+ Binding value = bindings.getBinding(valueBinding);
+ Binding key = bindings.getBinding(keyBinding);
+ if (value == null || key == null) {
+ continue;
+ }
+ values.put(key.getValue().stringValue(),
value.getValue().stringValue());
+ }
+ } catch (QueryEvaluationException e) {
+ throw new FatalException("Can't load results: " + e, e);
+ }
+ return values;
+ }
+
+ /**
* Get list of value subjects connected to entity. The connection is either
* via statement or via reference or via qualifier.
*
- * @param entityId
+ * @param entityIds
* @return Set of value subjects
*/
- public Set<String> getValues(String entityId) {
+ public Multimap<String, String> getValues(Collection<String> entityIds) {
UpdateBuilder b = new UpdateBuilder(getValues);
- b.bindUri("entity:id", uris.entity() + entityId);
+ b.bindUris("entityList", entityIds);
b.bind("uris.value", uris.value());
b.bind("uris.statement", uris.statement());
b.bindUri("prov:wasDerivedFrom", Provenance.WAS_DERIVED_FROM);
- return resultToSet(query(b.toString()), "s");
+ return resultToMap(query(b.toString()), "entity", "s");
}
/**
* Get list of reference subjects connected to entity.
*
- * @param entityId
+ * @param entityIds
* @return Set of references
*/
- public Set<String> getRefs(String entityId) {
+ public Multimap<String, String> getRefs(Collection<String> entityIds) {
UpdateBuilder b = new UpdateBuilder(getRefs);
- b.bindUri("entity:id", uris.entity() + entityId);
+ b.bindUris("entityList", entityIds);
b.bind("uris.statement", uris.statement());
b.bindUri("prov:wasDerivedFrom", Provenance.WAS_DERIVED_FROM);
- return resultToSet(query(b.toString()), "s");
+ return resultToMap(query(b.toString()), "entity", "s");
}
/**
- * Synchronizes the RDF repository's representation of an entity to be
- * exactly the provided statements. You can think of the RDF managed for an
- * entity as a tree rooted at the entity. The managed tree ends where the
- * next entity's managed tree starts. For example Q23 from wikidata
includes
- * all statements about George Washington but not those about Martha
- * (Q191789) even though she is linked by the spouse attribute. On the
other
- * hand the qualifiers on statements about George are included in George.
+ * Provides the SPARQL needed to syncronize the data statements.
*
* @param entityId id of the entity to sync
* @param statements all known statements about the entity
* @param valueList list of used values, for cleanup
* @return the number of statements modified
*/
- public int sync(String entityId, Collection<Statement> statements,
Collection<String> valueList) {
+ public String getSyncQuery(String entityId, Collection<Statement>
statements, Collection<String> valueList) {
// TODO this is becoming a mess too
- log.debug("Updating data for {}", entityId);
+ log.debug("Generating update for {}", entityId);
UpdateBuilder b = new UpdateBuilder(syncBody);
b.bindUri("entity:id", uris.entity() + entityId);
b.bindUri("schema:about", SchemaDotOrg.ABOUT);
@@ -287,11 +317,45 @@
b.bind("cleanupQuery", "");
}
+ return b.toString();
+ }
+
+ /**
+ * Synchronizes the RDF repository's representation of an entity to be
+ * exactly the provided statements. You can think of the RDF managed for an
+ * entity as a tree rooted at the entity. The managed tree ends where the
+ * next entity's managed tree starts. For example Q23 from wikidata
includes
+ * all statements about George Washington but not those about Martha
+ * (Q191789) even though she is linked by the spouse attribute. On the
other
+ * hand the qualifiers on statements about George are included in George.
+ *
+ * @param entityId id of the entity to sync
+ * @param statements all known statements about the entity
+ * @param valueList list of used values, for cleanup
+ * @return the number of statements modified
+ */
+ public int sync(String entityId, Collection<Statement> statements,
Collection<String> valueList) {
long start = System.currentTimeMillis();
- int modified = execute("update", UPDATE_COUNT_RESPONSE, b.toString());
+ int modified = execute("update", UPDATE_COUNT_RESPONSE,
getSyncQuery(entityId, statements, valueList));
log.debug("Updating {} took {} millis and modified {} statements",
entityId,
System.currentTimeMillis() - start, modified);
return modified;
+ }
+
+ /**
+ * Synchronizes the RDF repository's representation of an entity to be
+ * exactly the provided statements.
+ *
+ * @param query Query text
+ * @return the number of statements modified
+ */
+ public int syncQuery(String query) {
+ long start = System.currentTimeMillis();
+ int modified = execute("update", UPDATE_COUNT_RESPONSE, query);
+ log.debug("Update query took {} millis and modified {} statements",
+ System.currentTimeMillis() - start, modified);
+ return modified;
+
}
/**
@@ -303,6 +367,23 @@
*/
public int sync(String entityId, Collection<Statement> statements) {
return sync(entityId, statements, null);
+ }
+
+ /**
+ * Filter set of changes and see which of them really need to be updated.
+ * The changes that have their revision or better in the repo do not need
update.
+ * @param candidates List of candidate changes
+ * @return Set of entity IDs for which the update is needed.
+ */
+ public Set<String> hasRevisions(Collection<Change> candidates) {
+ UpdateBuilder b = new UpdateBuilder(getRevisions);
+ StringBuilder values = new StringBuilder();
+ for (Change entry: candidates) {
+ values.append("( <" + uris.entity() + entry.entityId() + "> " +
entry.revision() + " )\n");
+ }
+ b.bind("values", values.toString());
+ b.bindUri("schema:version", SchemaDotOrg.VERSION);
+ return resultToSet(query(b.toString()), "s");
}
/**
@@ -506,10 +587,15 @@
*/
private static final Pattern ELAPSED_LINE =
Pattern.compile("><p>totalElapsed=[^ ]+ elapsed=([^<]+)</p");
/**
+ * The pattern for the response for an update, with extended times.
+ */
+ private static final Pattern EXTENDED_ELAPSED_LINE =
+ Pattern.compile("><p>totalElapsed=([^ ]+) elapsed=([^ ]+)
whereClause=([^ ]+) deleteClause=([^ ]+) insertClause=([^ <]+)</p");
+ /**
* The pattern for the response for a commit.
*/
private static final Pattern COMMIT_LINE = Pattern
- .compile("><hr><p>COMMIT: totalElapsed=[^ ]+ commitTime=[^ ]+
mutationCount=([^<]+)</p");
+ .compile("><hr><p>COMMIT: totalElapsed=([^ ]+) commitTime=[^
]+ mutationCount=([^<]+)</p");
/**
* The pattern for the response from a bulk update.
*/
@@ -527,15 +613,21 @@
try (BufferedReader reader = new BufferedReader(new
InputStreamReader(entity.getContent(), Charsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
- Matcher m = ELAPSED_LINE.matcher(line);
+ Matcher m;
+ m = EXTENDED_ELAPSED_LINE.matcher(line);
if (m.matches()) {
- log.trace("elapsed = {}", m.group(1));
+ log.debug("total = {} elapsed = {} where = {} delete =
{} insert = {}", m.group(1), m.group(2), m.group(3), m.group(4), m.group(5));
+ continue;
+ }
+ m = ELAPSED_LINE.matcher(line);
+ if (m.matches()) {
+ log.debug("elapsed = {}", m.group(1));
continue;
}
m = COMMIT_LINE.matcher(line);
if (m.matches()) {
- log.debug("mutation count = {}", m.group(1));
- mutationCount = Integer.valueOf(m.group(1));
+ log.debug("total = {} mutation count = {} ",
m.group(1), m.group(2));
+ mutationCount = Integer.valueOf(m.group(2));
continue;
}
m = BULK_UPDATE_LINE.matcher(line);
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
index 728a386..88e1d7f 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/UpdateBuilder.java
@@ -83,10 +83,17 @@
* Bind some uris to a string.
*/
public UpdateBuilder bindUris(String from, Collection<String> uris) {
+ return bindUris(from, uris, "");
+ }
+
+ /**
+ * Bind some uris to a string.
+ */
+ public UpdateBuilder bindUris(String from, Collection<String> uris, String
prefix) {
StringBuilder b = new StringBuilder(uris.size() * 80);
for (String s : uris) {
- b.append('<').append(s).append("> ");
+ b.append('<').append(prefix).append(s).append("> ");
}
bind(from, b.toString().trim());
return this;
@@ -124,7 +131,7 @@
StringBuilder sb = new StringBuilder(l.getLabel().length() * 2);
sb.append('"');
- sb.append(l.getLabel().replace("\\", "\\\\").replace("\"",
"\\\""));
+ sb.append(l.getLabel().replace("\\", "\\\\").replace("\"",
"\\\"").replace("\n", "\\n"));
sb.append('"');
if (l.getLanguage() != null) {
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
index 4b0a4b4..9cb7727 100644
---
a/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
+++
b/tools/src/main/java/org/wikidata/query/rdf/tool/wikibase/WikibaseRepository.java
@@ -102,6 +102,7 @@
public Collection<Statement> fetchRdfForEntity(String entityId) throws
RetryableException {
// TODO handle ?flavor=dump or whatever parameters we need
URI uri = uris.rdf(entityId);
+ long start = System.currentTimeMillis();
log.debug("Fetching rdf from {}", uri);
RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
StatementCollector collector = new StatementCollector();
@@ -125,6 +126,7 @@
} catch (RDFParseException | RDFHandlerException e) {
throw new ContainedException("RDF parsing error for " + uri, e);
}
+ log.debug("Done in {} ms", System.currentTimeMillis() - start);
return collector.getStatements();
}
diff --git
a/tools/src/test/java/org/wikidata/query/rdf/tool/AbstractUpdateIntegrationTestBase.java
b/tools/src/test/java/org/wikidata/query/rdf/tool/AbstractUpdateIntegrationTestBase.java
index 9189d72..51ebeeb 100644
---
a/tools/src/test/java/org/wikidata/query/rdf/tool/AbstractUpdateIntegrationTestBase.java
+++
b/tools/src/test/java/org/wikidata/query/rdf/tool/AbstractUpdateIntegrationTestBase.java
@@ -31,7 +31,8 @@
Change.Source<?> source = IdRangeChangeSource.forItems(from, to, 30);
ExecutorService executorService = new ThreadPoolExecutor(0, 10, 0,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>());
- Update<?> update = new Update<>(source, wikibaseRepository,
rdfRepository(), munger, executorService, 0);
+ WikibaseUris uris = new WikibaseUris("www.wikidata.org");
+ Update<?> update = new Update<>(source, wikibaseRepository,
rdfRepository(), munger, executorService, 0, uris);
update.run();
executorService.shutdown();
}
--
To view, visit https://gerrit.wikimedia.org/r/234674
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9c0727b8275e58db7559c51332af07b3a1b1adc5
Gerrit-PatchSet: 3
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
