Manybubbles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/202922
Change subject: WIP: skip bad uris during load
......................................................................
WIP: skip bad uris during load
When OpenRDF hits a uri it doesn't like it just gives up and throws a fatal
exception. We try to catch and work around these exceptions. We hope this
hack is temporary.
Note that this only applies to the dump processor - not the updater.
Change-Id: I46d786b214435520c72b73a7adb7b9ebbc17f5e6
---
M pom.xml
M tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
2 files changed, 33 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/22/202922/1
diff --git a/pom.xml b/pom.xml
index d678455..10c2ecf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -243,7 +243,7 @@
</signaturesFiles>
<excludes>
<!-- Some portions of the project need access to System.out
and things so we contain them in inner classes of this form. -->
- <exclude>**/*$ForbiddenOk.class</exclude>
+ <exclude>**/*$ForbiddenOk**.class</exclude>
</excludes>
</configuration>
</execution>
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
index a9283f3..6cfece6 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
@@ -12,6 +12,7 @@
import java.util.List;
import org.openrdf.model.Statement;
+import org.openrdf.model.URI;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandler;
import org.openrdf.rio.RDFHandlerException;
@@ -19,6 +20,7 @@
import org.openrdf.rio.RDFParser;
import org.openrdf.rio.RDFWriter;
import org.openrdf.rio.Rio;
+import org.openrdf.rio.turtle.TurtleParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.query.rdf.common.uri.Entity;
@@ -122,7 +124,9 @@
@Override
public void run() {
try {
- RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
+ // TODO this is a temporary hack
+ // RDFParser parser = Rio.createParser(RDFFormat.TURTLE);
+ RDFParser parser = new ForbiddenOk.HackedTurtleParser();
RDFWriter writer = Rio.createWriter(RDFFormat.TURTLE, to);
RDFHandler handler = new EntityMungingRdfHandler(entityDataUris,
munger, writer);
handler = new Normalizer(handler);
@@ -275,4 +279,31 @@
}
}
}
+
+ /**
+ * We need access to getMessage from exceptions. This is brittle but
+ * (hopefully) temporary.
+ */
+ private static class ForbiddenOk {
+ private static class HackedTurtleParser extends TurtleParser {
+ @Override
+ protected URI parseURI() throws IOException, RDFParseException {
+ try {
+ return super.parseURI();
+ } catch (RDFParseException e) {
+ if (e.getMessage().startsWith("IRI includes string
escapes: ")
+ || e.getMessage().startsWith("IRI included an
unencoded space: '32'")) {
+ log.warn("Attempting to recover from", e);
+ // Unless we hit a \> then consume until we get to a >
+ if (!e.getMessage().startsWith("IRI includes string
escapes: '\\62'")) {
+ while (readCodePoint() != '>') {
+ }
+ }
+ return super.resolveURI("http://example.com/error";);
+ }
+ throw e;
+ }
+ }
+ }
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/202922
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I46d786b214435520c72b73a7adb7b9ebbc17f5e6
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Manybubbles <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
