Manybubbles has uploaded a new change for review. https://gerrit.wikimedia.org/r/203837
Change subject: WIP: Blazegraph vocabulary and inline uri setup ...................................................................... WIP: Blazegraph vocabulary and inline uri setup This change improves load speed and disk space by about 15%. I suspect we can get another 10% or so if we solve an upstream bug ( http://trac.bigdata.com/ticket/1179#ticket) and then modify our strategy a bit. Change-Id: I864c9a07d497280c6806ab2c23c85fcd89ffe4da --- D blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseDateExtension.java M blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseExtensionFactory.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactory.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabulary.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/AbstractMultiTypeExtension.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/WikibaseDateExtension.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/PositiveIntegerSuffixInlineUriHandler.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/WikibaseEntityInlineUriHandler.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/OntologyVocabularyDecl.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/ProvenanceVocabularyDecl.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/SchemaDotOrgVocabularyDecl.java A blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/WikibaseUrisVocabularyDecl.java A blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/AbstractRandomizedBlazegraphTestBase.java D blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/DummyUnitTest.java A blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactoryUnitTest.java A blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabularyUnitTest.java A blazegraph/src/test/resources/log4j.properties A common/src/main/java/org/wikidata/query/rdf/common/StatementUtil.java M common/src/main/java/org/wikidata/query/rdf/common/uri/Ontology.java M common/src/main/java/org/wikidata/query/rdf/common/uri/WikibaseUris.java M tools/src/test/resources/blazegraph/RWStore.properties 21 files changed, 695 insertions(+), 157 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf refs/changes/37/203837/1 diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseDateExtension.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseDateExtension.java deleted file mode 100644 index b980126..0000000 --- a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseDateExtension.java +++ /dev/null @@ -1,137 +0,0 @@ -package org.wikidata.query.rdf.blazegraph; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.log4j.Logger; -import org.openrdf.model.Literal; -import org.openrdf.model.URI; -import org.openrdf.model.Value; -import org.openrdf.model.vocabulary.XMLSchema; -import org.wikidata.query.rdf.common.WikibaseDate; -import org.wikidata.query.rdf.common.WikibaseDate.ToStringFormat; - -import com.bigdata.rdf.internal.IDatatypeURIResolver; -import com.bigdata.rdf.internal.IExtension; -import com.bigdata.rdf.internal.IV; -import com.bigdata.rdf.internal.impl.literal.AbstractLiteralIV; -import com.bigdata.rdf.internal.impl.literal.LiteralExtensionIV; -import com.bigdata.rdf.internal.impl.literal.XSDNumericIV; -import com.bigdata.rdf.model.BigdataURI; -import com.bigdata.rdf.model.BigdataValue; -import com.bigdata.rdf.model.BigdataValueFactory; -import com.bigdata.util.InnerCause; - -/** - * This implementation of {@link IExtension} implements inlining for literals - * that represent xsd:dateTime literals. Unlike - * {@link com.bigdata.rdf.internal.impl.extensions.DateTimeExtension} on which - * this is based, it stores the literals as time in <strong>seconds</strong> - * since the epoch. The seconds are encoded as an inline long. Also unlike - * DateTimeExtension it only supports UTC as the default time zone because UTC - * is king. This is needed because Wikidata contains dates that who's - * <strong>milliseconds</strong> since epoch don't fit into a long. - */ -public class WikibaseDateExtension<V extends BigdataValue> implements IExtension<V> { - private static final Logger log = Logger.getLogger(WikibaseDateExtension.class); - - private static final List<URI> SUPPORTED_DATA_TYPES = Collections.unmodifiableList(Arrays.asList( - XMLSchema.DATETIME, XMLSchema.DATE)); - - @SuppressWarnings("rawtypes") - private final Map<IV, BigdataURI> dataTypes; - private final Set<BigdataURI> dataTypesSet; - - public WikibaseDateExtension(final IDatatypeURIResolver resolver) { - @SuppressWarnings("rawtypes") - Map<IV, BigdataURI> dataTypes = new HashMap<>(); - for (URI uri : SUPPORTED_DATA_TYPES) { - BigdataURI val = resolver.resolve(uri); - dataTypes.put(val.getIV(), val); - } - this.dataTypes = Collections.unmodifiableMap(dataTypes); - dataTypesSet = Collections.unmodifiableSet(new HashSet<>(this.dataTypes.values())); - } - - @Override - public Set<BigdataURI> getDatatypes() { - return dataTypesSet; - } - - /** - * Attempts to convert the supplied value into an epoch representation and - * encodes the long in a delegate {@link XSDNumericIV}, and returns an - * {@link LiteralExtensionIV} to wrap the native type. - */ - @Override - @SuppressWarnings({ "rawtypes", "unchecked" }) - public LiteralExtensionIV createIV(Value value) { - if (!(value instanceof Literal)) { - throw new IllegalArgumentException("Expected a literal but got " + value); - } - try { - Literal literal = (Literal) value; - BigdataURI dataType = resolveDataType(literal); - WikibaseDate date = WikibaseDate.fromString(value.stringValue()).cleanWeirdStuff(); - AbstractLiteralIV delegate = new XSDNumericIV(date.secondsSinceEpoch()); - return new LiteralExtensionIV(delegate, dataType.getIV()); - } catch (Exception e) { - /* - * Exception logging in blazegraph isn't great for this so we log - * here as well - */ - log.warn("Couldn't create IV", e); - throw e; - } - } - - private BigdataURI resolveDataType(Literal literal) { - URI dt = literal.getDatatype(); - if (dt == null) { - throw new IllegalArgumentException("Literal doesn't have a data type: " + literal); - } - - // TODO why loop instead of use a hash set or something? - for (BigdataURI val : dataTypes.values()) { - // Note: URI.stringValue() is efficient.... - if (val.stringValue().equals(dt.stringValue())) { - return val; - } - } - throw new IllegalArgumentException("Unrecognized data type: " + dt); - } - - /** - * Use the long value of the {@link XSDNumericIV} delegate which represents - * seconds since the epoch to create a WikibaseDate and then represent that - * properly using xsd's string representations. - */ - @Override - @SuppressWarnings({ "rawtypes", "unchecked" }) - public V asValue(final LiteralExtensionIV iv, final BigdataValueFactory vf) { - if (!dataTypes.containsKey(iv.getExtensionIV())) { - throw new IllegalArgumentException("Unrecognized datatype: " + iv.getExtensionIV()); - } - - WikibaseDate date = WikibaseDate.fromSecondsSinceEpoch(iv.getDelegate().longValue()); - try { - BigdataURI dt = dataTypes.get(iv.getExtensionIV()); - - if (dt.equals(XMLSchema.DATE)) { - return (V) vf.createLiteral(date.toString(ToStringFormat.DATE), dt); - } - - return (V) vf.createLiteral(date.toString(ToStringFormat.DATE_TIME), dt); - } catch (RuntimeException ex) { - if (InnerCause.isInnerCause(ex, InterruptedException.class)) { - throw ex; - } - throw new IllegalArgumentException("bad iv: " + iv, ex); - } - } -} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseExtensionFactory.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseExtensionFactory.java index 1b72abc..388279e 100644 --- a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseExtensionFactory.java +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseExtensionFactory.java @@ -3,6 +3,8 @@ import java.util.Collection; import java.util.Iterator; +import org.wikidata.query.rdf.blazegraph.inline.literal.WikibaseDateExtension; + import com.bigdata.rdf.internal.DefaultExtensionFactory; import com.bigdata.rdf.internal.IDatatypeURIResolver; import com.bigdata.rdf.internal.IExtension; diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactory.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactory.java new file mode 100644 index 0000000..f7321c7 --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactory.java @@ -0,0 +1,30 @@ +package org.wikidata.query.rdf.blazegraph; + +import org.wikidata.query.rdf.blazegraph.inline.uri.PositiveIntegerSuffixInlineUriHandler; +import org.wikidata.query.rdf.common.uri.WikibaseUris; + +import com.bigdata.rdf.internal.InlineURIFactory; + +public class WikibaseInlineUriFactory extends InlineURIFactory { + public WikibaseInlineUriFactory() { + // TODO lookup wikibase host and default to wikidata + WikibaseUris uris = WikibaseUris.WIKIDATA; + /* + * Order matters here because wikibase uris are substrings of one + * another. + */ + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.qualifier() + "P")); + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.qualifier() + "Q")); + // TODO add a proper inliner for references + // addHandler(new WikibaseEntityInlineUriHandler(uris.reference())); + // TODO value can have a -value suffix and a uuid suffix + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.value() + "P")); + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.value() + "Q")); + // TODO add a proper inliner for statements + // addHandler(new WikibaseEntityInlineUriHandler(uris.statement())); + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.truthy() + "P")); + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.truthy() + "Q")); + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.entity() + "P")); + addHandler(new PositiveIntegerSuffixInlineUriHandler(uris.entity() + "Q")); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabulary.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabulary.java new file mode 100644 index 0000000..b4a489a --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabulary.java @@ -0,0 +1,35 @@ +package org.wikidata.query.rdf.blazegraph; + +import org.wikidata.query.rdf.blazegraph.vocabulary.OntologyVocabularyDecl; +import org.wikidata.query.rdf.blazegraph.vocabulary.ProvenanceVocabularyDecl; +import org.wikidata.query.rdf.blazegraph.vocabulary.SchemaDotOrgVocabularyDecl; +import org.wikidata.query.rdf.blazegraph.vocabulary.WikibaseUrisVocabularyDecl; +import org.wikidata.query.rdf.common.uri.WikibaseUris; + +import com.bigdata.rdf.vocab.DefaultBigdataVocabulary; + +/** + * Versioned vocabulary classes for wikibase. All classes need a namespace and a + * default constructor or Blazegraph blows up on them. + */ +public class WikibaseVocabulary { + public static class V001 extends DefaultBigdataVocabulary { + public V001() { + } + + public V001(String namespace) { + super(namespace); + } + + @Override + protected void addValues() { + super.addValues(); + // TODO we should try to make the wikibase uris byteiv + // TODO lookup wikibase host and default to wikidata + addDecl(new WikibaseUrisVocabularyDecl(WikibaseUris.WIKIDATA)); + addDecl(new OntologyVocabularyDecl()); + addDecl(new SchemaDotOrgVocabularyDecl()); + addDecl(new ProvenanceVocabularyDecl()); + } + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/AbstractMultiTypeExtension.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/AbstractMultiTypeExtension.java new file mode 100644 index 0000000..10d76ff --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/AbstractMultiTypeExtension.java @@ -0,0 +1,118 @@ +package org.wikidata.query.rdf.blazegraph.inline.literal; + +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.openrdf.model.Literal; +import org.openrdf.model.URI; +import org.openrdf.model.Value; + +import com.bigdata.rdf.internal.IDatatypeURIResolver; +import com.bigdata.rdf.internal.IExtension; +import com.bigdata.rdf.internal.IV; +import com.bigdata.rdf.internal.impl.literal.AbstractLiteralIV; +import com.bigdata.rdf.internal.impl.literal.LiteralExtensionIV; +import com.bigdata.rdf.model.BigdataLiteral; +import com.bigdata.rdf.model.BigdataURI; +import com.bigdata.rdf.model.BigdataValue; +import com.bigdata.rdf.model.BigdataValueFactory; +import com.bigdata.util.InnerCause; + +/** + * Abstract base class for implementing IExtension for multiple dataTypes. + */ +public abstract class AbstractMultiTypeExtension<V extends BigdataValue> implements IExtension<V> { + private static final Logger log = Logger.getLogger(WikibaseDateExtension.class); + + @SuppressWarnings("rawtypes") + private final Map<IV, BigdataURI> dataTypes; + private final Set<BigdataURI> dataTypesSet; + + public AbstractMultiTypeExtension(IDatatypeURIResolver resolver, List<URI> supportedDataTypes) { + @SuppressWarnings("rawtypes") + Map<IV, BigdataURI> dataTypes = new HashMap<>(); + for (URI uri : supportedDataTypes) { + BigdataURI val = resolver.resolve(uri); + dataTypes.put(val.getIV(), val); + } + this.dataTypes = unmodifiableMap(dataTypes); + dataTypesSet = unmodifiableSet(new HashSet<>(this.dataTypes.values())); + } + + @Override + public Set<BigdataURI> getDatatypes() { + return dataTypesSet; + } + + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + public LiteralExtensionIV createIV(Value value) { + if (!(value instanceof Literal)) { + throw new IllegalArgumentException("Expected a literal but got " + value); + } + Literal literal = (Literal) value; + try { + BigdataURI dt = resolveDataType(literal); + return new LiteralExtensionIV(createDelegateIV(literal, dt), dt.getIV()); + } catch (Exception e) { + /* + * Exception logging in blazegraph isn't great for this so we log + * here as well + */ + log.warn("Couldn't create IV", e); + throw e; + } + } + + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + public V asValue(LiteralExtensionIV iv, BigdataValueFactory vf) { + BigdataURI dt = resolveDataType(iv); + try { + return (V) safeAsValue(iv, vf, dt); + } catch (RuntimeException ex) { + if (InnerCause.isInnerCause(ex, InterruptedException.class)) { + throw ex; + } + throw new IllegalArgumentException("bad iv: " + iv, ex); + } + } + + @SuppressWarnings("rawtypes") + protected abstract AbstractLiteralIV createDelegateIV(Literal literal, BigdataURI dt); + + @SuppressWarnings("rawtypes") + protected abstract BigdataLiteral safeAsValue(LiteralExtensionIV iv, BigdataValueFactory vf, BigdataURI dt); + + @SuppressWarnings("rawtypes") + protected BigdataURI resolveDataType(LiteralExtensionIV literal) { + BigdataURI dt = dataTypes.get(literal.getExtensionIV()); + if (dt == null) { + throw new IllegalArgumentException("Unrecognized datatype: " + literal.getExtensionIV()); + } + return dt; + } + + protected BigdataURI resolveDataType(Literal literal) { + URI dt = literal.getDatatype(); + if (dt == null) { + throw new IllegalArgumentException("Literal doesn't have a data type: " + literal); + } + + // TODO why loop instead of use a hash set or something? + for (BigdataURI val : dataTypes.values()) { + // Note: URI.stringValue() is efficient.... + if (val.stringValue().equals(dt.stringValue())) { + return val; + } + } + throw new IllegalArgumentException("Unrecognized data type: " + dt); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/WikibaseDateExtension.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/WikibaseDateExtension.java new file mode 100644 index 0000000..1a1311c --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/literal/WikibaseDateExtension.java @@ -0,0 +1,67 @@ +package org.wikidata.query.rdf.blazegraph.inline.literal; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.openrdf.model.Literal; +import org.openrdf.model.URI; +import org.openrdf.model.vocabulary.XMLSchema; +import org.wikidata.query.rdf.common.WikibaseDate; +import org.wikidata.query.rdf.common.WikibaseDate.ToStringFormat; + +import com.bigdata.rdf.internal.IDatatypeURIResolver; +import com.bigdata.rdf.internal.IExtension; +import com.bigdata.rdf.internal.impl.literal.AbstractLiteralIV; +import com.bigdata.rdf.internal.impl.literal.LiteralExtensionIV; +import com.bigdata.rdf.internal.impl.literal.XSDNumericIV; +import com.bigdata.rdf.model.BigdataLiteral; +import com.bigdata.rdf.model.BigdataURI; +import com.bigdata.rdf.model.BigdataValue; +import com.bigdata.rdf.model.BigdataValueFactory; + +/** + * This implementation of {@link IExtension} implements inlining for literals + * that represent xsd:dateTime literals. Unlike + * {@link com.bigdata.rdf.internal.impl.extensions.DateTimeExtension} on which + * this is based, it stores the literals as time in <strong>seconds</strong> + * since the epoch. The seconds are encoded as an inline long. Also unlike + * DateTimeExtension it only supports UTC as the default time zone because UTC + * is king. This is needed because Wikidata contains dates that who's + * <strong>milliseconds</strong> since epoch don't fit into a long. + */ +public class WikibaseDateExtension<V extends BigdataValue> extends AbstractMultiTypeExtension<V> { + private static final List<URI> SUPPORTED_DATA_TYPES = Collections.unmodifiableList(Arrays.asList( + XMLSchema.DATETIME, XMLSchema.DATE)); + + public WikibaseDateExtension(final IDatatypeURIResolver resolver) { + super(resolver, SUPPORTED_DATA_TYPES); + } + + /** + * Attempts to convert the supplied value into an epoch representation and + * encodes the long in a delegate {@link XSDNumericIV}, and returns an + * {@link LiteralExtensionIV} to wrap the native type. + */ + @Override + @SuppressWarnings("rawtypes") + protected AbstractLiteralIV createDelegateIV(Literal literal, BigdataURI dt) { + WikibaseDate date = WikibaseDate.fromString(literal.stringValue()).cleanWeirdStuff(); + return new XSDNumericIV(date.secondsSinceEpoch()); + } + + /** + * Use the long value of the {@link XSDNumericIV} delegate which represents + * seconds since the epoch to create a WikibaseDate and then represent that + * properly using xsd's string representations. + */ + @Override + @SuppressWarnings("rawtypes") + protected BigdataLiteral safeAsValue(LiteralExtensionIV iv, BigdataValueFactory vf, BigdataURI dt) { + WikibaseDate date = WikibaseDate.fromSecondsSinceEpoch(iv.getDelegate().longValue()); + if (dt.equals(XMLSchema.DATE)) { + return vf.createLiteral(date.toString(ToStringFormat.DATE), dt); + } + return vf.createLiteral(date.toString(ToStringFormat.DATE_TIME), dt); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/PositiveIntegerSuffixInlineUriHandler.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/PositiveIntegerSuffixInlineUriHandler.java new file mode 100644 index 0000000..bac8255 --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/PositiveIntegerSuffixInlineUriHandler.java @@ -0,0 +1,48 @@ +package org.wikidata.query.rdf.blazegraph.inline.uri; + +import org.apache.log4j.Logger; + +import com.bigdata.rdf.internal.InlineURIHandler; +import com.bigdata.rdf.internal.impl.literal.AbstractLiteralIV; +import com.bigdata.rdf.internal.impl.literal.XSDNumericIV; + +/** + * InlineURIHandler that transforms integer suffixes into integers. + * + * We use this temporarily until we can use WikibaseEntityInlineUriHandler. + */ +public class PositiveIntegerSuffixInlineUriHandler extends InlineURIHandler { + private static final Logger log = Logger.getLogger(PositiveIntegerSuffixInlineUriHandler.class); + + public PositiveIntegerSuffixInlineUriHandler(String namespace) { + super(namespace); + } + + @Override + @SuppressWarnings("rawtypes") + protected AbstractLiteralIV createInlineIV(String localName) { + long value; + try { + value = Long.valueOf(localName, 10); + } catch (NumberFormatException e) { + if (log.isDebugEnabled()) { + log.debug("Invalid integer", e); + } + return null; + } + /* + * We could probably do better with unsigned bytes but they don't read + * broken for uris right now. + */ + if (Byte.MIN_VALUE <= value && value <= Byte.MAX_VALUE) { + return new XSDNumericIV((byte) value); + } + if (Short.MIN_VALUE <= value && value <= Short.MAX_VALUE) { + return new XSDNumericIV((short) value); + } + if (Integer.MIN_VALUE <= value && value <= Integer.MAX_VALUE) { + return new XSDNumericIV((int) value); + } + return new XSDNumericIV(value); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/WikibaseEntityInlineUriHandler.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/WikibaseEntityInlineUriHandler.java new file mode 100644 index 0000000..7f250fa --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/inline/uri/WikibaseEntityInlineUriHandler.java @@ -0,0 +1,54 @@ +package org.wikidata.query.rdf.blazegraph.inline.uri; + +import org.apache.log4j.Logger; + +import com.bigdata.rdf.internal.InlineURIHandler; +import com.bigdata.rdf.internal.impl.literal.AbstractLiteralIV; +import com.bigdata.rdf.internal.impl.literal.XSDNumericIV; + +/** + * Inlines an entity style (Q101231) wikibase uri. + */ +public class WikibaseEntityInlineUriHandler extends InlineURIHandler { + private static final Logger log = Logger.getLogger(WikibaseEntityInlineUriHandler.class); + + public WikibaseEntityInlineUriHandler(String prefix) { + // TODO fix this class once http://trac.bigdata.com/ticket/1179#ticket + /* + * This class won't work with 1179 because it can't transform the ids + * back into P and Q form on load - they are stuck as negative and + * positive integers respectively. + */ + super(prefix); + } + + @Override + @SuppressWarnings("rawtypes") + protected AbstractLiteralIV createInlineIV(String localName) { + try { + String entityId = localName; + char leading = entityId.charAt(0); + int id = Integer.valueOf(entityId.substring(1), 10); + if (id < 0) { + throw new RuntimeException("Hit a negative id on entity style uri: " + entityId); + } + switch (leading) { + case 'P': + case 'p': + id = -id; + break; + case 'Q': + case 'q': + break; + default: + // TODO check if users typing can get these! + throw new RuntimeException("Hit unexpected prefix on entity style uri: " + entityId); + } + return new XSDNumericIV(id); + + } catch (NumberFormatException e) { + log.warn("Hit an invalid entity style uri: " + localName); + return null; + } + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/OntologyVocabularyDecl.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/OntologyVocabularyDecl.java new file mode 100644 index 0000000..7be13cf --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/OntologyVocabularyDecl.java @@ -0,0 +1,24 @@ +package org.wikidata.query.rdf.blazegraph.vocabulary; + +import static org.wikidata.query.rdf.common.uri.Ontology.BEST_RANK; +import static org.wikidata.query.rdf.common.uri.Ontology.DEPRECATED_RANK; +import static org.wikidata.query.rdf.common.uri.Ontology.NAMESPACE; +import static org.wikidata.query.rdf.common.uri.Ontology.NORMAL_RANK; +import static org.wikidata.query.rdf.common.uri.Ontology.PREFERRED_RANK; +import static org.wikidata.query.rdf.common.uri.Ontology.RANK; + +import org.wikidata.query.rdf.common.uri.Ontology.Time; + +import com.bigdata.rdf.vocab.BaseVocabularyDecl; + +/** + * Vocabulary containing the URIs from + * {@linkplain org.wikidata.query.rdf.common.uri.Ontology} that are imported + * into Blazegraph. + */ +public class OntologyVocabularyDecl extends BaseVocabularyDecl { + public OntologyVocabularyDecl() { + super(NAMESPACE, RANK, BEST_RANK, PREFERRED_RANK, NORMAL_RANK, DEPRECATED_RANK, Time.VALUE, Time.PRECISION, + Time.TIMEZONE, Time.CALENDAR_MODEL); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/ProvenanceVocabularyDecl.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/ProvenanceVocabularyDecl.java new file mode 100644 index 0000000..7379334 --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/ProvenanceVocabularyDecl.java @@ -0,0 +1,18 @@ +package org.wikidata.query.rdf.blazegraph.vocabulary; + +import static org.wikidata.query.rdf.common.uri.Provenance.NAMESPACE; +import static org.wikidata.query.rdf.common.uri.Provenance.WAS_DERIVED_FROM; + +import com.bigdata.rdf.vocab.BaseVocabularyDecl; + + +/** + * Vocabulary containing the URIs from + * {@linkplain org.wikidata.query.rdf.common.uri.Ontology} that are imported + * into Blazegraph. + */ +public class ProvenanceVocabularyDecl extends BaseVocabularyDecl { + public ProvenanceVocabularyDecl() { + super(NAMESPACE, WAS_DERIVED_FROM); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/SchemaDotOrgVocabularyDecl.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/SchemaDotOrgVocabularyDecl.java new file mode 100644 index 0000000..2af494d --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/SchemaDotOrgVocabularyDecl.java @@ -0,0 +1,22 @@ +package org.wikidata.query.rdf.blazegraph.vocabulary; + +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.ABOUT; +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.ARTICLE; +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.DATE_MODIFIED; +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.DESCRIPTION; +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.IN_LANGUAGE; +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.NAMESPACE; +import static org.wikidata.query.rdf.common.uri.SchemaDotOrg.VERSION; + +import com.bigdata.rdf.vocab.BaseVocabularyDecl; + +/** + * Vocabulary containing the URIs from + * {@linkplain org.wikidata.query.rdf.common.uri.Ontology} that are imported + * into Blazegraph. + */ +public class SchemaDotOrgVocabularyDecl extends BaseVocabularyDecl { + public SchemaDotOrgVocabularyDecl() { + super(NAMESPACE, VERSION, DATE_MODIFIED, ABOUT, ARTICLE, IN_LANGUAGE, DESCRIPTION); + } +} diff --git a/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/WikibaseUrisVocabularyDecl.java b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/WikibaseUrisVocabularyDecl.java new file mode 100644 index 0000000..5aabfe8 --- /dev/null +++ b/blazegraph/src/main/java/org/wikidata/query/rdf/blazegraph/vocabulary/WikibaseUrisVocabularyDecl.java @@ -0,0 +1,25 @@ +package org.wikidata.query.rdf.blazegraph.vocabulary; + +import org.wikidata.query.rdf.common.uri.WikibaseUris; + +import com.bigdata.rdf.vocab.BaseVocabularyDecl; + +/** + * Vocabulary containing the URIs from + * {@linkplain org.wikidata.query.rdf.common.uri.Ontology} that are imported + * into Blazegraph. + */ +public class WikibaseUrisVocabularyDecl extends BaseVocabularyDecl { + public WikibaseUrisVocabularyDecl(WikibaseUris uris) { + super(uris.entity(), uris.truthy(), uris.statement(), uris.value(), uris.reference(), uris.qualifier(),// + /* + * Note that these next two lines are required to make + * WikibaseInlineUriFactory work with + * IntegerSuffixInlineUriHandler which is required due to a + * defficiency in Blazegraph. When that defficiency is resolved + * these will no longer be required. + */ + uris.entity() + "P", uris.truthy() + "P", uris.value() + "P", uris.qualifier() + "P",// + uris.entity() + "Q", uris.truthy() + "Q", uris.value() + "Q", uris.qualifier() + "Q"); + } +} diff --git a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/AbstractRandomizedBlazegraphTestBase.java b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/AbstractRandomizedBlazegraphTestBase.java new file mode 100644 index 0000000..361fc29 --- /dev/null +++ b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/AbstractRandomizedBlazegraphTestBase.java @@ -0,0 +1,133 @@ +package org.wikidata.query.rdf.blazegraph; + +import static com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope.SUITE; + +import java.lang.Thread.UncaughtExceptionHandler; +import java.math.BigInteger; +import java.util.Properties; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.runner.RunWith; +import org.openrdf.model.Resource; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.impl.IntegerLiteralImpl; +import org.openrdf.model.impl.URIImpl; +import org.wikidata.query.rdf.common.uri.WikibaseUris; + +import com.bigdata.cache.SynchronizedHardReferenceQueueWithTimeout; +import com.bigdata.journal.TemporaryStore; +import com.bigdata.rdf.model.BigdataStatement; +import com.bigdata.rdf.store.ITripleStore; +import com.bigdata.rdf.store.TempTripleStore; +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; + +/** + * Randomized test that can create a triple store. + */ +@RunWith(RandomizedRunner.class) +@ThreadLeakScope(SUITE) +public class AbstractRandomizedBlazegraphTestBase extends RandomizedTest { + private WikibaseUris uris = WikibaseUris.WIKIDATA; + private static TemporaryStore temporaryStore; + private ITripleStore store; + + protected WikibaseUris uris() { + return uris; + } + + /** + * Get a triple store. Lazily initialized once per test method. + */ + protected ITripleStore store() { + if (store != null) { + return store; + } + Properties properties = new Properties(); + properties.setProperty("com.bigdata.rdf.store.AbstractTripleStore.vocabularyClass", + WikibaseVocabulary.V001.class.getName()); + properties.setProperty("com.bigdata.rdf.store.AbstractTripleStore.inlineURIFactory", + WikibaseInlineUriFactory.class.getName()); + store = new TempTripleStore(temporaryStore(), properties, null); + return store; + } + + /** + * Round trip a statement through Blazegraph. + */ + protected BigdataStatement roundTrip(Object s, Object p, Object o) { + return roundTrip((Resource) convert(s), (URI) convert(p), convert(o)); + } + + /** + * Round trip a statement through Blazegraph. + */ + protected BigdataStatement roundTrip(Resource s, URI p, Value o) { + store().addStatement(s, p, o, null); + return store().getStatement(s, p, o, null); + } + + /** + * Convert any object into an RDF value. + */ + protected Value convert(Object o) { + if (o instanceof Value) { + return (Value) o; + } + if (o instanceof String) { + String s = (String) o; + s = s.replace("data:", uris.entityData()); + s = s.replace("entity:", uris.entity()); + s = s.replace("truthy:", uris.truthy()); + s = s.replace("s:", uris.statement()); + s = s.replace("v:", uris.value()); + s = s.replace("ref:", uris.reference()); + s = s.replace("q:", uris.qualifier()); + return new URIImpl(s); + } + if (o instanceof Integer) { + return new IntegerLiteralImpl(BigInteger.valueOf((int) o)); + } + throw new RuntimeException("No idea how to convert " + o + " to a value. Its a " + o.getClass() + "."); + } + + /** + * Get a TemporaryStore. Lazily initialized once per test class. + */ + private static TemporaryStore temporaryStore() { + if (temporaryStore != null) { + return temporaryStore; + } + /* + * Initializing the temporary store replaces RandomizedRunner's + * painstakingly applied UncaughtExceptionHandler. That is bad so we + * replace it. + */ + UncaughtExceptionHandler uncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); + temporaryStore = new TemporaryStore(); + Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler); + return temporaryStore; + } + + @After + public void closeStore() { + if (store == null) { + return; + } + store.close(); + store = null; + } + + @AfterClass + public static void closeTemporaryStore() { + if (temporaryStore == null) { + return; + } + temporaryStore.close(); + SynchronizedHardReferenceQueueWithTimeout.stopStaleReferenceCleaner(); + temporaryStore = null; + } +} diff --git a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/DummyUnitTest.java b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/DummyUnitTest.java deleted file mode 100644 index ae22fc9..0000000 --- a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/DummyUnitTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.wikidata.query.rdf.blazegraph; - -import static org.hamcrest.Matchers.lessThan; - -import org.junit.Test; -import org.junit.runner.RunWith; - -import com.carrotsearch.randomizedtesting.RandomizedRunner; -import com.carrotsearch.randomizedtesting.RandomizedTest; - -@RunWith(RandomizedRunner.class) -public class DummyUnitTest extends RandomizedTest { - @Test - public void dummy() { - // TODO remove me when there are real tests here - assertThat(randomIntBetween(0, 10), lessThan(11)); - } -} diff --git a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactoryUnitTest.java b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactoryUnitTest.java new file mode 100644 index 0000000..1399213 --- /dev/null +++ b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseInlineUriFactoryUnitTest.java @@ -0,0 +1,67 @@ +package org.wikidata.query.rdf.blazegraph; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.instanceOf; + +import org.hamcrest.Matcher; +import org.junit.Test; + +import com.bigdata.rdf.internal.IV; +import com.bigdata.rdf.internal.impl.TermId; +import com.bigdata.rdf.internal.impl.literal.XSDIntegerIV; +import com.bigdata.rdf.internal.impl.uri.URIExtensionIV; +import com.bigdata.rdf.model.BigdataStatement; + +public class WikibaseInlineUriFactoryUnitTest extends AbstractRandomizedBlazegraphTestBase { + @Test + public void entityAndTruthyAreInlined() { + BigdataStatement statement = roundTrip("entity:Q23", "truthy:P509", "entity:Q356405"); + assertThat(statement.getSubject().getIV(), uriIv(uris().entity(), "Q23")); + assertThat(statement.getPredicate().getIV(), uriIv(uris().truthy(), "P509")); + assertThat(statement.getObject().getIV(), uriIv(uris().entity(), "Q356405")); + } + + // @Test + // NOTE this won't work until we can use WikibaseEntityInlineUriHandler + public void lowecaseEntityAndTruthyAreInlined() { + BigdataStatement statement = roundTrip("entity:q23", "truthy:p509", "entity:q356405"); + assertThat(statement.getSubject().getIV(), uriIv(uris().entity(), "Q23")); + assertThat(statement.getPredicate().getIV(), uriIv(uris().truthy(), "P509")); + assertThat(statement.getObject().getIV(), uriIv(uris().entity(), "Q356405")); + } + + @Test + public void valueIsInlined() { + BigdataStatement statement = roundTrip("s:Q23-1EDEEEE-F0DF-4A07-980F-5E76866B74D7", "v:P1711", 100686); + // NOTE subject won't be a termid when we do statements properly + assertThat(statement.getSubject().getIV(), instanceOf(TermId.class)); + assertThat(statement.getPredicate().getIV(), uriIv(uris().value(), "P1711")); + assertThat(statement.getObject().getIV(), instanceOf(XSDIntegerIV.class)); + } + + @Test + public void qualifiersAreInlined() { + BigdataStatement statement = roundTrip("s:Q23-1EDEEEE-F0DF-4A07-980F-5E76866B74D7", "q:P1711", 100686); + // NOTE subject won't be a termid when we do statements properly + assertThat(statement.getSubject().getIV(), instanceOf(TermId.class)); + assertThat(statement.getPredicate().getIV(), uriIv(uris().qualifier(), "P1711")); + assertThat(statement.getObject().getIV(), instanceOf(XSDIntegerIV.class)); + } + + @SuppressWarnings("rawtypes") + public static Matcher<IV> uriIv(String namespace, String localName) { + /* + * NOTE this works around the hack we have to do for + * IntegerSuffixInlineUriHandler and can be removed when we no longer + * use it. + */ + namespace += localName.charAt(0); + localName = localName.substring(1); + return allOf(// + instanceOf((Class<? extends IV>) URIExtensionIV.class),// + hasProperty("namespace", equalTo(namespace)),// + hasProperty("localName", equalTo(localName))); + } +} diff --git a/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabularyUnitTest.java b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabularyUnitTest.java new file mode 100644 index 0000000..cb4e3e0 --- /dev/null +++ b/blazegraph/src/test/java/org/wikidata/query/rdf/blazegraph/WikibaseVocabularyUnitTest.java @@ -0,0 +1,22 @@ +package org.wikidata.query.rdf.blazegraph; + +import static org.hamcrest.Matchers.instanceOf; + +import org.junit.Test; +import org.wikidata.query.rdf.common.uri.Ontology; + +import com.bigdata.rdf.internal.impl.AbstractInlineIV; +import com.bigdata.rdf.internal.impl.TermId; +import com.bigdata.rdf.model.BigdataStatement; + +public class WikibaseVocabularyUnitTest extends AbstractRandomizedBlazegraphTestBase { + @Test + public void ranksAreIV() { + BigdataStatement statement = roundTrip("s:Q23-uuidhere", Ontology.RANK, Ontology.BEST_RANK); + assertThat(statement.getSubject().getIV(), instanceOf(TermId.class)); + assertThat(statement.getPredicate().getIV(), instanceOf(AbstractInlineIV.class)); + assertThat(statement.getObject().getIV(), instanceOf(AbstractInlineIV.class)); + } + + // TODO verify more things! +} diff --git a/blazegraph/src/test/resources/log4j.properties b/blazegraph/src/test/resources/log4j.properties new file mode 100644 index 0000000..9c13c52 --- /dev/null +++ b/blazegraph/src/test/resources/log4j.properties @@ -0,0 +1,6 @@ +log4j.rootLogger=WARN, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n diff --git a/common/src/main/java/org/wikidata/query/rdf/common/StatementUtil.java b/common/src/main/java/org/wikidata/query/rdf/common/StatementUtil.java new file mode 100644 index 0000000..6f1a43d --- /dev/null +++ b/common/src/main/java/org/wikidata/query/rdf/common/StatementUtil.java @@ -0,0 +1,5 @@ +package org.wikidata.query.rdf.common; + +public class StatementUtil { + +} diff --git a/common/src/main/java/org/wikidata/query/rdf/common/uri/Ontology.java b/common/src/main/java/org/wikidata/query/rdf/common/uri/Ontology.java index 590706a..a70e4a6 100644 --- a/common/src/main/java/org/wikidata/query/rdf/common/uri/Ontology.java +++ b/common/src/main/java/org/wikidata/query/rdf/common/uri/Ontology.java @@ -8,10 +8,16 @@ /** * Wikibase exports all items with an assertion that their RDF.TYPE is this - * and we filter that out. + * and we filter that out. Its also used as a inline literal type for + * inlining uris in Blazegraph. */ public static final String ITEM = NAMESPACE + "Item"; /** + * Wikibase exports all items with an assertion that their RDF.TYPE is this. + * Its also used as a inline literal type for inlining uris in Blazegraph. + */ + public static final String PROPERTY = NAMESPACE + "Property"; + /** * Wikibase exports all statements with an assertion that their RDF.TYPE is * this and we filter that out. */ diff --git a/common/src/main/java/org/wikidata/query/rdf/common/uri/WikibaseUris.java b/common/src/main/java/org/wikidata/query/rdf/common/uri/WikibaseUris.java index 02bada2..2d413fc 100644 --- a/common/src/main/java/org/wikidata/query/rdf/common/uri/WikibaseUris.java +++ b/common/src/main/java/org/wikidata/query/rdf/common/uri/WikibaseUris.java @@ -12,6 +12,7 @@ private final String entityData; private final String entity; + private final String truthy; private final String statement; private final String value; private final String reference; @@ -21,6 +22,7 @@ String root = "http://"; + host; entityData = root + "/wiki/Special:EntityData/"; entity = root + "/entity/"; + truthy = root + "/assert/"; statement = entity + "statement/"; value = entity + "value/"; reference = entity + "reference/"; @@ -33,6 +35,7 @@ public StringBuilder prefixes(StringBuilder query) { query.append("PREFIX data: <").append(entityData).append(">\n"); query.append("PREFIX entity: <").append(entity).append(">\n"); + query.append("PREFIX t: <").append(truthy).append(">\n"); query.append("PREFIX s: <").append(statement).append(">\n"); query.append("PREFIX v: <").append(value).append(">\n"); query.append("PREFIX ref: <").append(reference).append(">\n"); @@ -57,6 +60,13 @@ /** * Prefix wikibase uses for statements. */ + public String truthy() { + return truthy; + } + + /** + * Prefix wikibase uses for statements. + */ public String statement() { return statement; } diff --git a/tools/src/test/resources/blazegraph/RWStore.properties b/tools/src/test/resources/blazegraph/RWStore.properties index 3a59411..6088420 100644 --- a/tools/src/test/resources/blazegraph/RWStore.properties +++ b/tools/src/test/resources/blazegraph/RWStore.properties @@ -23,7 +23,8 @@ com.bigdata.rdf.store.AbstractTripleStore.axiomsClass=com.bigdata.rdf.axioms.NoAxioms # Use the default vocabulary for now. -com.bigdata.rdf.store.AbstractTripleStore.vocabularyClass=com.bigdata.rdf.vocab.DefaultBigdataVocabulary +com.bigdata.rdf.store.AbstractTripleStore.vocabularyClass=org.wikidata.query.rdf.blazegraph.WikibaseVocabulary$V001 +com.bigdata.rdf.store.AbstractTripleStore.inlineURIFactory=org.wikidata.query.rdf.blazegraph.WikibaseInlineUriFactory com.bigdata.rdf.store.AbstractTripleStore.extensionFactoryClass=org.wikidata.query.rdf.blazegraph.WikibaseExtensionFactory # These seem to be ubiquitous overwrites. Not sure why they aren't the default but it works. -- To view, visit https://gerrit.wikimedia.org/r/203837 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I864c9a07d497280c6806ab2c23c85fcd89ffe4da Gerrit-PatchSet: 1 Gerrit-Project: wikidata/query/rdf Gerrit-Branch: master Gerrit-Owner: Manybubbles <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
