Smalyshev has uploaded a new change for review.
https://gerrit.wikimedia.org/r/183317
Change subject: 0.9 port phase 2 - data import works now
......................................................................
0.9 port phase 2 - data import works now
Change-Id: Ic07366dc430925180632b17d51af03564727a53a
---
A src/main/groovy/org/wikidata/gremlin/DateUtils.groovy
M src/main/groovy/org/wikidata/gremlin/Loader.groovy
M src/main/groovy/org/wikidata/gremlin/Schema.groovy
3 files changed, 172 insertions(+), 37 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/gremlin
refs/changes/17/183317/1
diff --git a/src/main/groovy/org/wikidata/gremlin/DateUtils.groovy
b/src/main/groovy/org/wikidata/gremlin/DateUtils.groovy
new file mode 100644
index 0000000..47cf3f3
--- /dev/null
+++ b/src/main/groovy/org/wikidata/gremlin/DateUtils.groovy
@@ -0,0 +1,113 @@
+package org.wikidata.gremlin
+import org.joda.time.*;
+import org.joda.time.format.*;
+class DateUtils {
+ // Rough number of seconds in a year, for storing whole years
+ // Source: https://en.wikipedia.org/wiki/Year#Astronomical_years
+ public final static long SECONDS_IN_YEAR = 31557600L;
+ // Largest year that Java can represent accurately.
+ // This is just an threshold, beyond this year we do not try to parse
dates
+ // but store (number of whole years)*SECONDS_IN_YEAR.
+ public final static long LARGEST_YEAR = 292000000L;
+ // Parser for year as dates
+ private final static def yearParser =
DateTimeFormat.forPattern('y').withZone(DateTimeZone.UTC)
+ // Seconds for largest Java-representable date
+ public final static long LARGEST_DATE_SECONDS =
yearParser.parseDateTime(LARGEST_YEAR as String).getMillis()/1000;
+ // Seconds for lower bound of dates we represent as Java dates
+ public final static long LOWEST_DATE_SECONDS =
yearParser.parseDateTime("0").getMillis()/1000;
+
+ public final static DateTime dateMax = new
DateTime(LARGEST_DATE_SECONDS*1000+1, DateTimeZone.UTC)
+ public final static DateTime dateMin = new
DateTime(LOWEST_DATE_SECONDS*1000-1, DateTimeZone.UTC)
+
+ /**
+ * Can this date be converted to Java date?
+ * @param seconds
+ * @return
+ */
+ public static boolean javaDate(long seconds)
+ {
+ return seconds < LARGEST_DATE_SECONDS && seconds >
LOWEST_DATE_SECONDS;
+ }
+
+ /**
+ * Create Java DateTime object from seconds
+ * @param seconds
+ * @return DateTime
+ */
+ public static DateTime toJavaDate(long seconds) {
+ if(!javaDate(seconds)) {
+ /* TODO: not sure what to return here, maybe exception?
*/
+ return seconds>0?dateMax:dateMin;
+ }
+ return new DateTime(seconds*1000, DateTimeZone.UTC)
+ }
+
+ /**
+ * Get seconds from year (assuming it is YYYY-01-01 00:00:00)
+ * @param year
+ * @return
+ */
+ public static long fromYear(long year) {
+ if(year >= LARGEST_YEAR) {
+ return
SECONDS_IN_YEAR*(year-LARGEST_YEAR)+LARGEST_DATE_SECONDS;
+ }
+ if(year <= 0) {
+ return SECONDS_IN_YEAR*year+LOWEST_DATE_SECONDS;
+ }
+ yearParser.parseDateTime(year as String).getMillis()/1000
+ }
+
+ /**
+ * Translate from seconds to year
+ * Only to be used for non-Java date years
+ * @param seconds
+ * @return
+ */
+ public static long asYear(long seconds) {
+ if(seconds > 0) {
+ return
(seconds-LARGEST_DATE_SECONDS)/SECONDS_IN_YEAR+LARGEST_YEAR;
+ }
+ return (seconds-LOWEST_DATE_SECONDS)/SECONDS_IN_YEAR;
+ }
+
+ private final static def df = DateTimeFormat.forPattern('y-M-d')
+ /**
+ * Get seconds from date YYYY-MM-DD
+ * This makes sense only within gregorian calendar times
+ * @param isoDate
+ * @return
+ */
+ public static long fromDate(String isoDate) {
+ return fromDate(df.parseDateTime(isoDate))
+ }
+
+ /**
+ * Get seconds from Java date
+ * @param isoDate
+ * @return
+ */
+ public static long fromDate(Date javaDate) {
+ return javaDate.getTime()/1000;
+ }
+
+ /**
+ * Get seconds from Java date
+ * @param isoDate
+ * @return
+ */
+ public static long fromDate(DateTime javaDate) {
+ return javaDate.getMillis()/1000;
+ }
+
+ private final static def printForm =
ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.UTC)
+ /**
+ * Returns ISO8601 (extended) representation of the date
+ * @return String
+ */
+ public static String asString(long seconds) {
+ if(!javaDate(seconds)) {
+ return sprintf("%04d-01-01T00:00:00Z", asYear(seconds));
+ }
+ return printForm.print(new DateTime(seconds*1000,
DateTimeZone.UTC))
+ }
+}
diff --git a/src/main/groovy/org/wikidata/gremlin/Loader.groovy
b/src/main/groovy/org/wikidata/gremlin/Loader.groovy
index 60aa7e2..04323e0 100644
--- a/src/main/groovy/org/wikidata/gremlin/Loader.groovy
+++ b/src/main/groovy/org/wikidata/gremlin/Loader.groovy
@@ -211,6 +211,14 @@
}
/**
+ * Is this data type a link?
+ */
+ private boolean isLinkType(datatype)
+ {
+ return datatype == 'wikibase-item' || datatype == 'wikibase-property'
+ }
+
+ /**
* Check if property exists and create it if not
* Creates endge labels, properties, etc.
*/
@@ -221,8 +229,7 @@
// For value properties, we also need the value prop
initProperty(getValueName(item['id']), getDataType(item['datatype']),
item['id'])
initProperty(getQualifierName(item['id']),
getDataType(item['datatype']), item['id'])
- if(item['datatype'] && item['datatype'] != 'wikibase-item'
- && item['datatype'] != 'wikibase-property' &&
item['datatype'] != 'string') {
+ if(item['datatype'] && !isLinkType(item['datatype']) &&
item['datatype'] != 'string') {
initProperty(getAllValuesName(getValueName(item['id'])))
initProperty(getAllValuesName(getQualifierName(item['id'])))
}
@@ -236,7 +243,7 @@
def v = bgraph.getVertex(id)
if(!v) {
v = bgraph.addVertex(id)
- v.setProperty("stub", true)
+ v.property("stub", true)
}
return v
}
@@ -368,15 +375,23 @@
log.info "Cannot update in batch mode, skipping..."
return
}
- for(cl in v.outE.has('edgeType', 'claim')) {
+ for(cl in v.outE('claim')) {
if(!(cl.contentHash in claimsById)) {
- def prop = cl.getProperty('property')
- def target = cl.inV().next()
+ // This one is not there anymore, remove it
+ def prop = cl.property('property').value()
+ def targetId = null
log.debug "Dropping old claim ${cl.wikibaseId}"
- v.outE.has('contentHash',
cl.contentHash).remove()
+ if(isLinkType(cl.property('datatype').value()))
{
+ // Try to find target id for this link
+ def e = v.outE(prop).has('contentHash',
cl.contentHash).next()
+ targetId =
e.property(getValueName(prop)).value()
+ }
+ // Drop related claims
+ v.outE('claim').has('contentHash',
cl.contentHash).remove()
+ v.outE(prop).has('contentHash',
cl.contentHash).remove()
// update also the links
- if(target.wikibaseId && target.wikibaseId[0] ==
'Q'
- && !v.out(prop).has('wikibaseId',
target.wikibaseId).hasNext()) {
+ if(targetId && targetId[0] == 'Q'
+ && !v.out(prop).has('wikibaseId',
targetId).hasNext()) {
def lname = getLinkName(prop)
// if this link targeted v(target) and
there are no links to it anymore
// drop it from links
@@ -386,7 +401,7 @@
break;
}
}
- v.setProperty(getLinkName(prop)+"_",
v[lname].join(' '))
+ v.singleProperty(getLinkName(prop)+"_",
v[lname].join(' '))
}
} else {
claimsById[cl.contentHash].exists = true
@@ -426,8 +441,8 @@
}
def value = extractPropertyValueFromClaim(qitem)
if(value) {
- item.setProperty(qname, value)
- if(qitem.datatype != 'wikibase-item' &&
qitem.datatype != 'string' && qitem.datatype != 'wikibase-property') {
+ item.property(qname, value)
+ if(!isLinkType(qitem.datatype) &&
qitem.datatype != 'string') {
addAllValues(qitem.datavalue.value,
qname, item)
}
}
@@ -509,7 +524,7 @@
}
v[it.key] = it.value
}
- item.setProperty(getAllValuesName(property), v)
+ item.property(getAllValuesName(property), v)
}
/**
@@ -525,34 +540,39 @@
def data = claim.mainsnak
def value = extractPropertyValueFromClaim(data)
def outgoing = getTargetFromSnak(claim.mainsnak, value)
- def isValue = (data.datatype != 'wikibase-item' && data.datatype !=
'wikibase-property')
+ def isValue = !isLinkType(data.datatype)
+ def dtype = data.datatype ?: data.snaktype ?: ""
+ log.debug("Processing claim {} with {}", claim.wikibaseId, data)
+
if(!outgoing) {
- return null
+ log.debug "Could not find target, skipping..."
+ return null
}
claims++;
// Here is the strange thing: v.outE() is slow but v.outE('claim') is
fast even though they return the same
// So we provide edge for all claims so we could fetch them e.g. for
deleting old ones
- // TODO: may not be needed anymore if we can use contentHash + property
-// def claimC = v.addEdge('claim', outgoing)
-// claimC.setProperty('wikibaseId', claim.id)
-// claimC.setProperty('property', data.property)
-// claimC.setProperty('contentHash', contentHash)
-
+ def claimC = v.addEdge('claim', outgoing)
+ claimC.property('wikibaseId', claim.id)
+ claimC.property('property', data.property)
+ claimC.property('contentHash', contentHash)
+ claimC.property('datatype', dtype)
+
log.debug "Updating claim ${claim.id}"
def claimE = v.addEdge(data.property, outgoing)
- claimE.setProperty('edgeType', 'claim')
- claimE.setProperty('contentHash', contentHash)
- claimE.setProperty('rank', claim.rank == "preferred")
- claimE.setProperty('wikibaseId', claim.id)
- claimE.setProperty('property', data.property)
-
+ claimE.property('edgeType', 'claim')
+ claimE.property('contentHash', contentHash)
+ claimE.property('rank', claim.rank == "preferred")
+ claimE.property('wikibaseId', claim.id)
+ claimE.property('property', data.property)
+ claimE.property('datatype', dtype)
+
if(claim.qualifiers) {
addQualifiers(claimE, claim.qualifiers)
}
if(isValue && value) {
- claimE.setProperty(getValueName(data.property), value)
+ claimE.property(getValueName(data.property), value)
if(data.datatype != 'string' ) {
addAllValues(data.datavalue.value,
getValueName(data.property), claimE)
}
@@ -561,7 +581,7 @@
// Since vertex and its edges are stored together but going to
other side
// requires loading another vertex
if(!isValue && outgoing.wikibaseId) {
- claimE.setProperty(getValueName(data.property),
outgoing.wikibaseId)
+ claimE.property(getValueName(data.property),
outgoing.wikibaseId)
}
}
@@ -581,10 +601,10 @@
if(!(outgoing.wikibaseId in linkCache[lname])) {
// We need the check above since despite being called SET,
the property
// would not accept the same value twice
- v.addProperty(lname, outgoing.wikibaseId)
+ v.property(lname, outgoing.wikibaseId)
linkCache[lname] << outgoing.wikibaseId
// This is the same in a form of a string, for ES to index since ES
does not index SET properties
- v.setProperty(lname+"_", linkCache[lname].join(' '))
+ v.property(lname+"_", linkCache[lname].join(' '))
}
}
@@ -614,11 +634,11 @@
case 'wikibase-property': return
allowLink?"P"+value['numeric-id']:null
case 'wikibase-item': return
allowLink?"Q"+value['numeric-id']:null
default:
- log.info "Unknown datatype on
${getCurrentVertex()?.wikibaseId}: ${claim.datatype}. Skipping."
+ log.warn "Unknown datatype on
${getCurrentVertex()?.wikibaseId}: ${claim.datatype}. Skipping."
return null
}
} catch(Exception e) {
- log.info "Value parsing failed on
${getCurrentVertex()?.wikibaseId}: ${claim.datatype} with: $e"
+ log.warn "Value parsing failed on
${getCurrentVertex()?.wikibaseId}: ${claim.datatype} with: $e"
return null
}
}
@@ -717,7 +737,9 @@
def rank = s.addProperty(mgmt, 'rank', Boolean.class)
def hash = s.addProperty(mgmt, "contentHash", String.class)
def etype = s.addProperty(mgmt, 'edgeType', String.class)
- def label = s.addEdgeLabel(mgmt, name, rank, wikibaseId, hash, etype)
+ def datatype = addProperty(mgmt, 'datatype', String.class)
+ def label = s.addEdgeLabel(mgmt, name, rank, wikibaseId, hash, etype,
datatype)
+
s.addVIndex(mgmt, label, "by_rank"+name, rank)
s.addVIndex(mgmt, label, "by_hash"+name, hash, wikibaseId)
diff --git a/src/main/groovy/org/wikidata/gremlin/Schema.groovy
b/src/main/groovy/org/wikidata/gremlin/Schema.groovy
index b6aac20..ddc67db 100644
--- a/src/main/groovy/org/wikidata/gremlin/Schema.groovy
+++ b/src/main/groovy/org/wikidata/gremlin/Schema.groovy
@@ -39,7 +39,7 @@
def wikibaseId = addProperty(mgmt, 'wikibaseId', String.class)
def vid = addProperty(mgmt, 'vid', String.class)
def type = addProperty(mgmt, 'type', String.class)
- addProperty(mgmt, 'datatype', String.class)
+ def datatype = addProperty(mgmt, 'datatype', String.class)
addProperty(mgmt, 'stub', Boolean.class)
addProperty(mgmt, 'lastrevid', Integer.class)
// There are two nodes with the special_value_node parameter: 'unknown',
'no_value'. They are used to
@@ -50,7 +50,7 @@
def etype = addProperty(mgmt, 'edgeType', String.class)
def rank = addProperty(mgmt, 'rank', Boolean.class)
def hash = addProperty(mgmt, "contentHash", String.class)
- def claims = addEdgeLabel(mgmt, 'claim', prop, wikibaseId, hash)
+ def claims = addEdgeLabel(mgmt, 'claim', prop, wikibaseId, hash, datatype)
addIndex(mgmt, 'by_wikibaseId', Vertex.class, [wikibaseId], true)
addIndex(mgmt, 'by_wikibaseIdE', Edge.class, [wikibaseId]) // may not be
needed
@@ -63,7 +63,7 @@
addIndex(mgmt, 'by_Ehash', Edge.class, [hash])
// ??? Maybe not needed anymore
- //addVIndex(mgmt, claims, 'by_claims', rank, prop, hash)
+ addVIndex(mgmt, claims, 'by_claims', rank, prop, hash)
// Mixed indexes
// Index for P123link values - using P123link_ for now, no direct support
for SET properties
--
To view, visit https://gerrit.wikimedia.org/r/183317
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic07366dc430925180632b17d51af03564727a53a
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/gremlin
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
