Smalyshev has uploaded a new change for review.
https://gerrit.wikimedia.org/r/251571
Change subject: Add option to keep types in Munger
......................................................................
Add option to keep types in Munger
Bug: T115242
Change-Id: If4d3a95c3408daebd6a5f5a52dd7ce05921dd316
---
M dist/src/script/munge.sh
M tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
M tools/src/main/java/org/wikidata/query/rdf/tool/OptionsUtils.java
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
4 files changed, 39 insertions(+), 21 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/71/251571/1
diff --git a/dist/src/script/munge.sh b/dist/src/script/munge.sh
index 7c5d640..5d27a68 100755
--- a/dist/src/script/munge.sh
+++ b/dist/src/script/munge.sh
@@ -19,6 +19,9 @@
esac
done
+# allow extra args
+shift $((OPTIND-1))
+
if [ -z "$FROM" -o ! -f "$FROM" ]
then
echo "Usage: $0 -f <dumpfile> [-d <directory>] [-l languages]"
@@ -36,5 +39,5 @@
CP=lib/wikidata-query-tools-*-jar-with-dependencies.jar
MAIN=org.wikidata.query.rdf.tool.Munge
-java -cp $CP $MAIN --from $FROM --to $LOCATION/$FORMAT $ARGS --chunkSize $CHUNK
+java -cp $CP $MAIN --from $FROM --to $LOCATION/$FORMAT $ARGS --chunkSize
$CHUNK "$@"
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
index ef247d9..8f284f3 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/Munge.java
@@ -270,7 +270,7 @@
private boolean haveNonEntityDataStatements;
/**
* The current entity being read. When we hit a new entity we start
send
- * the old statements to the munger and theyn sync them to next.
+ * the old statements to the munger and then sync them to next.
*/
private String entityId;
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/OptionsUtils.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/OptionsUtils.java
index 64d7402..e0b8901 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/OptionsUtils.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/OptionsUtils.java
@@ -56,6 +56,9 @@
@Option(description = "Skip site links")
boolean skipSiteLinks();
+
+ @Option(description = "Preserve all types")
+ boolean keepTypes();
}
/**
@@ -124,6 +127,9 @@
if (options.singleLabelLanguages() != null) {
munger =
munger.singleLabelMode(splitByComma(options.singleLabelLanguages()));
}
+ if (options.keepTypes()) {
+ munger = munger.keepTypes(true);
+ }
return munger;
}
diff --git a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
index a8a6b6e..a4aeb3a 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/Munger.java
@@ -26,7 +26,6 @@
import org.openrdf.model.vocabulary.XMLSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.wikidata.query.rdf.common.WikibaseDate;
import org.wikidata.query.rdf.common.uri.OWL;
import org.wikidata.query.rdf.common.uri.Ontology;
import org.wikidata.query.rdf.common.uri.Provenance;
@@ -73,6 +72,11 @@
*/
private final boolean removeSiteLinks;
+ /**
+ * True if we want to keep types for Statement and Item.
+ */
+ private boolean keepTypes;
+
public Munger(WikibaseUris uris) {
this(uris, null, null, false);
}
@@ -83,6 +87,16 @@
this.limitLabelLanguages = limitLabelLanguages;
this.singleLabelModeLanguages = singleLabelModeLanguages;
this.removeSiteLinks = removeSiteLinks;
+ }
+
+ /**
+ * Set the keep types parameter.
+ * @param keep
+ * @return
+ */
+ public Munger keepTypes(boolean keep) {
+ keepTypes = keep;
+ return this;
}
/**
@@ -377,7 +391,7 @@
default:
// Noop - fall out is ok as we just remove them.
}
- // All EntityDate statements are removed.
+ // All EntityData statements are removed.
return false;
}
@@ -398,6 +412,9 @@
}
switch (predicate) {
case RDF.TYPE:
+ if (keepTypes) {
+ return true;
+ }
/*
* We don't need wd:Q1 a ontology:Item because its super common
* and not super interesting.
@@ -415,26 +432,9 @@
case SKOS.ALT_LABEL:
return limitLabelLanguage();
case OWL.SAME_AS:
- // TODO: remove when T100463 is fixed
- redirectFix();
return true;
default:
return entityStatementWithUnrecognizedPredicate();
- }
- }
-
- /**
- * Temporary fix for redirects not having revision/timestamp.
- * TODO: remove when T100463 is fixed.
- */
- private void redirectFix() {
- if (revisionId == null) {
- revisionId = new NumericLiteralImpl(1);
- }
- if (lastModified == null) {
- // This is horrible but we don't have better option now than
invent the time
- WikibaseDate wb =
WikibaseDate.fromSecondsSinceEpoch(Calendar.getInstance(TimeZone.getTimeZone("UTC"),
Locale.ROOT).getTimeInMillis() / 1000);
- lastModified = new
LiteralImpl(wb.toString(WikibaseDate.ToStringFormat.DATE_TIME),
XMLSchema.DATETIME);
}
}
@@ -459,6 +459,9 @@
private boolean entityStatementStatement() {
switch (predicate) {
case RDF.TYPE:
+ if (keepTypes) {
+ return true;
+ }
/*
* We don't need s:<uuid> a ontology:Statement because its
super
* common and not super interesting.
@@ -507,6 +510,9 @@
}
switch (predicate) {
case RDF.TYPE:
+ if (keepTypes) {
+ return true;
+ }
/*
* We don't need r:<uuid> a ontology:Reference because its
super
* common and not super interesting.
@@ -549,6 +555,9 @@
}
switch (predicate) {
case RDF.TYPE:
+ if (keepTypes) {
+ return true;
+ }
/*
* We don't need v:<uuid> a ontology:Value because its super
* common and not super interesting.
--
To view, visit https://gerrit.wikimedia.org/r/251571
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If4d3a95c3408daebd6a5f5a52dd7ce05921dd316
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
