This is an automated email from the ASF dual-hosted git repository. bchapuis pushed a commit to branch wikidata in repository https://gitbox.apache.org/repos/asf/incubator-baremaps.git
commit 7352fb55fb1dc02d45f3e6741378608e14bde144 Author: Bertil Chapuis <[email protected]> AuthorDate: Wed Nov 8 18:31:12 2023 +0100 Add wikidata processor --- .../java/org/apache/baremaps/workflow/Task.java | 3 +- .../baremaps/workflow/tasks/ImportWikidata.java | 59 ++++++++++++++++++++++ examples/wikidata/workflow.json | 14 +++++ pom.xml | 26 ++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java index 06fdad9e..20cf880e 100644 --- a/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/Task.java @@ -51,7 +51,8 @@ import org.apache.baremaps.workflow.tasks.*; @JsonSubTypes.Type(value = CreateIplocIndex.class, name = "CreateIplocIndex"), @JsonSubTypes.Type(value = ImportDaylightTranslations.class, name = "ImportDaylightTranslations"), - @JsonSubTypes.Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures") + @JsonSubTypes.Type(value = ImportDaylightFeatures.class, name = "ImportDaylightFeatures"), + @JsonSubTypes.Type(value = ImportWikidata.class, name = "ImportWikidata") }) public interface Task { diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java new file mode 100644 index 00000000..4b218b3e --- /dev/null +++ b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ImportWikidata.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.baremaps.workflow.tasks; + +import org.apache.baremaps.workflow.Task; +import org.apache.baremaps.workflow.WorkflowContext; +import org.wikidata.wdtk.datamodel.interfaces.*; +import org.wikidata.wdtk.dumpfiles.DumpProcessingController; + +public record ImportWikidata(Object database) implements Task { + + @Override + public void execute(WorkflowContext context) throws Exception { + DumpProcessingController controller = new DumpProcessingController("wikidatawiki"); + controller.setOfflineMode(false); + controller.registerEntityDocumentProcessor(new CustomProcessor(), null, true); + controller.processMostRecentJsonDump(); + } + + public class CustomProcessor implements EntityDocumentDumpProcessor { + + @Override + public void processItemDocument(ItemDocument itemDocument) {} + + @Override + public void processPropertyDocument(PropertyDocument propertyDocument) {} + + @Override + public void processLexemeDocument(LexemeDocument lexemeDocument) {} + + @Override + public void processMediaInfoDocument(MediaInfoDocument mediaInfoDocument) {} + + @Override + public void open() { + + } + + @Override + public void close() { + + } + } +} diff --git a/examples/wikidata/workflow.json b/examples/wikidata/workflow.json new file mode 100644 index 00000000..77ed204c --- /dev/null +++ b/examples/wikidata/workflow.json @@ -0,0 +1,14 @@ +{ + "steps": [ + { + "id": "natural_earth_vector", + "needs": [], + "tasks": [ + { + "type": "ImportWikidata", + "database": "jdbc:postgresql://localhost:5432/baremaps?&user=baremaps&password=baremaps" + } + ] + } + ] +} diff --git a/pom.xml b/pom.xml index 7bb72aff..d87a6a75 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,7 @@ limitations under the License. <version.lib.slf4j>2.0.9</version.lib.slf4j> <version.lib.sqlite>3.43.2.1</version.lib.sqlite> <version.lib.testcontainers>1.19.1</version.lib.testcontainers> + <version.lib.wdtk>0.14.6</version.lib.wdtk> <version.lucene>9.4.2</version.lucene> <version.lucene-spatial>9.4.2</version.lucene-spatial> <version.plugin.jacoco-maven-plugin>0.8.11</version.plugin.jacoco-maven-plugin> @@ -530,6 +531,31 @@ limitations under the License. <groupId>org.testcontainers</groupId> <artifactId>testcontainers</artifactId> </dependency> + <dependency> + <groupId>org.wikidata.wdtk</groupId> + <artifactId>wdtk-datamodel</artifactId> + <version>${version.lib.wdtk}</version> + </dependency> + <dependency> + <groupId>org.wikidata.wdtk</groupId> + <artifactId>wdtk-dumpfiles</artifactId> + <version>${version.lib.wdtk}</version> + </dependency> + <dependency> + <groupId>org.wikidata.wdtk</groupId> + <artifactId>wdtk-rdf</artifactId> + <version>${version.lib.wdtk}</version> + </dependency> + <dependency> + <groupId>org.wikidata.wdtk</groupId> + <artifactId>wdtk-util</artifactId> + <version>${version.lib.wdtk}</version> + </dependency> + <dependency> + <groupId>org.wikidata.wdtk</groupId> + <artifactId>wdtk-wikibaseapi</artifactId> + <version>${version.lib.wdtk}</version> + </dependency> <dependency> <groupId>org.wololo</groupId> <artifactId>flatgeobuf</artifactId>
