This is an automated email from the ASF dual-hosted git repository. upthewaterspout pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/geode-examples.git
The following commit(s) were added to refs/heads/develop by this push: new 80066fe GEODE-3907: Adding an example of lucene spatial querying 80066fe is described below commit 80066fe44d92d6d4606cb141b9faff8716e97b16 Author: Dan Smith <upthewatersp...@apache.org> AuthorDate: Tue Oct 17 15:59:46 2017 -0700 GEODE-3907: Adding an example of lucene spatial querying Adding an example of using the LuceneSerializer API to enable spatial queries on geode objects. --- luceneSpatial/README.md | 59 ++++++++++++++++++ settings.gradle => luceneSpatial/build.gradle | 23 +++---- luceneSpatial/scripts/start.gfsh | 29 +++++++++ luceneSpatial/scripts/stop.gfsh | 18 ++++++ .../geode/examples/luceneSpatial/Example.java | 72 ++++++++++++++++++++++ .../examples/luceneSpatial/SpatialHelper.java | 60 ++++++++++++++++++ .../geode/examples/luceneSpatial/TrainStop.java | 48 +++++++++++++++ .../luceneSpatial/TrainStopSerializer.java | 49 +++++++++++++++ .../geode/examples/luceneSpatial/ExampleTest.java | 37 +++++++++++ .../examples/luceneSpatial/SpatialHelperTest.java | 60 ++++++++++++++++++ .../luceneSpatial/TrainStopSerializerTest.java | 34 ++++++++++ settings.gradle | 1 + 12 files changed, 477 insertions(+), 13 deletions(-) diff --git a/luceneSpatial/README.md b/luceneSpatial/README.md new file mode 100644 index 0000000..ef7a988 --- /dev/null +++ b/luceneSpatial/README.md @@ -0,0 +1,59 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +# Geode Lucene Spatial Indexing Example + +This examples demonstrates how to use Geode's LuceneSerializer and LuceneQueryProvider APIs +to customize how Geode data is stored and indexed in Lucene. + +In this example two servers host a partitioned region that stores train station stop information, +including GPS coordinates. The region has lucene index that allows spatial queries to be performed +against the data. The example shows how to do a spatial query to find nearby train stations. + +This example assumes that Java and Geode are installed. + +## Set up the Lucene index and region +1. Set directory ```geode-examples/luceneSpatial``` to be the +current working directory. +Each step in this example specifies paths relative to that directory. + +2. Build the example + + $ ../gradlew build + +3. Run a script that starts a locator and two servers, creates a Lucene index +called ```simpleIndex``` with a custom LuceneSerializer that indexes spatial data. The script +then creates the ```example-region``` region. + + $ gfsh run --file=scripts/start.gfsh + +4. Run the example. This program adds data to the example-region, and then looks +for train stations with a 1 mile of a specific GPS coordinate. Look at Example.java to see +what this program does. + + + $ ../gradlew run + + +5. Shut down the cluster + + $ gfsh run --file=scripts/stop.gfsh + +6. Clean up any generated directories and files so this example can be rerun. + + $ ../gradlew cleanServer + diff --git a/settings.gradle b/luceneSpatial/build.gradle similarity index 75% copy from settings.gradle copy to luceneSpatial/build.gradle index 2c89886..133e947 100644 --- a/settings.gradle +++ b/luceneSpatial/build.gradle @@ -14,17 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -rootProject.name = 'geode-examples' +dependencies { + compile "org.apache.geode:geode-lucene:$geodeVersion" + compile "org.apache.lucene:lucene-spatial-extras:6.4.1" +} -include 'replicated' -include 'partitioned' -include 'queries' -include 'lucene' -include 'loader' -include 'putall' -include 'clientSecurity' -include 'functions' -include 'persistence' -include 'writer' -include 'listener' -include 'async' +task copyDependencies(type:Copy) { + into "$buildDir/libs" + from configurations['runtime'] +} + +build.dependsOn(copyDependencies) diff --git a/luceneSpatial/scripts/start.gfsh b/luceneSpatial/scripts/start.gfsh new file mode 100644 index 0000000..82eb78a --- /dev/null +++ b/luceneSpatial/scripts/start.gfsh @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +start locator --name=locator --bind-address=127.0.0.1 + +set variable --name=STAR --value=* +start server --name=server1 --locators=127.0.0.1[10334] --server-port=0 --classpath=../build/libs/${STAR} --enable-time-statistics --statistic-archive-file=lucene1.gfs +start server --name=server2 --locators=127.0.0.1[10334] --server-port=0 --classpath=../build/libs/${STAR} --enable-time-statistics --statistic-archive-file=lucene2.gfs + +## Create a lucene index with our custom serializer +create lucene index --name=simpleIndex --region=example-region --field=name --serializer=org.apache.geode.examples.luceneSpatial.TrainStopSerializer + +create region --name=example-region --type=PARTITION + +list members +describe region --name=example-region diff --git a/luceneSpatial/scripts/stop.gfsh b/luceneSpatial/scripts/stop.gfsh new file mode 100644 index 0000000..9281b31 --- /dev/null +++ b/luceneSpatial/scripts/stop.gfsh @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +connect --locator=127.0.0.1[10334] +shutdown --include-locators=true \ No newline at end of file diff --git a/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/Example.java b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/Example.java new file mode 100644 index 0000000..bf02bd1 --- /dev/null +++ b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/Example.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.geode.cache.Region; +import org.apache.geode.cache.client.ClientCache; +import org.apache.geode.cache.client.ClientCacheFactory; +import org.apache.geode.cache.client.ClientRegionShortcut; +import org.apache.geode.cache.lucene.LuceneQuery; +import org.apache.geode.cache.lucene.LuceneQueryException; +import org.apache.geode.cache.lucene.LuceneService; +import org.apache.geode.cache.lucene.LuceneServiceProvider; + +public class Example { + public static void main(String[] args) throws InterruptedException, LuceneQueryException { + // connect to the locator using default port 10334 + ClientCache cache = new ClientCacheFactory().addPoolLocator("127.0.0.1", 10334) + .set("log-level", "WARN").create(); + + // create a local region that matches the server region + Region<String, TrainStop> region = + cache.<String, TrainStop>createClientRegionFactory(ClientRegionShortcut.PROXY) + .create("example-region"); + + + LuceneService luceneService = LuceneServiceProvider.get(cache); + // Add some entries into the region + putEntries(luceneService, region); + findNearbyTrainStops(luceneService); + cache.close(); + } + + public static void findNearbyTrainStops(LuceneService luceneService) + throws InterruptedException, LuceneQueryException { + LuceneQuery<Integer, TrainStop> query = + luceneService.createLuceneQueryFactory().create("simpleIndex", "example-region", + index -> SpatialHelper.findWithin(-122.8515139, 45.5099231, 0.25)); + + Collection<TrainStop> results = query.findValues(); + System.out.println("Found stops: " + results); + } + + public static void putEntries(LuceneService luceneService, Map<String, TrainStop> region) + throws InterruptedException { + region.put("Elmonica/SW 170th Ave", + new TrainStop("Elmonica/SW 170th Ave", -122.85146341202486, 45.509962691078009)); + region.put("Willow Creek/SW 185th Ave TC", + new TrainStop("Willow Creek/SW 185th Ave TC", -122.87021024485213, 45.517251954169652)); + region.put("Merlo Rd/SW 158th Ave", + new TrainStop("Merlo Rd/SW 158th Ave", -122.84216239020598, 45.505240564251949)); + + // Lucene indexing happens asynchronously, so wait for + // the entries to be in the lucene index. + luceneService.waitUntilFlushed("simpleIndex", "example-region", 1, TimeUnit.MINUTES); + } +} diff --git a/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/SpatialHelper.java b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/SpatialHelper.java new file mode 100644 index 0000000..8f67747 --- /dev/null +++ b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/SpatialHelper.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import static org.locationtech.spatial4j.distance.DistanceUtils.EARTH_MEAN_RADIUS_MI; + +import org.apache.lucene.document.Field; +import org.apache.lucene.search.Query; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; +import org.apache.lucene.spatial.vector.PointVectorStrategy; +import org.locationtech.spatial4j.context.SpatialContext; +import org.locationtech.spatial4j.distance.DistanceUtils; +import org.locationtech.spatial4j.shape.Point; +import org.locationtech.spatial4j.shape.impl.GeoCircle; +import org.locationtech.spatial4j.shape.impl.PointImpl; + +public class SpatialHelper { + private static final SpatialContext CONTEXT = SpatialContext.GEO; + private static final PointVectorStrategy STRATEGY = + new PointVectorStrategy(CONTEXT, "location", PointVectorStrategy.DEFAULT_FIELDTYPE); + + /** + * Return a lucene query that finds all points within the given radius from the given point + */ + public static Query findWithin(double longitude, double latitude, double radiusMiles) { + // Covert the radius in miles to a radius in degrees + double radiusDEG = DistanceUtils.dist2Degrees(radiusMiles, EARTH_MEAN_RADIUS_MI); + + // Create a query that looks for all points within a circle around the given point + SpatialArgs args = new SpatialArgs(SpatialOperation.IsWithin, + new GeoCircle(createPoint(longitude, latitude), radiusDEG, CONTEXT)); + return STRATEGY.makeQuery(args); + } + + /** + * Return a list of fields that should be added to lucene document to index the given point + */ + public static Field[] getIndexableFields(double longitude, double latitude) { + Point point = createPoint(longitude, latitude); + return STRATEGY.createIndexableFields(point); + } + + private static Point createPoint(double longitude, double latitude) { + return new PointImpl(longitude, latitude, CONTEXT); + } + +} diff --git a/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/TrainStop.java b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/TrainStop.java new file mode 100644 index 0000000..76da515 --- /dev/null +++ b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/TrainStop.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import java.io.Serializable; + +public class TrainStop implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private double latitude; + private double longitude; + + public TrainStop(String name, double longitude, double latitude) { + this.name = name; + this.longitude = longitude; + this.latitude = latitude; + } + + public String getName() { + return name; + } + + public double getLatitude() { + return latitude; + } + + public double getLongitude() { + return longitude; + } + + @Override + public String toString() { + return "TrainStop [name=" + name + ", location=" + longitude + ", " + latitude + "]"; + } +} diff --git a/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/TrainStopSerializer.java b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/TrainStopSerializer.java new file mode 100644 index 0000000..1114726 --- /dev/null +++ b/luceneSpatial/src/main/java/org/apache/geode/examples/luceneSpatial/TrainStopSerializer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import java.util.Collection; +import java.util.Collections; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.spatial.vector.PointVectorStrategy; + +import org.apache.geode.cache.lucene.LuceneIndex; +import org.apache.geode.cache.lucene.LuceneSerializer; + +/** + * LuceneSerializer that converts train stops into lucene documents with the gps coordinates indexed + * using lucene's {@link PointVectorStrategy} + */ +public class TrainStopSerializer implements LuceneSerializer<TrainStop> { + @Override + public Collection<Document> toDocuments(LuceneIndex index, TrainStop value) { + + Document doc = new Document(); + // Index the name of the train stop + doc.add(new TextField("name", value.getName(), Field.Store.NO)); + + Field[] fields = SpatialHelper.getIndexableFields(value.getLongitude(), value.getLatitude()); + + for (Field field : fields) { + doc.add(field); + } + + return Collections.singleton(doc); + } + +} diff --git a/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/ExampleTest.java b/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/ExampleTest.java new file mode 100644 index 0000000..925de87 --- /dev/null +++ b/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/ExampleTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import org.apache.geode.cache.lucene.LuceneService; + +public class ExampleTest { + + @Test + public void testPutEntries() throws InterruptedException { + LuceneService service = mock(LuceneService.class); + Map<String, TrainStop> region = new HashMap<String, TrainStop>(); + Example.putEntries(service, region); + assertEquals(3, region.size()); + + } +} diff --git a/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/SpatialHelperTest.java b/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/SpatialHelperTest.java new file mode 100644 index 0000000..05758cb --- /dev/null +++ b/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/SpatialHelperTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.RAMDirectory; +import org.junit.Test; + +public class SpatialHelperTest { + + @Test + public void queryFindsADocumentThatWasAdded() throws IOException { + + // Create an in memory lucene index to add a document to + RAMDirectory directory = new RAMDirectory(); + IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig()); + + // Add a document to the lucene index + Document document = new Document(); + document.add(new TextField("name", "name", Field.Store.YES)); + Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231); + for (Field field : fields) { + document.add(field); + } + writer.addDocument(document); + writer.commit(); + + + // Make sure a findWithin query locates the document + Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1); + SearcherManager searcherManager = new SearcherManager(writer, null); + IndexSearcher searcher = searcherManager.acquire(); + TopDocs results = searcher.search(query, 100); + assertEquals(1, results.totalHits); + } +} diff --git a/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/TrainStopSerializerTest.java b/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/TrainStopSerializerTest.java new file mode 100644 index 0000000..5bb772b --- /dev/null +++ b/luceneSpatial/src/test/java/org/apache/geode/examples/luceneSpatial/TrainStopSerializerTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.examples.luceneSpatial; + +import static org.junit.Assert.assertEquals; + +import java.util.Collection; + +import org.apache.lucene.document.Document; +import org.junit.Test; + +public class TrainStopSerializerTest { + + @Test + public void serializerReturnsSingleDocument() { + TrainStopSerializer serializer = new TrainStopSerializer(); + Collection<Document> documents = + serializer.toDocuments(null, new TrainStop("here", -122.8515139, 45.5099231)); + + assertEquals(1, documents.size()); + } +} diff --git a/settings.gradle b/settings.gradle index 2c89886..e43f7f8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,3 +28,4 @@ include 'persistence' include 'writer' include 'listener' include 'async' +include 'luceneSpatial' -- To stop receiving notification emails like this one, please contact ['"commits@geode.apache.org" <commits@geode.apache.org>'].