HIVEMALL-96: Geospatial functions
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/6e077dff Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/6e077dff Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/6e077dff Branch: refs/heads/dev/v0.4.2 Commit: 6e077dffacb35f1bc8e7ea6c8c9681878263ba14 Parents: 8df2e36 Author: Takuya Kitazawa <[email protected]> Authored: Thu Sep 21 10:37:41 2017 +0900 Committer: Takuya Kitazawa <[email protected]> Committed: Fri Sep 22 15:49:02 2017 +0900 ---------------------------------------------------------------------- .../geospatial/HaversineDistanceUDF.java | 110 +++++++++++++ .../java/hivemall/geospatial/Lat2TileYUDF.java | 96 +++++++++++ .../java/hivemall/geospatial/Lon2TileXUDF.java | 96 +++++++++++ .../java/hivemall/geospatial/MapURLUDF.java | 160 +++++++++++++++++++ .../main/java/hivemall/geospatial/TileUDF.java | 102 ++++++++++++ .../java/hivemall/geospatial/TileX2LonUDF.java | 96 +++++++++++ .../java/hivemall/geospatial/TileY2LatUDF.java | 96 +++++++++++ .../utils/geospatial/GeoSpatialUtils.java | 105 ++++++++++++ .../java/hivemall/utils/math/MathUtils.java | 9 +- .../geospatial/HaversineDistanceUDFTest.java | 105 ++++++++++++ .../hivemall/geospatial/Lat2TileYUDFTest.java | 48 ++++++ .../hivemall/geospatial/Lon2TileXUDFTest.java | 48 ++++++ .../hivemall/geospatial/TileX2LonUDFTest.java | 48 ++++++ .../hivemall/geospatial/TileY2LatUDFTest.java | 48 ++++++ .../utils/geospatial/GeoSpatialUtilsTest.java | 50 ++++++ resources/ddl/define-all-as-permanent.hive | 25 +++ resources/ddl/define-all.hive | 25 +++ resources/ddl/define-all.spark | 25 +++ resources/ddl/define-udfs.td.hql | 7 + 19 files changed, 1298 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/HaversineDistanceUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/HaversineDistanceUDF.java b/core/src/main/java/hivemall/geospatial/HaversineDistanceUDF.java new file mode 100644 index 0000000..3d745db --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/HaversineDistanceUDF.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** + * A UDF to return Haversine distance between given two points + * + * @link http://www.movable-type.co.uk/scripts/latlong.html + * @link https://en.wikipedia.org/wiki/Haversine_formula + * @link https://rosettacode.org/wiki/Haversine_formula + */ +@Description( + name = "haversine_distance", + value = "_FUNC_(double lat1, double lon1, double lat2, double lon2, [const boolean mile=false])::double" + + " - return distance between two locations in km [or miles] using `haversine` formula", + extended = "Usage: select latlon_distance(lat1, lon1, lat2, lon2) from ...") +@UDFType(deterministic = true, stateful = false) +public final class HaversineDistanceUDF extends GenericUDF { + + private PrimitiveObjectInspector lat1OI, lon1OI; + private PrimitiveObjectInspector lat2OI, lon2OI; + + private boolean inMiles; + private DoubleWritable result; + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 4 && argOIs.length != 5) { + throw new UDFArgumentException("_FUNC_ takes 4 or 5 arguments: " + argOIs.length); + } + this.lat1OI = HiveUtils.asDoubleCompatibleOI(argOIs[0]); + this.lon1OI = HiveUtils.asDoubleCompatibleOI(argOIs[1]); + this.lat2OI = HiveUtils.asDoubleCompatibleOI(argOIs[2]); + this.lon2OI = HiveUtils.asDoubleCompatibleOI(argOIs[3]); + this.inMiles = (argOIs.length == 5) && HiveUtils.getConstBoolean(argOIs[4]); + + this.result = new DoubleWritable(); + return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector; + } + + @Override + public DoubleWritable evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + Object arg2 = arguments[2].get(); + Object arg3 = arguments[3].get(); + + if (arg0 == null || arg1 == null || arg2 == null || arg3 == null) { + return null; + } + double lat1 = PrimitiveObjectInspectorUtils.getDouble(arg0, lat1OI); + double lon1 = PrimitiveObjectInspectorUtils.getDouble(arg1, lon1OI); + double lat2 = PrimitiveObjectInspectorUtils.getDouble(arg2, lat2OI); + double lon2 = PrimitiveObjectInspectorUtils.getDouble(arg3, lon2OI); + + final double distance; + try { + distance = GeoSpatialUtils.haversineDistance(lat1, lon1, lat2, lon2); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + + if (inMiles) { + double miles = distance / 1.609344d; + result.set(miles); + } else { + result.set(distance); + } + + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "haversine_distance(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/Lat2TileYUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/Lat2TileYUDF.java b/core/src/main/java/hivemall/geospatial/Lat2TileYUDF.java new file mode 100644 index 0000000..3a16293 --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/Lat2TileYUDF.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; +import hivemall.utils.lang.Preconditions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.io.IntWritable; + +/** + * @link http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames + */ +@Description( + name = "lat2tiley", + value = "_FUNC_(double lat, int zoom)::int - Returns the tile number of the given latitude and zoom level") +@UDFType(deterministic = true, stateful = false) +public final class Lat2TileYUDF extends GenericUDF { + + private PrimitiveObjectInspector latOI; + private PrimitiveObjectInspector zoomOI; + + private IntWritable result; + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 2) { + throw new UDFArgumentException("_FUNC_ takes exactly 2 arguments: " + argOIs.length); + } + this.latOI = HiveUtils.asDoubleCompatibleOI(argOIs[0]); + this.zoomOI = HiveUtils.asIntegerOI(argOIs[1]); + + this.result = new IntWritable(); + return PrimitiveObjectInspectorFactory.writableIntObjectInspector; + } + + @Override + public IntWritable evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + + if (arg0 == null) { + return null; + } + if (arg1 == null) { + throw new UDFArgumentException("zoom level should not be null"); + } + + double lat = PrimitiveObjectInspectorUtils.getDouble(arg0, latOI); + int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI); + Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class); + + final int y; + try { + y = GeoSpatialUtils.lat2tiley(lat, zoom); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + + result.set(y); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "lat2tiley(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/Lon2TileXUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/Lon2TileXUDF.java b/core/src/main/java/hivemall/geospatial/Lon2TileXUDF.java new file mode 100644 index 0000000..36103ec --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/Lon2TileXUDF.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; +import hivemall.utils.lang.Preconditions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.io.IntWritable; + +/** + * @link http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames + */ +@Description( + name = "lon2tilex", + value = "_FUNC_(double lon, int zoom)::int - Returns the tile number of the given longitude and zoom level") +@UDFType(deterministic = true, stateful = false) +public final class Lon2TileXUDF extends GenericUDF { + + private PrimitiveObjectInspector lonOI; + private PrimitiveObjectInspector zoomOI; + + private IntWritable result; + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 2) { + throw new UDFArgumentException("_FUNC_ takes exactly 2 arguments: " + argOIs.length); + } + this.lonOI = HiveUtils.asDoubleCompatibleOI(argOIs[0]); + this.zoomOI = HiveUtils.asIntegerOI(argOIs[1]); + + this.result = new IntWritable(); + return PrimitiveObjectInspectorFactory.writableIntObjectInspector; + } + + @Override + public IntWritable evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + + if (arg0 == null) { + return null; + } + if (arg1 == null) { + throw new UDFArgumentException("zoom level should not be null"); + } + + double lon = PrimitiveObjectInspectorUtils.getDouble(arg0, lonOI); + int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI); + Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class); + + final int x; + try { + x = GeoSpatialUtils.lon2tilex(lon, zoom); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + + result.set(x); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "lon2tilex(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/MapURLUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/MapURLUDF.java b/core/src/main/java/hivemall/geospatial/MapURLUDF.java new file mode 100644 index 0000000..8f3b91f --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/MapURLUDF.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.UDFWithOptions; +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; + +import java.util.Arrays; + +import javax.annotation.Nonnull; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.io.Text; + +@Description( + name = "map_url", + value = "_FUNC_(double lat, double lon, int zoom [, const string option]) - Returns a URL string", + extended = "OpenStreetMap: http://tile.openstreetmap.org/${zoom}/${xtile}/${ytile}.png\n"; + + "Google Maps: https://www.google.com/maps/@${lat},${lon},${zoom}z";) +@UDFType(deterministic = true, stateful = false) +public final class MapURLUDF extends UDFWithOptions { + + private PrimitiveObjectInspector latOI; + private PrimitiveObjectInspector lonOI; + private PrimitiveObjectInspector zoomOI; + + private MapType type = MapType.openstreetmap; + private Text result; + + @Override + protected Options getOptions() { + Options opts = new Options(); + opts.addOption("t", "type", true, + "Map type [default: openstreetmap|osm, googlemaps|google]"); + return opts; + } + + @Override + protected CommandLine processOptions(String optionValue) throws UDFArgumentException { + CommandLine cl = parseOptions(optionValue); + + this.type = MapType.resolve(cl.getOptionValue("type", "openstreetmap")); + return cl; + } + + public enum MapType { + openstreetmap, googlemaps; + + @Nonnull + public static final MapType resolve(@Nonnull String type) throws UDFArgumentException { + if ("openstreetmap".equalsIgnoreCase(type) || "osm".equalsIgnoreCase(type)) { + return openstreetmap; + } else if ("googlemaps".equalsIgnoreCase(type) || "google".equalsIgnoreCase(type)) { + return googlemaps; + } else { + throw new UDFArgumentException("Illegal map type: " + type); + } + } + } + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 3 && argOIs.length != 4) { + throw new UDFArgumentException("_FUNC_ takes 3 or 4 arguments: " + argOIs.length); + } + if (argOIs.length == 4) { + String opts = HiveUtils.getConstString(argOIs[3]); + processOptions(opts); + } + + this.latOI = HiveUtils.asDoubleCompatibleOI(argOIs[0]); + this.lonOI = HiveUtils.asDoubleCompatibleOI(argOIs[1]); + this.zoomOI = HiveUtils.asIntegerOI(argOIs[2]); + + this.result = new Text(); + return PrimitiveObjectInspectorFactory.writableStringObjectInspector; + } + + @Override + public Text evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + Object arg2 = arguments[2].get(); + + if (arg0 == null || arg1 == null) { + return null; + } + if (arg2 == null) { + throw new UDFArgumentException("zoom level is null"); + } + + double lat = PrimitiveObjectInspectorUtils.getDouble(arg0, latOI); + double lon = PrimitiveObjectInspectorUtils.getDouble(arg1, lonOI); + int zoom = PrimitiveObjectInspectorUtils.getInt(arg2, zoomOI); + + result.set(toMapURL(lat, lon, zoom, type)); + return result; + } + + @Nonnull + private static String toMapURL(double lat, double lon, int zoom, @Nonnull MapType type) + throws UDFArgumentException { + if (type == MapType.openstreetmap) {// http://tile.openstreetmap.org/${zoom}/${xtile}/${ytile}.png + if (zoom < 0 || zoom > 19) { + throw new UDFArgumentException( + "Illegal zoom level. Supported zoom level for openstreetmap is [0,19]: " + zoom); + } + final int xtile, ytile; + try { + xtile = GeoSpatialUtils.lon2tilex(lon, zoom); + ytile = GeoSpatialUtils.lat2tiley(lat, zoom); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + return "http://tile.openstreetmap.org/"; + Integer.toString(zoom) + '/' + + Integer.toString(xtile) + '/' + Integer.toString(ytile) + ".png"; + } else if (type == MapType.googlemaps) {// https://www.google.com/maps/@${lat},${lon},${zoom}z + if (zoom < 0 || zoom > 21) { + throw new UDFArgumentException( + "Illegal zoom level. Supported zoom level for Google Maps is [0,21]: " + zoom); + } + return "https://www.google.com/maps/@"; + Double.toString(lat) + ',' + + Double.toString(lon) + ',' + Integer.toString(zoom) + 'z'; + } else { + throw new UDFArgumentException("Unexpected map type: " + type); + } + } + + @Override + public String getDisplayString(String[] children) { + return "map_url(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/TileUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/TileUDF.java b/core/src/main/java/hivemall/geospatial/TileUDF.java new file mode 100644 index 0000000..164aa56 --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/TileUDF.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; +import hivemall.utils.lang.Preconditions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.io.LongWritable; + +/** + * @link http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames + */ +@Description( + name = "tile", + value = "_FUNC_(double lat, double lon, int zoom)::bigint - Returns a tile number 2^2n where n is zoom level.\n" + + "_FUNC_(lat,lon,zoom) = xtile(lon,zoom) + ytile(lat,zoom) * 2^zoom", + extended = "refer http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames for detail") +@UDFType(deterministic = true, stateful = false) +public final class TileUDF extends GenericUDF { + + private PrimitiveObjectInspector latOI; + private PrimitiveObjectInspector lonOI; + private PrimitiveObjectInspector zoomOI; + + private LongWritable result; + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 3) { + throw new UDFArgumentException("_FUNC_ takes exactly 3 arguments: " + argOIs.length); + } + this.latOI = HiveUtils.asDoubleCompatibleOI(argOIs[0]); + this.lonOI = HiveUtils.asDoubleCompatibleOI(argOIs[1]); + this.zoomOI = HiveUtils.asIntegerOI(argOIs[2]); + + this.result = new LongWritable(); + return PrimitiveObjectInspectorFactory.writableLongObjectInspector; + } + + @Override + public LongWritable evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + Object arg2 = arguments[2].get(); + + if (arg0 == null || arg1 == null) { + return null; + } + if (arg2 == null) { + throw new UDFArgumentException("zoom level is null"); + } + + double lat = PrimitiveObjectInspectorUtils.getDouble(arg0, latOI); + double lon = PrimitiveObjectInspectorUtils.getDouble(arg1, lonOI); + int zoom = PrimitiveObjectInspectorUtils.getInt(arg2, zoomOI); + Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class); + + final long tile; + try { + tile = GeoSpatialUtils.tile(lat, lon, zoom); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + + result.set(tile); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "tile(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/TileX2LonUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/TileX2LonUDF.java b/core/src/main/java/hivemall/geospatial/TileX2LonUDF.java new file mode 100644 index 0000000..5979227 --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/TileX2LonUDF.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; +import hivemall.utils.lang.Preconditions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** + * @link http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames + */ +@Description( + name = "tilex2lon", + value = "_FUNC_(int x, int zoom)::double - Returns longitude of the given tile x and zoom level") +@UDFType(deterministic = true, stateful = false) +public final class TileX2LonUDF extends GenericUDF { + + private PrimitiveObjectInspector xOI; + private PrimitiveObjectInspector zoomOI; + + private DoubleWritable result; + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 2) { + throw new UDFArgumentException("_FUNC_ takes exactly 2 arguments: " + argOIs.length); + } + this.xOI = HiveUtils.asIntegerOI(argOIs[0]); + this.zoomOI = HiveUtils.asIntegerOI(argOIs[1]); + + this.result = new DoubleWritable(); + return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector; + } + + @Override + public DoubleWritable evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + + if (arg0 == null) { + return null; + } + if (arg1 == null) { + throw new UDFArgumentException("zoom level should not be null"); + } + + int x = PrimitiveObjectInspectorUtils.getInt(arg0, xOI); + int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI); + Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class); + + final double lon; + try { + lon = GeoSpatialUtils.tilex2lon(x, zoom); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + + result.set(lon); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "tilex2lon(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/geospatial/TileY2LatUDF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/geospatial/TileY2LatUDF.java b/core/src/main/java/hivemall/geospatial/TileY2LatUDF.java new file mode 100644 index 0000000..f2e6da0 --- /dev/null +++ b/core/src/main/java/hivemall/geospatial/TileY2LatUDF.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import hivemall.utils.geospatial.GeoSpatialUtils; +import hivemall.utils.hadoop.HiveUtils; +import hivemall.utils.lang.Preconditions; + +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; + +/** + * @link http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames + */ +@Description( + name = "tiley2lat", + value = "_FUNC_(int y, int zoom)::double - Returns latitude of the given tile y and zoom level") +@UDFType(deterministic = true, stateful = false) +public final class TileY2LatUDF extends GenericUDF { + + private PrimitiveObjectInspector yOI; + private PrimitiveObjectInspector zoomOI; + + private DoubleWritable result; + + @Override + public ObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + if (argOIs.length != 2) { + throw new UDFArgumentException("_FUNC_ takes exactly 2 arguments: " + argOIs.length); + } + this.yOI = HiveUtils.asIntegerOI(argOIs[0]); + this.zoomOI = HiveUtils.asIntegerOI(argOIs[1]); + + this.result = new DoubleWritable(); + return PrimitiveObjectInspectorFactory.writableDoubleObjectInspector; + } + + @Override + public DoubleWritable evaluate(DeferredObject[] arguments) throws HiveException { + Object arg0 = arguments[0].get(); + Object arg1 = arguments[1].get(); + + if (arg0 == null) { + return null; + } + if (arg1 == null) { + throw new UDFArgumentException("zoom level should not be null"); + } + + int y = PrimitiveObjectInspectorUtils.getInt(arg0, yOI); + int zoom = PrimitiveObjectInspectorUtils.getInt(arg1, zoomOI); + Preconditions.checkArgument(zoom >= 0, "Invalid zoom level", UDFArgumentException.class); + + final double lat; + try { + lat = GeoSpatialUtils.tiley2lat(y, zoom); + } catch (IllegalArgumentException ex) { + throw new UDFArgumentException(ex); + } + + result.set(lat); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return "tiley2lat(" + Arrays.toString(children) + ")"; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/utils/geospatial/GeoSpatialUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/utils/geospatial/GeoSpatialUtils.java b/core/src/main/java/hivemall/utils/geospatial/GeoSpatialUtils.java new file mode 100644 index 0000000..a534f09 --- /dev/null +++ b/core/src/main/java/hivemall/utils/geospatial/GeoSpatialUtils.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.utils.geospatial; + +import static hivemall.utils.math.MathUtils.sec; +import static java.lang.Math.PI; +import static java.lang.Math.atan; +import static java.lang.Math.atan2; +import static java.lang.Math.cos; +import static java.lang.Math.floor; +import static java.lang.Math.log; +import static java.lang.Math.pow; +import static java.lang.Math.sin; +import static java.lang.Math.sinh; +import static java.lang.Math.sqrt; +import static java.lang.Math.tan; +import static java.lang.Math.toDegrees; +import static java.lang.Math.toRadians; + +import javax.annotation.Nonnegative; + +public final class GeoSpatialUtils { + + public static final double MAX_LATITUDE = 85.0511d; + public static final double MIN_LATITUDE = -85.0511d; + + private GeoSpatialUtils() {} + + public static int lon2tilex(final double lon, @Nonnegative final int zoom) { + if (lon < -180.d || lon > 180.d) { + throw new IllegalArgumentException("Longitude must be in range [-180,+180]: " + lon); + } + return (int) floor((lon + 180.d) / 360.d * (1 << zoom)); + } + + public static int lat2tiley(final double lat, @Nonnegative final int zoom) { + if (lat < MIN_LATITUDE || lat > MAX_LATITUDE) { + throw new IllegalArgumentException("Latitude must be in range [-85.0511,+85.0511]: " + + lat + "\nSee http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames";); + } + double lat_rad = toRadians(lat); + int n = 1 << zoom; + return (int) floor((1.d - log(tan(lat_rad) + sec(lat_rad)) / PI) / 2.d * n); + } + + public static double tilex2lon(final int x, @Nonnegative final int zoom) { + return x / pow(2.d, zoom) * 360.d - 180.d; + } + + public static double tiley2lat(final int y, @Nonnegative final int zoom) { + double n = PI - (2.d * PI * y) / pow(2.d, zoom); + return toDegrees(atan(sinh(n))); + } + + /** + * @link https://en.wikipedia.org/wiki/Tiled_web_map#Tile_numbering_schemes + */ + public static long tile(final double lat, final double lon, @Nonnegative final int zoom) { + int xtile = lon2tilex(lon, zoom); + int ytile = lat2tiley(lat, zoom); + long n = 1L << zoom; // 2^z + return xtile + (n * ytile); + } + + public static int tiles(final int zoom) { + return 1 << (zoom * 2); // 2^2z + } + + /** + * Return a Haversine distance in Kilometers between two points. + * + * @link http://www.movable-type.co.uk/scripts/latlong.html + * @link http://rosettacode.org/wiki/Haversine_formula#Java + * @return distance between two points in Kilometers + */ + public static double haversineDistance(final double lat1, final double lon1, final double lat2, + final double lon2) { + double R = 6371.0d; // Radius of the earth in Km + double dLat = toRadians(lat2 - lat1); // deg2rad below + double dLon = toRadians(lon2 - lon1); + double sinDLat = sin(dLat / 2.d); + double sinDLon = sin(dLon / 2.d); + double a = sinDLat * sinDLat + cos(toRadians(lat1)) * cos(toRadians(lat2)) * sinDLon + * sinDLon; + double c = 2.d * atan2(sqrt(a), sqrt(1.d - a)); + return R * c; // Distance in Km + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/main/java/hivemall/utils/math/MathUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/utils/math/MathUtils.java b/core/src/main/java/hivemall/utils/math/MathUtils.java index c748749..7a6f53f 100644 --- a/core/src/main/java/hivemall/utils/math/MathUtils.java +++ b/core/src/main/java/hivemall/utils/math/MathUtils.java @@ -43,6 +43,13 @@ public final class MathUtils { private MathUtils() {} /** + * @return secant 1 / cos(d) + */ + public static double sec(final double d) { + return 1.d / Math.cos(d); + } + + /** * Returns a bit mask for the specified number of bits. */ public static int bitMask(final int numberOfBits) { @@ -110,7 +117,7 @@ public final class MathUtils { * in GPU Computing Gems, volume 2, 2010. The source code is available <a * href="http://gpucomputing.net/?q=node/1828";>here</a>. * </p> - * + * * @param x the value * @return t such that x = erf(t) */ http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java b/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java new file mode 100644 index 0000000..af13c6f --- /dev/null +++ b/core/src/test/java/hivemall/geospatial/HaversineDistanceUDFTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Assert; +import org.junit.Test; + +public class HaversineDistanceUDFTest { + + @Test + public void testKilometers1() throws HiveException, IOException { + HaversineDistanceUDF udf = new HaversineDistanceUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector}); + + // Tokyo + double lat1 = 35.6833d, lon1 = 139.7667d; + // Osaka + double lat2 = 34.6603d, lon2 = 135.5232d; + + DoubleWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(lat1), + new DeferredJavaObject(lon1), new DeferredJavaObject(lat2), + new DeferredJavaObject(lon2)}); + Assert.assertEquals(402.092d, result1.get(), 0.001d); + + udf.close(); + } + + @Test + public void testKilometers2() throws HiveException, IOException { + HaversineDistanceUDF udf = new HaversineDistanceUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + ObjectInspectorUtils.getConstantObjectInspector( + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, false)}); + + // Tokyo + double lat1 = 35.6833d, lon1 = 139.7667d; + // Osaka + double lat2 = 34.6603d, lon2 = 135.5232d; + + DoubleWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(lat1), + new DeferredJavaObject(lon1), new DeferredJavaObject(lat2), + new DeferredJavaObject(lon2)}); + Assert.assertEquals(402.092d, result1.get(), 0.001d); + + udf.close(); + } + + @Test + public void testMiles() throws HiveException, IOException { + HaversineDistanceUDF udf = new HaversineDistanceUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + ObjectInspectorUtils.getConstantObjectInspector( + PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, true)}); + + // Tokyo + double lat1 = 35.6833d, lon1 = 139.7667d; + // Osaka + double lat2 = 34.6603d, lon2 = 135.5232d; + + DoubleWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(lat1), + new DeferredJavaObject(lon1), new DeferredJavaObject(lat2), + new DeferredJavaObject(lon2), new DeferredJavaObject(true)}); + Assert.assertEquals(249.84d, result1.get(), 0.1d); + + udf.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java b/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java new file mode 100644 index 0000000..8fa1ada --- /dev/null +++ b/core/src/test/java/hivemall/geospatial/Lat2TileYUDFTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.junit.Assert; +import org.junit.Test; + +public class Lat2TileYUDFTest { + + @Test + public void testEvaluate() throws HiveException, IOException { + Lat2TileYUDF udf = new Lat2TileYUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector}); + + IntWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(49.60055d), + new DeferredJavaObject(13)}); + Assert.assertEquals(2792, result1.get()); + + udf.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java b/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java new file mode 100644 index 0000000..cd82826 --- /dev/null +++ b/core/src/test/java/hivemall/geospatial/Lon2TileXUDFTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.junit.Assert; +import org.junit.Test; + +public class Lon2TileXUDFTest { + + @Test + public void testEvaluate() throws HiveException, IOException { + Lon2TileXUDF udf = new Lon2TileXUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector}); + + IntWritable result1 = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(11.01296d), + new DeferredJavaObject(13)}); + Assert.assertEquals(4346, result1.get()); + + udf.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java b/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java new file mode 100644 index 0000000..ce2b0c9 --- /dev/null +++ b/core/src/test/java/hivemall/geospatial/TileX2LonUDFTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TileX2LonUDFTest { + + @Test + public void testEvaluate() throws IOException, HiveException { + TileX2LonUDF udf = new TileX2LonUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector}); + + DoubleWritable result = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(3551), + new DeferredJavaObject(13)}); + Assert.assertEquals(-23.95019531d, result.get(), 0.001); + + udf.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java b/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java new file mode 100644 index 0000000..5f4b516 --- /dev/null +++ b/core/src/test/java/hivemall/geospatial/TileY2LatUDFTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.geospatial; + +import java.io.IOException; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TileY2LatUDFTest { + + @Test + public void testEvaluate() throws IOException, HiveException { + TileY2LatUDF udf = new TileY2LatUDF(); + udf.initialize(new ObjectInspector[] { + PrimitiveObjectInspectorFactory.javaIntObjectInspector, + PrimitiveObjectInspectorFactory.javaIntObjectInspector}); + + DoubleWritable result = udf.evaluate(new DeferredObject[] {new DeferredJavaObject(503), + new DeferredJavaObject(14)}); + Assert.assertEquals(83.99996604d, result.get(), 0.001); + + udf.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/core/src/test/java/hivemall/utils/geospatial/GeoSpatialUtilsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/utils/geospatial/GeoSpatialUtilsTest.java b/core/src/test/java/hivemall/utils/geospatial/GeoSpatialUtilsTest.java new file mode 100644 index 0000000..b590c93 --- /dev/null +++ b/core/src/test/java/hivemall/utils/geospatial/GeoSpatialUtilsTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.utils.geospatial; + +import org.junit.Assert; +import org.junit.Test; + +public class GeoSpatialUtilsTest { + + @Test + public void testTile() { + double[] lat_array = new double[] {GeoSpatialUtils.MIN_LATITUDE, 0.d, + GeoSpatialUtils.MAX_LATITUDE}; + double[] lon_array = new double[] {-180.d, 0.d, 180.d}; + for (double lat : lat_array) { + for (double lon : lon_array) { + Assert.assertTrue( + String.format("lat=%s, lon=%s, tile=%s", lat, lon, + GeoSpatialUtils.tile(lat, lon, 4)), GeoSpatialUtils.tile(lat, lon, 4) >= 0); + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testTileInvalidLat1() { + GeoSpatialUtils.tile(GeoSpatialUtils.MIN_LATITUDE - 0.1, 0, 4); + } + + @Test(expected = IllegalArgumentException.class) + public void testTileInvalidLat2() { + GeoSpatialUtils.tile(GeoSpatialUtils.MAX_LATITUDE + 0.1, 0, 4); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/resources/ddl/define-all-as-permanent.hive ---------------------------------------------------------------------- diff --git a/resources/ddl/define-all-as-permanent.hive b/resources/ddl/define-all-as-permanent.hive index 9b2f67a..ee36635 100644 --- a/resources/ddl/define-all-as-permanent.hive +++ b/resources/ddl/define-all-as-permanent.hive @@ -563,6 +563,31 @@ CREATE FUNCTION train_ffm as 'hivemall.fm.FieldAwareFactorizationMachineUDTF' US DROP FUNCTION IF EXISTS ffm_predict; CREATE FUNCTION ffm_predict as 'hivemall.fm.FFMPredictUDF' USING JAR '${hivemall_jar}'; +--------------------------- +-- Geo-Spatial functions -- +--------------------------- + +DROP FUNCTION IF EXISTS tile; +CREATE FUNCTION tile as 'hivemall.geospatial.TileUDF' USING JAR '${hivemall_jar}'; + +DROP FUNCTION IF EXISTS map_url; +CREATE FUNCTION map_url as 'hivemall.geospatial.MapURLUDF' USING JAR '${hivemall_jar}'; + +DROP FUNCTION IF EXISTS lat2tiley; +CREATE FUNCTION lat2tiley as 'hivemall.geospatial.Lat2TileYUDF' USING JAR '${hivemall_jar}'; + +DROP FUNCTION IF EXISTS lon2tilex; +CREATE FUNCTION lon2tilex as 'hivemall.geospatial.Lon2TileXUDF' USING JAR '${hivemall_jar}'; + +DROP FUNCTION IF EXISTS tilex2lon; +CREATE FUNCTION tilex2lon as 'hivemall.geospatial.TileX2LonUDF' USING JAR '${hivemall_jar}'; + +DROP FUNCTION IF EXISTS tiley2lat; +CREATE FUNCTION tiley2lat as 'hivemall.geospatial.TileY2LatUDF' USING JAR '${hivemall_jar}'; + +DROP FUNCTION IF EXISTS haversine_distance; +CREATE FUNCTION haversine_distance as 'hivemall.geospatial.HaversineDistanceUDF' USING JAR '${hivemall_jar}'; + ---------------------------- -- Smile related features -- ---------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/resources/ddl/define-all.hive ---------------------------------------------------------------------- diff --git a/resources/ddl/define-all.hive b/resources/ddl/define-all.hive index ae91aa2..022d821 100644 --- a/resources/ddl/define-all.hive +++ b/resources/ddl/define-all.hive @@ -559,6 +559,31 @@ create temporary function train_ffm as 'hivemall.fm.FieldAwareFactorizationMachi drop temporary function ffm_predict; create temporary function ffm_predict as 'hivemall.fm.FFMPredictUDF'; +--------------------------- +-- Geo-Spatial functions -- +--------------------------- + +drop temporary function if exists tile; +create temporary function tile as 'hivemall.geospatial.TileUDF'; + +drop temporary function if exists map_url; +create temporary function map_url as 'hivemall.geospatial.MapURLUDF'; + +drop temporary function if exists lat2tiley; +create temporary function lat2tiley as 'hivemall.geospatial.Lat2TileYUDF'; + +drop temporary function if exists lon2tilex; +create temporary function lon2tilex as 'hivemall.geospatial.Lon2TileXUDF'; + +drop temporary function if exists tilex2lon; +create temporary function tilex2lon as 'hivemall.geospatial.TileX2LonUDF'; + +drop temporary function if exists tiley2lat; +create temporary function tiley2lat as 'hivemall.geospatial.TileY2LatUDF'; + +drop temporary function if exists haversine_distance; +create temporary function haversine_distance as 'hivemall.geospatial.HaversineDistanceUDF'; + ---------------------------- -- Smile related features -- ---------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/resources/ddl/define-all.spark ---------------------------------------------------------------------- diff --git a/resources/ddl/define-all.spark b/resources/ddl/define-all.spark index 39a0480..14c07a6 100644 --- a/resources/ddl/define-all.spark +++ b/resources/ddl/define-all.spark @@ -463,6 +463,31 @@ sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS train_fm") sqlContext.sql("CREATE TEMPORARY FUNCTION train_fm AS 'hivemall.fm.FactorizationMachineUDTF'") /** + * Geo Spatial Functions + */ + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS tile") +sqlContext.sql("CREATE TEMPORARY FUNCTION tile AS 'hivemall.geospatial.TileUDF'") + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS map_url") +sqlContext.sql("CREATE TEMPORARY FUNCTION map_url AS 'hivemall.geospatial.MapURLUDF'") + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS lat2tiley") +sqlContext.sql("CREATE TEMPORARY FUNCTION lat2tiley AS 'hivemall.geospatial.Lat2TileYUDF'") + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS lon2tilex") +sqlContext.sql("CREATE TEMPORARY FUNCTION lon2tilex AS 'hivemall.geospatial.Lon2TileXUDF'") + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS tilex2lon") +sqlContext.sql("CREATE TEMPORARY FUNCTION tilex2lon AS 'hivemall.geospatial.TileX2LonUDF'") + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS tiley2lat") +sqlContext.sql("CREATE TEMPORARY FUNCTION tiley2lat AS 'hivemall.geospatial.TileY2LatUDF'") + +sqlContext.sql("DROP TEMPORARY FUNCTION IF EXISTS haversine_distance") +sqlContext.sql("CREATE TEMPORARY FUNCTION haversine_distance AS 'hivemall.geospatial.HaversineDistanceUDF'") + +/** * Smile related features */ http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/6e077dff/resources/ddl/define-udfs.td.hql ---------------------------------------------------------------------- diff --git a/resources/ddl/define-udfs.td.hql b/resources/ddl/define-udfs.td.hql index f11298c..0e704c7 100644 --- a/resources/ddl/define-udfs.td.hql +++ b/resources/ddl/define-udfs.td.hql @@ -145,6 +145,13 @@ create temporary function rf_ensemble as 'hivemall.smile.tools.RandomForestEnsem create temporary function guess_attribute_types as 'hivemall.smile.tools.GuessAttributesUDF'; create temporary function to_ordered_list as 'hivemall.tools.list.UDAFToOrderedList'; create temporary function singularize as 'hivemall.tools.text.SingularizeUDF'; +create temporary function tile as 'hivemall.geospatial.TileUDF'; +create temporary function map_url as 'hivemall.geospatial.MapURLUDF'; +create temporary function lat2tiley as 'hivemall.geospatial.Lat2TileYUDF'; +create temporary function lon2tilex as 'hivemall.geospatial.Lon2TileXUDF'; +create temporary function tilex2lon as 'hivemall.geospatial.TileX2LonUDF'; +create temporary function tiley2lat as 'hivemall.geospatial.TileY2LatUDF'; +create temporary function haversine_distance as 'hivemall.geospatial.HaversineDistanceUDF'; -- NLP features create temporary function tokenize_ja as 'hivemall.nlp.tokenizer.KuromojiUDF';
![http://tile.openstreetmap.org/${zoom}/${xtile}/${ytile}.png\n"](https://tile.openstreetmap.org/${zoom}/${xtile}/${ytile}.png\n"){kind=link}
