This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new f0221db3 [SEDONA-317] Add KeplerGL to pipfile (#891)
f0221db3 is described below
commit f0221db3d6f611865c45caafc7d42e7c4abe91ad
Author: Nilesh Gajwani <[email protected]>
AuthorDate: Mon Jul 10 21:07:14 2023 -0700
[SEDONA-317] Add KeplerGL to pipfile (#891)
---
...eSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb | 356 ++-------------------
binder/Pipfile | 8 +-
2 files changed, 24 insertions(+), 340 deletions(-)
diff --git a/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
b/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
index 7b630bd2..f1d883f5 100644
--- a/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
+++ b/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
@@ -24,7 +24,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@@ -37,7 +37,9 @@
"\n",
"\n",
"from sedona.spark import *\n",
- "from utilities import getConfig"
+ "from utilities import getConfig\n",
+ "\n",
+ "from keplergl import KeplerGl"
]
},
{
@@ -49,47 +51,9 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- ":: loading settings :: url =
jar:file:/Users/nileshgajwani/Desktop/spark/spark-3.4.0-bin-hadoop3/jars/ivy-2.5.1.jar!/org/apache/ivy/core/settings/ivysettings.xml\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Ivy Default Cache set to: /Users/nileshgajwani/.ivy2/cache\n",
- "The jars for the packages stored in: /Users/nileshgajwani/.ivy2/jars\n",
- "org.apache.sedona#sedona-spark-shaded-3.0_2.12 added as a dependency\n",
- "org.datasyslab#geotools-wrapper added as a dependency\n",
- ":: resolving dependencies ::
org.apache.spark#spark-submit-parent-ccdb40df-96c2-42bf-bb17-f6b17333162d;1.0\n",
- "\tconfs: [default]\n",
- "\tfound org.apache.sedona#sedona-spark-shaded-3.0_2.12;1.4.1 in
central\n",
- "\tfound org.datasyslab#geotools-wrapper;1.4.0-28.2 in central\n",
- ":: resolution report :: resolve 98ms :: artifacts dl 2ms\n",
- "\t:: modules in use:\n",
- "\torg.apache.sedona#sedona-spark-shaded-3.0_2.12;1.4.1 from central in
[default]\n",
- "\torg.datasyslab#geotools-wrapper;1.4.0-28.2 from central in
[default]\n",
-
"\t---------------------------------------------------------------------\n",
- "\t| | modules || artifacts
|\n",
- "\t| conf | number| search|dwnlded|evicted||
number|dwnlded|\n",
-
"\t---------------------------------------------------------------------\n",
- "\t| default | 2 | 0 | 0 | 0 || 2 | 0
|\n",
-
"\t---------------------------------------------------------------------\n",
- ":: retrieving ::
org.apache.spark#spark-submit-parent-ccdb40df-96c2-42bf-bb17-f6b17333162d\n",
- "\tconfs: [default]\n",
- "\t0 artifacts copied, 2 already retrieved (0kB/2ms)\n",
- "23/07/03 23:08:06 WARN NativeCodeLoader: Unable to load native-hadoop
library for your platform... using builtin-java classes where applicable\n",
- "Setting default log level to \"WARN\".\n",
- "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use
setLogLevel(newLevel).\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"config = SedonaContext.builder() .\\\n",
" config('spark.jars.packages',\n",
@@ -112,120 +76,9 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "root\n",
- " |-- geometry: geometry (nullable = true)\n",
- " |-- featurecla: string (nullable = true)\n",
- " |-- scalerank: string (nullable = true)\n",
- " |-- LABELRANK: string (nullable = true)\n",
- " |-- SOVEREIGNT: string (nullable = true)\n",
- " |-- SOV_A3: string (nullable = true)\n",
- " |-- ADM0_DIF: string (nullable = true)\n",
- " |-- LEVEL: string (nullable = true)\n",
- " |-- TYPE: string (nullable = true)\n",
- " |-- ADMIN: string (nullable = true)\n",
- " |-- ADM0_A3: string (nullable = true)\n",
- " |-- GEOU_DIF: string (nullable = true)\n",
- " |-- GEOUNIT: string (nullable = true)\n",
- " |-- GU_A3: string (nullable = true)\n",
- " |-- SU_DIF: string (nullable = true)\n",
- " |-- SUBUNIT: string (nullable = true)\n",
- " |-- SU_A3: string (nullable = true)\n",
- " |-- BRK_DIFF: string (nullable = true)\n",
- " |-- NAME: string (nullable = true)\n",
- " |-- NAME_LONG: string (nullable = true)\n",
- " |-- BRK_A3: string (nullable = true)\n",
- " |-- BRK_NAME: string (nullable = true)\n",
- " |-- BRK_GROUP: string (nullable = true)\n",
- " |-- ABBREV: string (nullable = true)\n",
- " |-- POSTAL: string (nullable = true)\n",
- " |-- FORMAL_EN: string (nullable = true)\n",
- " |-- FORMAL_FR: string (nullable = true)\n",
- " |-- NAME_CIAWF: string (nullable = true)\n",
- " |-- NOTE_ADM0: string (nullable = true)\n",
- " |-- NOTE_BRK: string (nullable = true)\n",
- " |-- NAME_SORT: string (nullable = true)\n",
- " |-- NAME_ALT: string (nullable = true)\n",
- " |-- MAPCOLOR7: string (nullable = true)\n",
- " |-- MAPCOLOR8: string (nullable = true)\n",
- " |-- MAPCOLOR9: string (nullable = true)\n",
- " |-- MAPCOLOR13: string (nullable = true)\n",
- " |-- POP_EST: string (nullable = true)\n",
- " |-- POP_RANK: string (nullable = true)\n",
- " |-- GDP_MD_EST: string (nullable = true)\n",
- " |-- POP_YEAR: string (nullable = true)\n",
- " |-- LASTCENSUS: string (nullable = true)\n",
- " |-- GDP_YEAR: string (nullable = true)\n",
- " |-- ECONOMY: string (nullable = true)\n",
- " |-- INCOME_GRP: string (nullable = true)\n",
- " |-- WIKIPEDIA: string (nullable = true)\n",
- " |-- FIPS_10_: string (nullable = true)\n",
- " |-- ISO_A2: string (nullable = true)\n",
- " |-- ISO_A3: string (nullable = true)\n",
- " |-- ISO_A3_EH: string (nullable = true)\n",
- " |-- ISO_N3: string (nullable = true)\n",
- " |-- UN_A3: string (nullable = true)\n",
- " |-- WB_A2: string (nullable = true)\n",
- " |-- WB_A3: string (nullable = true)\n",
- " |-- WOE_ID: string (nullable = true)\n",
- " |-- WOE_ID_EH: string (nullable = true)\n",
- " |-- WOE_NOTE: string (nullable = true)\n",
- " |-- ADM0_A3_IS: string (nullable = true)\n",
- " |-- ADM0_A3_US: string (nullable = true)\n",
- " |-- ADM0_A3_UN: string (nullable = true)\n",
- " |-- ADM0_A3_WB: string (nullable = true)\n",
- " |-- CONTINENT: string (nullable = true)\n",
- " |-- REGION_UN: string (nullable = true)\n",
- " |-- SUBREGION: string (nullable = true)\n",
- " |-- REGION_WB: string (nullable = true)\n",
- " |-- NAME_LEN: string (nullable = true)\n",
- " |-- LONG_LEN: string (nullable = true)\n",
- " |-- ABBREV_LEN: string (nullable = true)\n",
- " |-- TINY: string (nullable = true)\n",
- " |-- HOMEPART: string (nullable = true)\n",
- " |-- MIN_ZOOM: string (nullable = true)\n",
- " |-- MIN_LABEL: string (nullable = true)\n",
- " |-- MAX_LABEL: string (nullable = true)\n",
- " |-- NE_ID: string (nullable = true)\n",
- " |-- WIKIDATAID: string (nullable = true)\n",
- " |-- NAME_AR: string (nullable = true)\n",
- " |-- NAME_BN: string (nullable = true)\n",
- " |-- NAME_DE: string (nullable = true)\n",
- " |-- NAME_EN: string (nullable = true)\n",
- " |-- NAME_ES: string (nullable = true)\n",
- " |-- NAME_FR: string (nullable = true)\n",
- " |-- NAME_EL: string (nullable = true)\n",
- " |-- NAME_HI: string (nullable = true)\n",
- " |-- NAME_HU: string (nullable = true)\n",
- " |-- NAME_ID: string (nullable = true)\n",
- " |-- NAME_IT: string (nullable = true)\n",
- " |-- NAME_JA: string (nullable = true)\n",
- " |-- NAME_KO: string (nullable = true)\n",
- " |-- NAME_NL: string (nullable = true)\n",
- " |-- NAME_PL: string (nullable = true)\n",
- " |-- NAME_PT: string (nullable = true)\n",
- " |-- NAME_RU: string (nullable = true)\n",
- " |-- NAME_SV: string (nullable = true)\n",
- " |-- NAME_TR: string (nullable = true)\n",
- " |-- NAME_VI: string (nullable = true)\n",
- " |-- NAME_ZH: string (nullable = true)\n",
- "\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "23/07/03 23:08:10 WARN package: Truncated the string representation of
a plan since it was too large. This behavior can be adjusted by setting
'spark.sql.debug.maxToStringFields'.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"countries = ShapefileReader.readToGeometryRDD(sc,
\"data/ne_50m_admin_0_countries_lakes/\")\n",
"countries_df = Adapter.toDf(countries, sedona)\n",
@@ -243,29 +96,9 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "root\n",
- " |-- geometry: geometry (nullable = true)\n",
- " |-- scalerank: string (nullable = true)\n",
- " |-- featurecla: string (nullable = true)\n",
- " |-- type: string (nullable = true)\n",
- " |-- name: string (nullable = true)\n",
- " |-- abbrev: string (nullable = true)\n",
- " |-- location: string (nullable = true)\n",
- " |-- gps_code: string (nullable = true)\n",
- " |-- iata_code: string (nullable = true)\n",
- " |-- wikipedia: string (nullable = true)\n",
- " |-- natlscale: string (nullable = true)\n",
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"airports = ShapefileReader.readToGeometryRDD(sc,
\"data/ne_50m_airports/\")\n",
"airports_df = Adapter.toDf(airports, sedona)\n",
@@ -282,7 +115,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -298,17 +131,9 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "[('3.0', '2.12', '1.4.1')]\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"airports_rdd = Adapter.toSpatialRdd(airports_df, \"geometry\")\n",
"# Drop the duplicate name column in countries_df\n",
@@ -345,75 +170,9 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "23/07/03 23:08:12 WARN JoinQuery: UseIndex is true, but no index
exists. Will build index on the fly.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
-
"+--------------------+--------------------+--------------------+--------------------+\n",
- "| country_geom| NAME_EN| airport_geom|
name|\n",
-
"+--------------------+--------------------+--------------------+--------------------+\n",
- "|MULTIPOLYGON (((1...|Taiwan ...|POINT (121.231370...|Taoyuan
...|\n",
- "|MULTIPOLYGON (((5...|Netherlands ...|POINT
(4.76437693...|Schiphol ...|\n",
- "|POLYGON ((103.969...|Singapore ...|POINT
(103.986413...|Singapore Changi ...|\n",
- "|MULTIPOLYGON (((-...|United Kingdom ...|POINT (-0.4531566...|London
Heathrow ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT
(-149.98172...|Anchorage Int'l ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT
(-84.425397...|Hartsfield-Jackso...|\n",
- "|MULTIPOLYGON (((1...|People's Republic...|POINT (116.588174...|Beijing
Capital ...|\n",
- "|MULTIPOLYGON (((-...|Colombia ...|POINT
(-74.143371...|Eldorado Int'l ...|\n",
- "|MULTIPOLYGON (((6...|India ...|POINT
(72.8745639...|Chhatrapati Shiva...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-71.016406...|Gen E L
Logan Int...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT
(-76.668642...|Baltimore-Washing...|\n",
- "|POLYGON ((36.8713...|Egypt ...|POINT (31.3997430...|Cairo
Int'l ...|\n",
- "|POLYGON ((-2.2196...|Morocco ...|POINT
(-7.6632188...|Casablanca-Anfa ...|\n",
- "|MULTIPOLYGON (((-...|Venezuela ...|POINT (-67.005748...|Simon
Bolivar Int...|\n",
- "|MULTIPOLYGON (((2...|South Africa ...|POINT (18.5976565...|Cape
Town Int'l ...|\n",
- "|MULTIPOLYGON (((1...|People's Republic...|POINT
(103.956136...|Chengdushuang Liu...|\n",
- "|MULTIPOLYGON (((6...|India ...|POINT (77.0878362...|Indira
Gandhi Int...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-104.67379...|Denver
Int'l ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT
(-97.040371...|Dallas-Ft. Worth ...|\n",
- "|MULTIPOLYGON (((1...|Thailand ...|POINT (100.602578...|Don
Muang Int'l ...|\n",
-
"+--------------------+--------------------+--------------------+--------------------+\n",
- "only showing top 20 rows\n",
- "\n",
-
"+--------------------+--------------------+--------------------+--------------------+\n",
- "| country_geom| NAME_EN| airport_geom|
name|\n",
-
"+--------------------+--------------------+--------------------+--------------------+\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-80.145258...|Fort
Lauderdale H...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-80.278971...|Miami
Int'l ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-95.333704...|George
Bush Inter...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-90.256693...|New
Orleans Int'l...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-81.307371...|Orlando
Int'l ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-82.534824...|Tampa
Int'l ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-112.01363...|Sky
Harbor Int'l ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-118.40246...|Los
Angeles Int'l...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT (-116.97547...|General
Abelardo ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT
(-97.040371...|Dallas-Ft. Worth ...|\n",
- "|MULTIPOLYGON (((-...|United States of ...|POINT
(-84.425397...|Hartsfield-Jackso...|\n",
- "|POLYGON ((-69.965...|Peru ...|POINT (-77.107565...|Jorge
Chavez ...|\n",
- "|MULTIPOLYGON (((-...|Panama ...|POINT (-79.387134...|Tocumen
Int'l ...|\n",
- "|POLYGON ((-83.157...|Nicaragua ...|POINT (-86.171284...|Augusto
Cesar San...|\n",
- "|MULTIPOLYGON (((-...|Mexico ...|POINT (-96.183570...|Gen.
Heriberto Ja...|\n",
- "|MULTIPOLYGON (((-...|Mexico ...|POINT (-106.27001...|General
Rafael Bu...|\n",
- "|MULTIPOLYGON (((-...|Mexico ...|POINT (-99.754508...|General
Juan N Al...|\n",
- "|MULTIPOLYGON (((-...|Mexico ...|POINT (-99.570649...|Jose
Maria Morelo...|\n",
- "|MULTIPOLYGON (((-...|Mexico ...|POINT (-98.375759...|Puebla
...|\n",
- "|MULTIPOLYGON (((-...|Mexico ...|POINT (-99.082607...|Lic
Benito Juarez...|\n",
-
"+--------------------+--------------------+--------------------+--------------------+\n",
- "only showing top 20 rows\n",
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# The result of SQL API\n",
"result.show()\n",
@@ -430,54 +189,11 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": null,
"metadata": {
"scrolled": true
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "+--------------------+--------------------+------------+\n",
- "| NAME_EN| country_geom|AirportCount|\n",
- "+--------------------+--------------------+------------+\n",
- "|Cuba ...|MULTIPOLYGON (((-...| 1|\n",
- "|Mexico ...|MULTIPOLYGON (((-...| 12|\n",
- "|Panama ...|MULTIPOLYGON (((-...| 1|\n",
- "|Nicaragua ...|POLYGON ((-83.157...| 1|\n",
- "|Honduras ...|MULTIPOLYGON (((-...| 1|\n",
- "|Colombia ...|MULTIPOLYGON (((-...| 4|\n",
- "|United States of ...|MULTIPOLYGON (((-...| 35|\n",
- "|Ecuador ...|MULTIPOLYGON (((-...| 1|\n",
- "|The Bahamas ...|MULTIPOLYGON (((-...| 1|\n",
- "|Peru ...|POLYGON ((-69.965...| 1|\n",
- "|Guatemala ...|POLYGON ((-92.235...| 1|\n",
- "|Canada ...|MULTIPOLYGON (((-...| 15|\n",
- "|Venezuela ...|MULTIPOLYGON (((-...| 3|\n",
- "|Argentina ...|MULTIPOLYGON (((-...| 3|\n",
- "|Bolivia ...|MULTIPOLYGON (((-...| 2|\n",
- "|Paraguay ...|POLYGON ((-58.159...| 1|\n",
- "|Benin ...|POLYGON ((1.62265...| 1|\n",
- "|Guinea ...|POLYGON ((-10.283...| 1|\n",
- "|Chile ...|MULTIPOLYGON (((-...| 5|\n",
- "|Nigeria ...|MULTIPOLYGON (((7...| 3|\n",
- "+--------------------+--------------------+------------+\n",
- "only showing top 20 rows\n",
- "\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\r",
- "[Stage 20:=================================================> (6
+ 1) / 7]\r",
- "\r",
- "
\r"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# result.createOrReplaceTempView(\"result\")\n",
"result2.createOrReplaceTempView(\"result\")\n",
@@ -494,50 +210,14 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"metadata": {},
"outputs": [],
- "source": [
- "try:\n",
- " from keplergl import KeplerGl\n",
- "except ImportError:\n",
- " !pip install keplergl\n",
- " from keplergl import KeplerGl"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "User Guide: https://docs.kepler.gl/docs/keplergl-jupyter\n";
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "9df63933b69f4b18b5c66243b257dd34",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "KeplerGl(config={'version': 'v1', 'config': {'visState': {'filters':
[], 'layers': [{'id': 'ikzru0t', 'type': …"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
"source": [
"df = groupedresult.toPandas()\n",
"gdf = gpd.GeoDataFrame(df,
geometry=\"country_geom\").rename(columns={'country_geom':'geometry'})\n",
"\n",
"map_1 = KeplerGl(data={\"AirportCount\": gdf}, config=getConfig())\n",
- "\n",
"map_1"
]
}
diff --git a/binder/Pipfile b/binder/Pipfile
index 01844df5..47631489 100644
--- a/binder/Pipfile
+++ b/binder/Pipfile
@@ -6,7 +6,7 @@ verify_ssl = true
[dev-packages]
pytest="*"
notebook="==6.0.0"
-jupyter="*"
+jupyter = "*"
mkdocs="*"
pytest-cov = "*"
@@ -16,10 +16,14 @@ shapely="==1.8.4"
geopandas="==0.11.1"
pyspark="==3.3.2"
attrs="*"
-ipykernel = "*"
apache-sedona="==1.4.1"
matplotlib = "*"
descartes = "*"
+keplergl = "==0.3.2"
+ipywidgets = "*"
+jupyterlab-widgets = "*"
+ipykernel = "*"
+jupyterlab = "==3.6.4"
[requires]
python_version = "3.9"
