belom88 opened a new issue, #1771:
URL: https://github.com/apache/sedona/issues/1771
I have an issue trying to use Apache Sedona on the Python backend:
```
@app.get("/original")
def read_original_dataset():
fileName = "Addresses"
config = (
SedonaContext.builder()
.master("local[*]")
.config(
"spark.jars.packages",
"org.scala-lang.modules:scala-collection-compat_2.12:2.12.0,"
"org.apache.sedona:sedona-spark-3.5_2.12:1.7.0,"
"org.datasyslab:geotools-wrapper:1.7.0-28.5,"
)
.getOrCreate()
)
sedona = SedonaContext.create(config)
df = sedona.read.format("shapefile").load(fileName).alias("df")
df_lng_lat = df.drop("geometry").select(ST_Point(df.LONGITUDE,
df.LATITUDE).alias("geometry"), "*")
# Convert to Pandas DataFrame for GeoPandas compatibility
pdf = df_lng_lat.toPandas()
# pdf['geometry'] = gpd.GeoSeries.from_wkt(pdf['geom'])
gdf = gpd.GeoDataFrame(pdf, geometry='geometry')
# Write to GeoParquet in memory
buffer = io.BytesIO()
gdf.to_parquet(buffer, engine='pyarrow', index=False)
# Return as a binary response
buffer.seek(0)
return Response(content=buffer.read(),
media_type="application/octet-stream")
```
`ST_Point(df.LONGITUDE, df.LATITUDE)` fails with
```
INFO: 127.0.0.1:58898 - "GET /original HTTP/1.1" 500 Internal Server
Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py",
line 409, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py",
line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/fastapi/applications.py",
line 1054, in __call__
await super().__call__(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/applications.py",
line 112, in __call__
await self.middleware_stack(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/middleware/errors.py",
line 187, in __call__
raise exc
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/middleware/errors.py",
line 165, in __call__
await self.app(scope, receive, _send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/middleware/cors.py",
line 93, in __call__
await self.simple_response(scope, receive, send, request_headers=headers)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/middleware/cors.py",
line 144, in simple_response
await self.app(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/middleware/exceptions.py",
line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/_exception_handler.py",
line 53, in wrapped_app
raise exc
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/_exception_handler.py",
line 42, in wrapped_app
await app(scope, receive, sender)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/routing.py",
line 715, in __call__
await self.middleware_stack(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/routing.py",
line 735, in app
await route.handle(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/routing.py",
line 288, in handle
await self.app(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/routing.py",
line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/_exception_handler.py",
line 53, in wrapped_app
raise exc
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/_exception_handler.py",
line 42, in wrapped_app
await app(scope, receive, sender)
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/routing.py",
line 73, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/fastapi/routing.py",
line 301, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/fastapi/routing.py",
line 214, in run_endpoint_function
return await run_in_threadpool(dependant.call, **values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/starlette/concurrency.py",
line 37, in run_in_threadpool
return await anyio.to_thread.run_sync(func)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/anyio/to_thread.py",
line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/anyio/_backends/_asyncio.py",
line 2364, in run_sync_in_worker_thread
return await future
^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/anyio/_backends/_asyncio.py",
line 864, in run
result = context.run(func, *args)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/vbelomestnov/apps/data_validation/sedona_api/main.py", line
85, in read_original_dataset
df_lng_lat = df.drop("geometry").select(ST_Point(df.LONGITUDE,
df.LATITUDE).alias("geometry"), "*")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/sedona/sql/dataframe_api.py",
line 191, in validated_function
return f(*args, **kwargs)
^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/sedona/sql/st_constructors.py",
line 229, in ST_Point
return _call_constructor_function("ST_Point", (x, y))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/vbelomestnov/anaconda3/envs/sedona_1_backend/lib/python3.11/site-packages/sedona/sql/dataframe_api.py",
line 62, in call_sedona_function
raise ValueError(
ValueError: No active spark session was detected. Unable to call sedona
function.
25/01/24 09:29:36 WARN SparkStringUtils: Truncated the string representation
of a plan since it was too large. This behavior can be adjusted by setting
'spark.sql.debug.maxToStringFields'.
```
The same code works in a Jupiter Notebook. Backend sample app and Jupiter
notebook are attached.
[sedona_backend.zip](https://github.com/user-attachments/files/18532773/sedona_backend.zip)
[find_duplicates_notebook.zip](https://github.com/user-attachments/files/18532776/find_duplicates_notebook.zip)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
