yitao-li commented on a change in pull request #525:
URL: https://github.com/apache/incubator-sedona/pull/525#discussion_r634579782
##########
File path: sql/src/test/scala/org/apache/sedona/sql/adapterTestScala.scala
##########
@@ -198,5 +199,51 @@ class adapterTestScala extends TestBaseScala with Matchers
with GivenWhenThen{
val spatialDf = Adapter.toDf(spatialRDD, sparkSession)
assert(spatialDf.schema.fields(1).name == "LST")
}
+
+ it("can convert spatial RDD with user data to a valid Dataframe") {
Review comment:
I think this should cover the case of spatial RDD -> dataframe
##########
File path: sql/src/test/scala/org/apache/sedona/sql/adapterTestScala.scala
##########
@@ -198,5 +199,51 @@ class adapterTestScala extends TestBaseScala with Matchers
with GivenWhenThen{
val spatialDf = Adapter.toDf(spatialRDD, sparkSession)
assert(spatialDf.schema.fields(1).name == "LST")
}
+
+ it("can convert spatial RDD with user data to a valid Dataframe") {
+ val srcDF = sparkSession.sql("select
ST_PointFromText('40.7128,-74.0060', ',') as geom, \"attr1\" as attr1,
\"attr2\" as attr2")
+ val rdd = Adapter.toSpatialRdd(srcDF, "geom")
+ val df = Adapter.toDf(rdd, Seq("attr1", "attr2"), sparkSession)
+ df.unpersist(true)
+ // verify the resulting Spark dataframe can be successfully evaluated
repeatedly
+ for (_ <- 1 to 5) {
+ val rows = df.collect
+ assert(rows.length == 1)
+ val geom = rows(0).get(0).asInstanceOf[Point]
+ assert(geom.getX == 40.7128)
+ assert(geom.getY == -74.006)
+ assert(rows(0).get(1).asInstanceOf[String] == "attr1")
+ assert(rows(0).get(2).asInstanceOf[String] == "attr2")
+ }
+ }
+
+ it("can convert spatial pair RDD with user data to a valid Dataframe") {
Review comment:
I think this should cover the case of pair RDD -> dataframe
##########
File path: sql/src/main/scala/org/apache/sedona/sql/utils/Adapter.scala
##########
@@ -166,8 +169,9 @@ object Adapter {
if (fieldNames != null && fieldNames.nonEmpty) {
val userData = "" + geom.getUserData.asInstanceOf[String]
val fields = userData.split("\t")
-// geom.setUserData(null) // Set to null will lead to the null pointer
exception of the previous line. Not sure why.
- (Seq(geom), fields)
Review comment:
I guess while we are at it, we should also address this issue here as
well (even though it's not as bad as a null pointer exception) -- also, see
test case with pairrdd that should cover this code path
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
