This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 811889461 [SEDONA-629] Change RS Functions to return struct instead of
array type (#1524)
811889461 is described below
commit 811889461e97bf8bae4654d1d527f8da100e40fb
Author: Feng Zhang <[email protected]>
AuthorDate: Thu Jul 18 08:51:36 2024 -0700
[SEDONA-629] Change RS Functions to return struct instead of array type
(#1524)
* [SEDONA-629] Change RS Functions to return struct instead of array type
* refactor RS_SummaryStatsAll
* refactor RS_ZonalStatsAll
* refactor RS_GeoTransform
* modify document
* update the document for RS_GeoTransform
---
docs/api/sql/Raster-operators.md | 70 ++---
.../expressions/raster/RasterAccessors.scala | 14 -
.../expressions/raster/RasterBandAccessors.scala | 21 --
.../expressions/raster/RasterFunctions.scala | 259 ++++++++++++++++
.../org/apache/sedona/sql/rasteralgebraTest.scala | 328 ++++++++++++++-------
5 files changed, 509 insertions(+), 183 deletions(-)
diff --git a/docs/api/sql/Raster-operators.md b/docs/api/sql/Raster-operators.md
index f7dbb719c..08b2d6295 100644
--- a/docs/api/sql/Raster-operators.md
+++ b/docs/api/sql/Raster-operators.md
@@ -452,14 +452,14 @@ SELECT RS_GeoReferrence(ST_MakeEmptyRaster(1, 3, 4,
100.0, 200.0,2.0, -3.0, 0.1,
### RS_GeoTransform
-Introduction: Returns an array of parameters that represent the
GeoTransformation of the raster. The array contains the following values:
+Introduction: Returns a struct of parameters that represent the
GeoTransformation of the raster. The struct has the following schema:
-- 0: pixel width along west-east axis (x-axis)
-- 1: pixel height along north-south axis (y-axis)
-- 2: Rotation of the raster
-- 3: Angular separation between x-axis and y-axis
-- 4: X ordinate of upper-left coordinate
-- 5: Y ordinate of upper-left coordinate
+- magnitudeI: size of a pixel along the transformed i axis
+- magnitudeJ: size of a pixel along the transformed j axis
+- thetaI: angle by which the raster is rotated (Radians positive clockwise)
+- thetaIJ: angle from transformed i axis to transformed j axis (Radians
positive counter-clockwise)
+- offsetX: X ordinate of the upper-left corner of the upper-left pixel
+- offsetY: Y ordinate of the upper-left corner of the upper-left pixel
!!!note
Refer to [this
image](https://www.researchgate.net/figure/Relation-between-the-cartesian-axes-x-y-and-i-j-axes-of-the-pixels_fig3_313860913)
for a clear understanding between i & j axis and x & y-axis.
@@ -479,7 +479,7 @@ SELECT RS_GeoTransform(
Output:
```
-[2.23606797749979, 2.23606797749979, -1.1071487177940904, -2.214297435588181,
1.0, 2.0]
+{2.23606797749979, 2.23606797749979, -1.1071487177940904, -2.214297435588181,
1.0, 2.0}
```
### RS_Height
@@ -1094,7 +1094,7 @@ Output:
### RS_SummaryStatsAll
-Introduction: Returns summary stats consisting of count, sum, mean, stddev,
min, max for a given band in raster. If band is not specified then it defaults
to `1`.
+Introduction: Returns summary stats struct consisting of count, sum, mean,
stddev, min, max for a given band in raster. If band is not specified then it
defaults to `1`.
!!!Note
If excludeNoDataValue is set `true` then it will only count pixels with
value not equal to the nodata value of the raster.
@@ -1122,7 +1122,7 @@ SELECT RS_SummaryStatsAll(RS_MakeEmptyRaster(2, 5, 5, 0,
0, 1, -1, 0, 0, 0), 1,
Output:
```
-25.0, 204.0, 8.16, 9.4678403028357, 0.0, 25.0
+{25.0, 204.0, 8.16, 9.4678403028357, 0.0, 25.0}
```
SQL Example
@@ -1134,7 +1134,7 @@ SELECT RS_SummaryStatsAll(RS_MakeEmptyRaster(2, 5, 5, 0,
0, 1, -1, 0, 0, 0), 1)
Output:
```
-14.0, 204.0, 14.571428571428571, 11.509091348732502, 1.0, 25.0
+{14.0, 204.0, 14.571428571428571, 11.509091348732502, 1.0, 25.0}
```
### RS_ZonalStats
@@ -1207,17 +1207,17 @@ Output:
### RS_ZonalStatsAll
-Introduction: Returns an array of statistic values, where each statistic is
computed over a region defined by the `zone` geometry. The array contains the
following values:
+Introduction: Returns a struct of statistic values, where each statistic is
computed over a region defined by the `zone` geometry. The struct has the
following schema:
-- 0: Count of the pixels.
-- 1: Sum of the pixel values.
-- 2: Arithmetic mean.
-- 3: Median.
-- 4: Mode.
-- 5: Standard deviation.
-- 6: Variance.
-- 7: Minimum value of the zone.
-- 8: Maximum value of the zone.
+- count: Count of the pixels.
+- sum: Sum of the pixel values.
+- mean: Arithmetic mean.
+- median: Median.
+- mode: Mode.
+- stddev: Standard deviation.
+- variance: Variance.
+- min: Minimum value of the zone.
+- max: Maximum value of the zone.
!!!note
If the coordinate reference system (CRS) of the input `zone` geometry
differs from that of the `raster`, then `zone` will be transformed to match the
CRS of the `raster` before computation.
@@ -1258,7 +1258,7 @@ RS_ZonalStatsAll(rast1, geom1, 1, false)
Output:
```
-[184792.0, 1.0690406E7, 57.851021689230684, 0.0, 0.0, 92.13277429243035,
8488.448098819916, 0.0, 255.0]
+{184792.0, 1.0690406E7, 57.851021689230684, 0.0, 0.0, 92.13277429243035,
8488.448098819916, 0.0, 255.0}
```
SQL Example
@@ -1270,7 +1270,7 @@ RS_ZonalStatsAll(rast2, geom2, 1, true)
Output:
```
-[14184.0, 3213526.0, 226.55992667794473, 255.0, 255.0, 74.87605357255357,
5606.423398599913, 1.0, 255.0]
+{14184.0, 3213526.0, 226.55992667794473, 255.0, 255.0, 74.87605357255357,
5606.423398599913, 1.0, 255.0}
```
## Raster Predicates
@@ -1541,18 +1541,18 @@ Output (Shown as heatmap):
### RS_MetaData
-Introduction: Returns the metadata of the raster as an array of double. The
array contains the following values:
+Introduction: Returns the metadata of the raster as a struct. The struct has
the following schema:
-- 0: upper left x coordinate of the raster, in terms of CRS units
-- 1: upper left y coordinate of the raster, in terms of CRS units
-- 2: width of the raster, in terms of pixels
-- 3: height of the raster, in terms of pixels
-- 4: ScaleX: the scaling factor in the x direction
-- 5: ScaleY: the scaling factor in the y direction
-- 6: skew in x direction (rotation x)
-- 7: skew in y direction (rotation y)
-- 8: srid of the raster
-- 9: number of bands
+- upperLeftX: upper left x coordinate of the raster, in terms of CRS units
+- upperLeftX: upper left y coordinate of the raster, in terms of CRS units
+- gridWidth: width of the raster, in terms of pixels
+- gridHeight: height of the raster, in terms of pixels
+- scaleX: ScaleX: the scaling factor in the x direction
+- scaleY: ScaleY: the scaling factor in the y direction
+- skewX: skew in x direction (rotation x)
+- skewY: skew in y direction (rotation y)
+- srid: srid of the raster
+- numSampleDimensions: number of bands
For more information about ScaleX, ScaleY, SkewX, SkewY, please refer to the
[Affine Transformations](Raster-affine-transformation.md) section.
@@ -1569,7 +1569,7 @@ SELECT RS_MetaData(raster) FROM raster_table
Output:
```
-[-1.3095817809482181E7, 4021262.7487925636, 512.0, 517.0, 72.32861272132695,
-72.32861272132695, 0.0, 0.0, 3857.0, 1.0]
+{-1.3095817809482181E7, 4021262.7487925636, 512.0, 517.0, 72.32861272132695,
-72.32861272132695, 0.0, 0.0, 3857.0, 1.0}
```
### RS_NormalizeAll
diff --git
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterAccessors.scala
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterAccessors.scala
index 350401303..6c4973f08 100644
---
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterAccessors.scala
+++
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterAccessors.scala
@@ -38,13 +38,6 @@ case class RS_SRID(inputExpressions: Seq[Expression])
}
}
-case class RS_Metadata(inputExpressions: Seq[Expression])
- extends InferredExpression(RasterAccessors.metadata _) {
- protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) =
{
- copy(inputExpressions = newChildren)
- }
-}
-
case class RS_Width(inputExpressions: Seq[Expression])
extends InferredExpression(RasterAccessors.getWidth _) {
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) =
{
@@ -103,13 +96,6 @@ case class RS_Rotation(inputExpressions: Seq[Expression])
}
}
-case class RS_GeoTransform(inputExpressions: Seq[Expression])
- extends InferredExpression(RasterAccessors.getGeoTransform _) {
- protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) =
{
- copy(inputExpressions = newChildren)
- }
-}
-
case class RS_SkewX(inputExpressions: Seq[Expression])
extends InferredExpression(RasterAccessors.getSkewX _) {
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]):
Expression = {
diff --git
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterBandAccessors.scala
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterBandAccessors.scala
index 1ea82cd5f..3108be64f 100644
---
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterBandAccessors.scala
+++
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterBandAccessors.scala
@@ -59,17 +59,6 @@ case class RS_ZonalStats(inputExpressions: Seq[Expression])
}
}
-case class RS_ZonalStatsAll(inputExpressions: Seq[Expression])
- extends InferredExpression(
- inferrableFunction2(RasterBandAccessors.getZonalStatsAll),
- inferrableFunction4(RasterBandAccessors.getZonalStatsAll),
- inferrableFunction3(RasterBandAccessors.getZonalStatsAll),
- inferrableFunction5(RasterBandAccessors.getZonalStatsAll)) {
- protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) =
{
- copy(inputExpressions = newChildren)
- }
-}
-
case class RS_SummaryStats(inputExpressions: Seq[Expression])
extends InferredExpression(
inferrableFunction2(RasterBandAccessors.getSummaryStats),
@@ -80,16 +69,6 @@ case class RS_SummaryStats(inputExpressions: Seq[Expression])
}
}
-case class RS_SummaryStatsAll(inputExpressions: Seq[Expression])
- extends InferredExpression(
- inferrableFunction1(RasterBandAccessors.getSummaryStatsAll),
- inferrableFunction2(RasterBandAccessors.getSummaryStatsAll),
- inferrableFunction3(RasterBandAccessors.getSummaryStatsAll)) {
- protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) =
{
- copy(inputExpressions = newChildren)
- }
-}
-
case class RS_Band(inputExpressions: Seq[Expression])
extends InferredExpression(RasterBandAccessors.getBand _) {
diff --git
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterFunctions.scala
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterFunctions.scala
new file mode 100644
index 000000000..971be31ae
--- /dev/null
+++
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/raster/RasterFunctions.scala
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.sedona_sql.expressions.raster
+
+import org.apache.sedona.common.raster.{RasterAccessors, RasterBandAccessors}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes,
Expression}
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.sedona_sql.UDT.{GeometryUDT, RasterUDT}
+import
org.apache.spark.sql.sedona_sql.expressions.implicits.InputExpressionEnhancer
+import
org.apache.spark.sql.sedona_sql.expressions.raster.implicits.RasterInputExpressionEnhancer
+import org.apache.spark.sql.types.{AbstractDataType, BooleanType, DataType,
DoubleType, IntegerType, StructField, StructType}
+
+case class RS_Metadata(inputExpressions: Seq[Expression])
+ extends Expression
+ with CodegenFallback
+ with ExpectsInputTypes {
+
+ override def nullable: Boolean = true
+
+ override def dataType: DataType = StructType(
+ Seq(
+ StructField("upperLeftX", DoubleType, nullable = false),
+ StructField("upperLeftY", DoubleType, nullable = false),
+ StructField("gridWidth", DoubleType, nullable = false),
+ StructField("gridHeight", DoubleType, nullable = false),
+ StructField("scaleX", DoubleType, nullable = false),
+ StructField("scaleY", DoubleType, nullable = false),
+ StructField("skewX", DoubleType, nullable = false),
+ StructField("skewY", DoubleType, nullable = false),
+ StructField("srid", DoubleType, nullable = false),
+ StructField("numSampleDimensions", DoubleType, nullable = false)))
+
+ override def eval(input: InternalRow): Any = {
+ // Evaluate the input expressions
+ val rasterGeom = inputExpressions(0).toRaster(input)
+
+ // Check if the raster geometry is null
+ if (rasterGeom == null) {
+ null
+ } else {
+ // Get the metadata using the Java method
+ val metaData = RasterAccessors.metadata(rasterGeom)
+
+ // Create an InternalRow with the metadata
+ InternalRow.fromSeq(metaData.map(_.asInstanceOf[Any]))
+ }
+ }
+
+ override def children: Seq[Expression] = inputExpressions
+
+ protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]):
RS_Metadata = {
+ copy(inputExpressions = newChildren)
+ }
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(RasterUDT)
+}
+
+case class RS_SummaryStatsAll(inputExpressions: Seq[Expression])
+ extends Expression
+ with CodegenFallback
+ with ExpectsInputTypes {
+
+ override def nullable: Boolean = true
+
+ override def dataType: DataType = StructType(
+ Seq(
+ StructField("count", DoubleType, nullable = false),
+ StructField("sum", DoubleType, nullable = false),
+ StructField("mean", DoubleType, nullable = false),
+ StructField("stddev", DoubleType, nullable = false),
+ StructField("min", DoubleType, nullable = false),
+ StructField("max", DoubleType, nullable = false)))
+
+ override def eval(input: InternalRow): Any = {
+ // Evaluate the input expressions
+ val rasterGeom = inputExpressions(0).toRaster(input)
+ val band = if (inputExpressions.length >= 2) {
+ inputExpressions(1).eval(input).asInstanceOf[Int]
+ } else {
+ 1
+ }
+ val noData = if (inputExpressions.length >= 3) {
+ inputExpressions(2).eval(input).asInstanceOf[Boolean]
+ } else {
+ true
+ }
+
+ // Check if the raster geometry is null
+ if (rasterGeom == null) {
+ null
+ } else {
+ val summaryStatsAll = RasterBandAccessors.getSummaryStatsAll(rasterGeom,
band, noData)
+
+ if (summaryStatsAll == null) {
+ return null
+ }
+ // Create an InternalRow with the summaryStatsAll
+ InternalRow.fromSeq(summaryStatsAll.map(_.asInstanceOf[Any]))
+ }
+ }
+
+ override def children: Seq[Expression] = inputExpressions
+
+ protected def withNewChildrenInternal(
+ newChildren: IndexedSeq[Expression]): RS_SummaryStatsAll = {
+ copy(inputExpressions = newChildren)
+ }
+
+ override def inputTypes: Seq[AbstractDataType] = {
+ if (inputExpressions.length == 1) {
+ Seq(RasterUDT)
+ } else if (inputExpressions.length == 2) {
+ Seq(RasterUDT, IntegerType)
+ } else if (inputExpressions.length == 3) {
+ Seq(RasterUDT, IntegerType, BooleanType)
+ } else {
+ Seq(RasterUDT)
+ }
+ }
+}
+
+case class RS_ZonalStatsAll(inputExpressions: Seq[Expression])
+ extends Expression
+ with CodegenFallback
+ with ExpectsInputTypes {
+
+ override def nullable: Boolean = true
+
+ override def dataType: DataType = StructType(
+ Seq(
+ StructField("count", DoubleType, nullable = false),
+ StructField("sum", DoubleType, nullable = false),
+ StructField("mean", DoubleType, nullable = false),
+ StructField("median", DoubleType, nullable = false),
+ StructField("mode", DoubleType, nullable = false),
+ StructField("stddev", DoubleType, nullable = false),
+ StructField("variance", DoubleType, nullable = false),
+ StructField("min", DoubleType, nullable = false),
+ StructField("max", DoubleType, nullable = false)))
+
+ override def eval(input: InternalRow): Any = {
+ // Evaluate the input expressions
+ val rasterGeom = inputExpressions(0).toRaster(input)
+ val roi = if (inputExpressions.length >= 2) {
+ inputExpressions(1).toGeometry(input)
+ } else {
+ null
+ }
+ val band = if (inputExpressions.length >= 3) {
+ inputExpressions(2).eval(input).asInstanceOf[Int]
+ } else {
+ 1
+ }
+ val noData = if (inputExpressions.length >= 4) {
+ inputExpressions(3).eval(input).asInstanceOf[Boolean]
+ } else {
+ true
+ }
+ val lenient = if (inputExpressions.length >= 5) {
+ inputExpressions(4).eval(input).asInstanceOf[Boolean]
+ } else {
+ true
+ }
+
+ // Check if the raster geometry is null
+ if (rasterGeom == null) {
+ null
+ } else {
+ val zonalStatsAll =
+ RasterBandAccessors.getZonalStatsAll(rasterGeom, roi, band, noData,
lenient)
+ // Create an InternalRow with the zonalStatsAll
+ if (zonalStatsAll == null) {
+ return null
+ }
+ InternalRow.fromSeq(zonalStatsAll.map(_.asInstanceOf[Any]))
+ }
+ }
+
+ override def children: Seq[Expression] = inputExpressions
+
+ protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]):
RS_ZonalStatsAll = {
+ copy(inputExpressions = newChildren)
+ }
+
+ override def inputTypes: Seq[AbstractDataType] = {
+ if (inputExpressions.length == 2) {
+ Seq(RasterUDT, GeometryUDT)
+ } else if (inputExpressions.length == 3) {
+ Seq(RasterUDT, GeometryUDT, IntegerType)
+ } else if (inputExpressions.length == 4) {
+ Seq(RasterUDT, GeometryUDT, IntegerType, BooleanType)
+ } else if (inputExpressions.length >= 5) {
+ Seq(RasterUDT, GeometryUDT, IntegerType, BooleanType)
+ } else {
+ Seq(RasterUDT, GeometryUDT)
+ }
+ }
+}
+
+case class RS_GeoTransform(inputExpressions: Seq[Expression])
+ extends Expression
+ with CodegenFallback
+ with ExpectsInputTypes {
+
+ override def nullable: Boolean = true
+
+ override def dataType: DataType = StructType(
+ Seq(
+ StructField("magnitudeI", DoubleType, nullable = false),
+ StructField("magnitudeJ", DoubleType, nullable = false),
+ StructField("thetaI", DoubleType, nullable = false),
+ StructField("thetaIJ", DoubleType, nullable = false),
+ StructField("offsetX", DoubleType, nullable = false),
+ StructField("offsetY", DoubleType, nullable = false)))
+
+ override def eval(input: InternalRow): Any = {
+ // Evaluate the input expressions
+ val rasterGeom = inputExpressions(0).toRaster(input)
+
+ // Check if the raster geometry is null
+ if (rasterGeom == null) {
+ null
+ } else {
+ // Get the metadata using the Java method
+ val geoTransform = RasterAccessors.getGeoTransform(rasterGeom)
+
+ if (geoTransform == null) {
+ return null
+ }
+ // Create an InternalRow with the metadata
+ InternalRow.fromSeq(geoTransform.map(_.asInstanceOf[Any]))
+ }
+ }
+
+ override def children: Seq[Expression] = inputExpressions
+
+ protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]):
RS_GeoTransform = {
+ copy(inputExpressions = newChildren)
+ }
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(RasterUDT)
+}
diff --git
a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
index ba6f8d266..3343ae64e 100644
--- a/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/sql/rasteralgebraTest.scala
@@ -21,8 +21,9 @@ package org.apache.sedona.sql
import org.apache.sedona.common.raster.MapAlgebra
import org.apache.sedona.common.utils.RasterUtils
import org.apache.spark.sql.expressions.Window
-import org.apache.spark.sql.{Row, SaveMode}
-import org.apache.spark.sql.functions.{col, collect_list, expr, lit,
monotonically_increasing_id, row_number}
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
+import org.apache.spark.sql.functions.{col, collect_list, expr, lit,
row_number}
+import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
import org.geotools.coverage.grid.GridCoverage2D
import org.junit.Assert.{assertEquals, assertNotNull, assertNull, assertTrue}
import org.locationtech.jts.geom.{Coordinate, Geometry}
@@ -432,18 +433,41 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
"RS_Metadata(rast_4326) as metadata_4326")
val result = dfResult.first()
assert(result.getInt(0) == 4326)
- val metadata = result.getSeq[Double](1)
- val metadata4326 = result.getSeq[Double](2)
- assert(metadata4326(8) == 4326)
- assert(metadata(0) == metadata4326(0))
- assert(metadata(1) == metadata4326(1))
- assert(metadata(2) == metadata4326(2))
- assert(metadata(3) == metadata4326(3))
- assert(metadata(4) == metadata4326(4))
- assert(metadata(5) == metadata4326(5))
- assert(metadata(6) == metadata4326(6))
- assert(metadata(7) == metadata4326(7))
- assert(metadata(9) == metadata4326(9))
+
+ // Assert SRID
+ assert(result.getInt(0) == 4326)
+
+ // Extract metadata and metadata_4326
+ val metadata = result.getStruct(1)
+ val metadata_4326 = result.getStruct(2)
+
+ // Assert metadata schema
+ val expectedSchema = StructType(
+ Seq(
+ StructField("upperLeftX", DoubleType, nullable = false),
+ StructField("upperLeftY", DoubleType, nullable = false),
+ StructField("gridWidth", DoubleType, nullable = false),
+ StructField("gridHeight", DoubleType, nullable = false),
+ StructField("scaleX", DoubleType, nullable = false),
+ StructField("scaleY", DoubleType, nullable = false),
+ StructField("skewX", DoubleType, nullable = false),
+ StructField("skewY", DoubleType, nullable = false),
+ StructField("srid", DoubleType, nullable = false),
+ StructField("numSampleDimensions", DoubleType, nullable = false)))
+
+ assert(dfResult.schema("metadata").dataType == expectedSchema)
+ assert(dfResult.schema("metadata_4326").dataType == expectedSchema)
+
+ // Assert metadata fields
+ assert(metadata.getDouble(0) == metadata_4326.getDouble(0))
+ assert(metadata.getDouble(1) == metadata_4326.getDouble(1))
+ assert(metadata.getDouble(2) == metadata_4326.getDouble(2))
+ assert(metadata.getDouble(3) == metadata_4326.getDouble(3))
+ assert(metadata.getDouble(4) == metadata_4326.getDouble(4))
+ assert(metadata.getDouble(5) == metadata_4326.getDouble(5))
+ assert(metadata.getDouble(6) == metadata_4326.getDouble(6))
+ assert(metadata.getDouble(7) == metadata_4326.getDouble(7))
+ assert(metadata.getDouble(9) == metadata_4326.getDouble(9))
}
it("Passed RS_SetGeoReference should handle null values") {
@@ -517,11 +541,11 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val expected = 3
assertEquals(expected, actual)
- val actualMetadata =
- resultDf.selectExpr("RS_Metadata(raster)").first().getSeq(0).slice(0,
9)
- val expectedMetadata =
- df.selectExpr("RS_Metadata(emptyRaster)").first().getSeq(0).slice(0, 9)
- assertEquals(expectedMetadata.toString(), actualMetadata.toString())
+ assertRSMetadata(
+ df,
+ "RS_Metadata(emptyRaster) as metadata",
+ resultDf,
+ "RS_Metadata(raster) as metadata")
val actualBandValues = resultDf.selectExpr("RS_BandAsArray(raster,
1)").first().getSeq(0)
val expectedBandValues = df.selectExpr("RS_BandAsArray(emptyRaster,
1)").first().getSeq(0)
@@ -538,10 +562,11 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val expected = 5
assertEquals(expected, actual)
- val actualMetadata =
-
resultDf.selectExpr("RS_Metadata(resultRaster)").first().getSeq(0).slice(0, 9)
- val expectedMetadata =
df.selectExpr("RS_Metadata(raster)").first().getSeq(0).slice(0, 9)
- assertEquals(expectedMetadata.toString(), actualMetadata.toString())
+ assertRSMetadata(
+ resultDf,
+ "RS_Metadata(resultRaster) as metadata",
+ df,
+ "RS_Metadata(raster) as metadata")
}
it("Passed RS_Union") {
@@ -570,10 +595,11 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
var expectedBandValues = df.selectExpr("RS_BandAsArray(raster2,
1)").first().getSeq(0)
assertTrue(expectedBandValues.equals(actualBandValues))
- var actualMetadata =
-
actualDf.selectExpr("RS_Metadata(actualRaster)").first().getSeq(0).slice(0, 9)
- var expectedMetadata =
df.selectExpr("RS_Metadata(raster1)").first().getSeq(0).slice(0, 9)
- assertTrue(expectedMetadata.equals(actualMetadata))
+ assertRSMetadata(
+ actualDf,
+ "RS_Metadata(actualRaster) as metadata",
+ df,
+ "RS_Metadata(raster1) as metadata")
actualDf = df.selectExpr("RS_Union(raster1, raster2, raster3) as
actualRaster")
@@ -588,10 +614,11 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
expectedBandValues = df.selectExpr("RS_BandAsArray(raster3,
1)").first().getSeq(0)
assertTrue(expectedBandValues.equals(actualBandValues))
- actualMetadata =
-
actualDf.selectExpr("RS_Metadata(actualRaster)").first().getSeq(0).slice(0, 9)
- expectedMetadata =
df.selectExpr("RS_Metadata(raster1)").first().getSeq(0).slice(0, 9)
- assertTrue(expectedMetadata.equals(actualMetadata))
+ assertRSMetadata(
+ actualDf,
+ "RS_Metadata(actualRaster) as metadata",
+ df,
+ "RS_Metadata(raster1) as metadata")
actualDf = df.selectExpr(
"RS_Union(raster1, raster2, raster3, raster1, raster2, raster3,
raster1) as actualRaster")
@@ -607,10 +634,11 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
expectedBandValues = df.selectExpr("RS_BandAsArray(raster3,
2)").first().getSeq(0)
assertTrue(expectedBandValues.equals(actualBandValues))
- actualMetadata =
-
actualDf.selectExpr("RS_Metadata(actualRaster)").first().getSeq(0).slice(0, 9)
- expectedMetadata =
df.selectExpr("RS_Metadata(raster1)").first().getSeq(0).slice(0, 9)
- assertTrue(expectedMetadata.equals(actualMetadata))
+ assertRSMetadata(
+ actualDf,
+ "RS_Metadata(actualRaster) as metadata",
+ df,
+ "RS_Metadata(raster1) as metadata")
}
it("Passed RS_AddBand with empty raster") {
@@ -637,10 +665,11 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val expectedBandValues = df.selectExpr("RS_BandAsArray(fromRaster,
1)").first().getSeq(0)
assertTrue(expectedBandValues.equals(actualBandValues))
- val actualMetadata =
-
actualDf.selectExpr("RS_Metadata(actualRaster)").first().getSeq(0).slice(0, 9)
- val expectedMetadata =
df.selectExpr("RS_Metadata(toRaster)").first().getSeq(0).slice(0, 9)
- assertTrue(expectedMetadata.equals(actualMetadata))
+ assertRSMetadata(
+ df,
+ "RS_Metadata(toRaster) as metadata",
+ actualDf,
+ "RS_Metadata(actualRaster) as metadata")
}
it("Passed RS_SetValues with raster and implicit CRS transformation") {
@@ -926,12 +955,32 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
"ST_GeomFromWKT('POLYGON ((-8682522.873537656 4572703.890837922,
-8673439.664183248 4572993.532747675, -8673155.57366801 4563873.2099182755,
-8701890.325907696 4562931.7093397, -8682522.873537656 4572703.890837922))',
3857) as geom")
val clippedDf = df.selectExpr("RS_Clip(raster, 1, geom, 200, false) as
clipped")
- val clippedMetadata =
df.selectExpr("RS_Metadata(raster)").first().getSeq(0).slice(0, 9)
- val originalMetadata =
-
clippedDf.selectExpr("RS_Metadata(clipped)").first().getSeq(0).slice(0, 9)
- assertTrue(originalMetadata.equals(clippedMetadata))
+ // Extract the metadata for the original raster
+ val clippedMetadata = df
+ .selectExpr("RS_Metadata(raster) as metadata")
+ .first()
+ .getStruct(0)
- var actualValues = clippedDf
+ // Extract the metadata for the clipped raster
+ val originalMetadata = clippedDf
+ .selectExpr("RS_Metadata(clipped) as metadata")
+ .first()
+ .getStruct(0)
+
+ // Convert Struct to Seq[Double]
+ def structToSeq(struct: Row): Seq[Double] = {
+ (0 until struct.size).map(struct.getDouble)
+ }
+
+ // Compare the metadata
+ val clippedMetadataSeq = structToSeq(clippedMetadata).slice(0, 9)
+ val originalMetadataSeq = structToSeq(originalMetadata).slice(0, 9)
+ assert(
+ clippedMetadataSeq == originalMetadataSeq,
+ s"Expected: $originalMetadataSeq, but got: $clippedMetadataSeq")
+
+ // Extract the RS_Values for the specified points
+ val actualValues = clippedDf
.selectExpr(
"RS_Values(clipped, " +
"Array(ST_GeomFromWKT('POINT(223802 4.21769e+06)',
26918),ST_GeomFromWKT('POINT(224759 4.20453e+06)', 26918)," +
@@ -952,9 +1001,10 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
"ST_GeomFromWKT('POLYGON ((236722 4204770, 243900 4204770, 243900
4197590, 221170 4197590, 236722 4204770))', 26918) as geom")
val clippedDf = df.selectExpr("RS_Clip(raster, 1, geom, 200, false) as
clipped")
- val clippedMetadata =
df.selectExpr("RS_Metadata(raster)").first().getSeq(0).slice(0, 9)
+ val clippedMetadata =
+
df.selectExpr("RS_Metadata(raster)").first().getStruct(0).toSeq.slice(0, 9)
val originalMetadata =
-
clippedDf.selectExpr("RS_Metadata(clipped)").first().getSeq(0).slice(0, 9)
+
clippedDf.selectExpr("RS_Metadata(clipped)").first().getStruct(0).toSeq.slice(0,
9)
assertTrue(originalMetadata.equals(clippedMetadata))
var actualValues = clippedDf
@@ -1112,7 +1162,7 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
it("Passed RS_GeoTransform") {
val df =
sparkSession.sql("SELECT RS_MakeEmptyRaster(2, 10, 15, 1, 2, 1, -2, 1,
2, 0) as raster")
- val actual = df.selectExpr("RS_GeoTransform(raster)").first().getSeq(0)
+ val actual =
df.selectExpr("RS_GeoTransform(raster)").first().getStruct(0).toSeq.slice(0, 6)
val expected = mutable.ArraySeq(2.23606797749979, 2.23606797749979,
-1.1071487177940904,
-2.214297435588181, 1.0, 2.0)
assertTrue(expected.equals(actual))
@@ -1134,11 +1184,24 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
it("Passed RS_Metadata") {
val df = sparkSession.read.format("binaryFile").load(resourceFolder +
"raster/test1.tiff")
- val result =
df.selectExpr("RS_Metadata(RS_FromGeoTiff(content))").first().getSeq(0)
- assertEquals(10, result.length)
- assertEquals(512.0, result(2), 0.001)
- assertEquals(517.0, result(3), 0.001)
- assertEquals(1.0, result(9), 0.001)
+
+ // Extract the metadata struct
+ val result =
+ df.selectExpr("RS_Metadata(RS_FromGeoTiff(content)) as
metadata").first().getStruct(0)
+
+ // Function to convert Struct to Seq[Double]
+ def structToSeq(struct: Row): Seq[Double] = {
+ (0 until struct.size).map(struct.getDouble)
+ }
+
+ // Convert the struct to a sequence of doubles
+ val metadataSeq = structToSeq(result)
+
+ // Assertions
+ assert(metadataSeq.length == 10)
+ assert(metadataSeq(2) == 512.0)
+ assert(metadataSeq(3) == 517.0)
+ assert(metadataSeq(9) == 1.0)
}
it("Passed RS_MakeEmptyRaster") {
@@ -1153,24 +1216,24 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
.sql(
s"SELECT RS_Metadata(RS_MakeEmptyRaster($numBands, $widthInPixel,
$heightInPixel, $upperLeftX, $upperLeftY, $cellSize))")
.first()
- .getSeq(0)
- assertEquals(numBands, result(9), 0.001)
+ .getStruct(0)
+ assertEquals(numBands, result.getDouble(9), 0.001)
// Test without skewX, skewY, srid
result = sparkSession
.sql(
s"SELECT RS_Metadata(RS_MakeEmptyRaster($numBands, 'I',
$widthInPixel, $heightInPixel, $upperLeftX, $upperLeftY, $cellSize))")
.first()
- .getSeq(0)
- assertEquals(numBands, result(9), 0.001)
+ .getStruct(0)
+ assertEquals(numBands, result.getDouble(9), 0.001)
// Test with integer type input
result = sparkSession
.sql(
s"SELECT RS_Metadata(RS_MakeEmptyRaster($numBands, $widthInPixel,
$heightInPixel, ${upperLeftX.toInt}, ${upperLeftY.toInt}, ${cellSize.toInt}))")
.first()
- .getSeq(0)
- assertEquals(numBands, result(9), 0.001)
+ .getStruct(0)
+ assertEquals(numBands, result.getDouble(9), 0.001)
// Test with skewX, skewY, srid but WITHOUT datatype
val skewX = 0.0
@@ -1180,16 +1243,16 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
.sql(
s"SELECT RS_Metadata(RS_MakeEmptyRaster($numBands, $widthInPixel,
$heightInPixel, $upperLeftX, $upperLeftY, $cellSize, -$cellSize, $skewX,
$skewY, $srid))")
.first()
- .getSeq(0)
- assertEquals(numBands, result(9), 0.001)
+ .getStruct(0)
+ assertEquals(numBands, result.getDouble(9), 0.001)
// Test with skewX, skewY, srid and datatype
result = sparkSession
.sql(
s"SELECT RS_Metadata(RS_MakeEmptyRaster($numBands, 'I',
$widthInPixel, $heightInPixel, $upperLeftX, $upperLeftY, $cellSize, -$cellSize,
$skewX, $skewY, $srid))")
.first()
- .getSeq(0)
- assertEquals(numBands, result(9), 0.001)
+ .getStruct(0)
+ assertEquals(numBands, result.getDouble(9), 0.001)
}
it("Passed RS_MakeRaster") {
@@ -1197,7 +1260,7 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
.format("binaryFile")
.load(resourceFolder + "raster/test1.tiff")
.withColumn("rast", expr("RS_FromGeoTiff(content)"))
- val metadata = df.selectExpr("RS_Metadata(rast)").first().getSeq(0)
+ val metadata =
df.selectExpr("RS_Metadata(rast)").first().getStruct(0).toSeq
val width = metadata(2).asInstanceOf[Double].toInt
val height = metadata(3).asInstanceOf[Double].toInt
val values = Array.tabulate(width * height) { i => i * i }
@@ -1228,7 +1291,8 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
it("Passed RS_BandAsArray") {
val df = sparkSession.read.format("binaryFile").load(resourceFolder +
"raster/test1.tiff")
- val metadata =
df.selectExpr("RS_Metadata(RS_FromGeoTiff(content))").first().getSeq(0)
+ val metadata =
+
df.selectExpr("RS_Metadata(RS_FromGeoTiff(content))").first().getStruct(0).toSeq
val width = metadata(2).asInstanceOf[Double].toInt
val height = metadata(3).asInstanceOf[Double].toInt
val result = df.selectExpr("RS_BandAsArray(RS_FromGeoTiff(content),
1)").first().getSeq(0)
@@ -1464,9 +1528,10 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val expectedBands = 3
assertEquals(expectedBands, actualBands)
- val actualMetadata =
df.selectExpr("RS_Metadata(rasters)").first().getSeq(0).slice(0, 9)
+ val actualMetadata =
+
df.selectExpr("RS_Metadata(rasters)").first().getStruct(0).toSeq.slice(0, 9)
val expectedMetadata =
- dfTest.selectExpr("RS_Metadata(raster)").first().getSeq(0).slice(0, 9)
+
dfTest.selectExpr("RS_Metadata(raster)").first().getStruct(0).toSeq.slice(0, 9)
assertTrue(expectedMetadata.equals(actualMetadata))
}
@@ -1528,20 +1593,21 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val rowsExpected = df.selectExpr("summary").collect()
val rowsActual = df.selectExpr("summary").collect()
- val expectedMetadata = df.selectExpr("meta").first().getSeq(0).slice(0,
9)
- val expectedSummary1 = rowsExpected(0).getSeq(0).slice(0, 6)
- val expectedSummary2 = rowsExpected(1).getSeq(0).slice(0, 6)
- val expectedSummary3 = rowsExpected(2).getSeq(0).slice(0, 6)
- val expectedSummary4 = rowsExpected(3).getSeq(0).slice(0, 6)
+ val expectedMetadata =
df.selectExpr("meta").first().getStruct(0).toSeq.slice(0, 9)
+ val expectedSummary1 = rowsExpected(0).getStruct(0).toSeq.slice(0, 6)
+ val expectedSummary2 = rowsExpected(1).getStruct(0).toSeq.slice(0, 6)
+ val expectedSummary3 = rowsExpected(2).getStruct(0).toSeq.slice(0, 6)
+ val expectedSummary4 = rowsExpected(3).getStruct(0).toSeq.slice(0, 6)
val expectedNumBands = mutable.WrappedArray.make(Array(4.0))
- val actualMetadata =
aggregatedDF2.selectExpr("meta").first().getSeq(0).slice(0, 9)
- val actualNumBands =
aggregatedDF2.selectExpr("meta").first().getSeq(0).slice(9, 10)
- val actualSummary1 = rowsActual(0).getSeq(0).slice(0, 6)
- val actualSummary2 = rowsActual(1).getSeq(0).slice(0, 6)
- val actualSummary3 = rowsActual(2).getSeq(0).slice(0, 6)
- val actualSummary4 = rowsActual(3).getSeq(0).slice(0, 6)
+ val actualMetadata =
aggregatedDF2.selectExpr("meta").first().getStruct(0).toSeq.slice(0, 9)
+ val actualNumBands =
+ aggregatedDF2.selectExpr("meta").first().getStruct(0).toSeq.slice(9,
10)
+ val actualSummary1 = rowsActual(0).getStruct(0).toSeq.slice(0, 6)
+ val actualSummary2 = rowsActual(1).getStruct(0).toSeq.slice(0, 6)
+ val actualSummary3 = rowsActual(2).getStruct(0).toSeq.slice(0, 6)
+ val actualSummary4 = rowsActual(3).getStruct(0).toSeq.slice(0, 6)
assertTrue(expectedMetadata.equals(actualMetadata))
assertTrue(actualNumBands == expectedNumBands)
@@ -1620,7 +1686,12 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
df = df.selectExpr(
"RS_FromGeoTiff(content) as raster",
"ST_GeomFromWKT('POLYGON ((-8673439.6642 4572993.5327, -8673155.5737
4563873.2099, -8701890.3259 4562931.7093, -8682522.8735 4572703.8908,
-8673439.6642 4572993.5327))', 3857) as geom")
- val actual = df.selectExpr("RS_ZonalStatsAll(raster, geom, 1,
true)").first().get(0)
+ val actual = df
+ .selectExpr("RS_ZonalStatsAll(raster, geom, 1, true)")
+ .first()
+ .getStruct(0)
+ .toSeq
+ .slice(0, 9)
val expected = Seq(184792.0, 1.0719726e7, 58.00968656653401, 0.0, 0.0,
92.15004748703687,
8491.631251863151, 0.0, 255.0)
assertTrue(expected.equals(actual))
@@ -1630,7 +1701,7 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
.selectExpr(
"RS_ZonalStatsAll(raster, ST_GeomFromWKT('POLYGON
((-78.22106647832458748 37.76411511479908967, -78.20183062098976734
37.72863564460374874, -78.18088490966962922 37.76753482276972562,
-78.22106647832458748 37.76411511479908967))'), 1, false)")
.first()
- .get(0)
+ .getStruct(0)
assertNull(actual2)
}
@@ -1641,7 +1712,12 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
df = df.selectExpr(
"RS_FromGeoTiff(content) as raster",
"ST_GeomFromWKT('POLYGON((-167.750000 87.750000, -155.250000
87.750000, -155.250000 40.250000, -180.250000 40.250000, -167.750000
87.750000))', 0) as geom")
- val actual = df.selectExpr("RS_ZonalStatsAll(raster, geom, 1,
true)").first().get(0)
+ val actual = df
+ .selectExpr("RS_ZonalStatsAll(raster, geom, 1, true)")
+ .first()
+ .getStruct(0)
+ .toSeq
+ .slice(0, 9)
val expected = Seq(14184.0, 3213526.0, 226.55992667794473, 255.0, 255.0,
74.87605357255357,
5606.423398599913, 1.0, 255.0)
assertTrue(expected.equals(actual))
@@ -1677,29 +1753,29 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
.format("binaryFile")
.load(resourceFolder + "raster/raster_with_no_data/test5.tiff")
df = df.selectExpr("RS_FromGeoTiff(content) as raster")
- var actual = df.selectExpr("RS_SummaryStatsAll(raster, 1,
false)").first().getSeq(0)
- assertEquals(1036800.0, actual.head, 0.1d)
- assertEquals(2.06233487e8, actual(1), 0.1d)
- assertEquals(198.91347125771605, actual(2), 1e-6d)
- assertEquals(95.09054096106192, actual(3), 1e-6d)
- assertEquals(0.0, actual(4), 0.1d)
- assertEquals(255.0, actual(5), 0.1d)
-
- actual = df.selectExpr("RS_SummaryStatsAll(raster, 1)").first().getSeq(0)
- assertEquals(928192.0, actual.head, 0.1d)
- assertEquals(2.06233487e8, actual(1), 0.1d)
- assertEquals(222.18839097945252, actual(2), 1e-6d)
- assertEquals(70.20559521132097, actual(3), 1e-6d)
- assertEquals(1.0, actual(4), 0.1d)
- assertEquals(255.0, actual(5), 0.1d)
-
- actual = df.selectExpr("RS_SummaryStatsAll(raster)").first().getSeq(0)
- assertEquals(928192.0, actual.head, 0.1d)
- assertEquals(2.06233487e8, actual(1), 0.1d)
- assertEquals(222.18839097945252, actual(2), 1e-6d)
- assertEquals(70.20559521132097, actual(3), 1e-6d)
- assertEquals(1.0, actual(4), 0.1d)
- assertEquals(255.0, actual(5), 0.1d)
+ var actual = df.selectExpr("RS_SummaryStatsAll(raster, 1,
false)").first().getStruct(0)
+ assertEquals(1036800.0, actual.getDouble(0), 0.1d)
+ assertEquals(2.06233487e8, actual.getDouble(1), 0.1d)
+ assertEquals(198.91347125771605, actual.getDouble(2), 1e-6d)
+ assertEquals(95.09054096106192, actual.getDouble(3), 1e-6d)
+ assertEquals(0.0, actual.getDouble(4), 0.1d)
+ assertEquals(255.0, actual.getDouble(5), 0.1d)
+
+ actual = df.selectExpr("RS_SummaryStatsAll(raster,
1)").first().getStruct(0)
+ assertEquals(928192.0, actual.getDouble(0), 0.1d)
+ assertEquals(2.06233487e8, actual.getDouble(1), 0.1d)
+ assertEquals(222.18839097945252, actual.getDouble(2), 1e-6d)
+ assertEquals(70.20559521132097, actual.getDouble(3), 1e-6d)
+ assertEquals(1.0, actual.getDouble(4), 0.1d)
+ assertEquals(255.0, actual.getDouble(5), 0.1d)
+
+ actual = df.selectExpr("RS_SummaryStatsAll(raster)").first().getStruct(0)
+ assertEquals(928192.0, actual.getDouble(0), 0.1d)
+ assertEquals(2.06233487e8, actual.getDouble(1), 0.1d)
+ assertEquals(222.18839097945252, actual.getDouble(2), 1e-6d)
+ assertEquals(70.20559521132097, actual.getDouble(3), 1e-6d)
+ assertEquals(1.0, actual.getDouble(4), 0.1d)
+ assertEquals(255.0, actual.getDouble(5), 0.1d)
}
it("Passed RS_BandIsNoData") {
@@ -2428,14 +2504,14 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
"RS_AddBandFromArray(RS_MakeEmptyRaster(1, 'd', 4, 3, 0, 0, 2, -2, 0,
0, 0), band, 1, null) as raster")
val rasterDf = df.selectExpr("RS_Resample(raster, 6, 5, 1, -1, false,
null) as raster")
val rasterOutput =
rasterDf.selectExpr("RS_AsMatrix(raster)").first().getString(0)
- val rasterMetadata =
rasterDf.selectExpr("RS_Metadata(raster)").first().getSeq[Double](0)
+ val rasterMetadata =
rasterDf.selectExpr("RS_Metadata(raster)").first().getStruct(0)
val expectedOutput =
"| 1.0 1.0 2.0 3.0 3.0 5.0|\n" + "| 1.0 1.0 2.0 3.0
3.0 5.0|\n" + "| 4.0 4.0 5.0 6.0 6.0 9.0|\n" + "| 7.0 7.0 8.0
9.0 9.0 10.0|\n" + "| 7.0 7.0 8.0 9.0 9.0 10.0|\n"
val expectedMetadata: Seq[Double] =
Seq(-0.33333333333333326, 0.19999999999999996, 6, 5,
1.388888888888889, -1.24, 0, 0, 0, 1)
assertEquals(expectedOutput, rasterOutput)
for (i <- expectedMetadata.indices) {
- assertEquals(expectedMetadata(i), rasterMetadata(i), 1e-6)
+ assertEquals(expectedMetadata(i), rasterMetadata.getDouble(i), 1e-6)
}
}
@@ -2445,13 +2521,13 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
"RS_AddBandFromArray(RS_MakeEmptyRaster(1, 'd', 4, 3, 0, 0, 2, -2, 0,
0, 0), band, 1, null) as raster")
val rasterDf = df.selectExpr("RS_Resample(raster, 1.2, -1.4, true, null)
as raster")
val rasterOutput =
rasterDf.selectExpr("RS_AsMatrix(raster)").first().getString(0)
- val rasterMetadata =
rasterDf.selectExpr("RS_Metadata(raster)").first().getSeq[Double](0)
+ val rasterMetadata =
rasterDf.selectExpr("RS_Metadata(raster)").first().getStruct(0)
val expectedOutput =
"| 1.0 1.0 2.0 3.0 3.0 5.0 5.0|\n" + "| 4.0 4.0
5.0 6.0 6.0 9.0 9.0|\n" + "| 4.0 4.0 5.0 6.0 6.0
9.0 9.0|\n" + "| 7.0 7.0 8.0 9.0 9.0 10.0 10.0|\n" + "|
NaN NaN NaN NaN NaN NaN NaN|\n"
val expectedMetadata: Seq[Double] = Seq(0, 0, 7, 5, 1.2, -1.4, 0, 0, 0,
1)
assertEquals(expectedOutput, rasterOutput)
for (i <- expectedMetadata.indices) {
- assertEquals(expectedMetadata(i), rasterMetadata(i), 1e-6)
+ assertEquals(expectedMetadata(i), rasterMetadata.getDouble(i), 1e-6)
}
}
@@ -2462,14 +2538,14 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
"RS_MakeEmptyRaster(2, 'd', 6, 5, 1, -1, 1.2, -1.4, 0, 0, 0) as
refRaster")
val rasterDf = df.selectExpr("RS_Resample(raster, refRaster, true, null)
as raster")
val rasterOutput =
rasterDf.selectExpr("RS_AsMatrix(raster)").first().getString(0)
- val rasterMetadata =
rasterDf.selectExpr("RS_Metadata(raster)").first().getSeq[Double](0)
+ val rasterMetadata =
rasterDf.selectExpr("RS_Metadata(raster)").first().getStruct(0)
val expectedOutput =
"| 1.0 1.0 2.0 3.0 3.0 5.0 5.0|\n" + "| 1.0 1.0 2.0
3.0 3.0 5.0 5.0|\n" + "| 4.0 4.0 5.0 6.0 6.0 9.0 9.0|\n" + "|
7.0 7.0 8.0 9.0 9.0 10.0 10.0|\n" + "| 7.0 7.0 8.0 9.0 9.0
10.0 10.0|\n"
val expectedMetadata: Seq[Double] =
Seq(-0.20000000298023224, 0.4000000059604645, 7.0, 5.0, 1.2, -1.4,
0.0, 0.0, 0.0, 1.0)
assertEquals(expectedOutput, rasterOutput)
for (i <- expectedMetadata.indices) {
- assertEquals(expectedMetadata(i), rasterMetadata(i), 1e-6)
+ assertEquals(expectedMetadata(i), rasterMetadata.getDouble(i), 1e-6)
}
}
@@ -2500,10 +2576,10 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val rasterDf = df.selectExpr("RS_FromNetCDF(content, 'O3') as raster")
val expectedMetadata = Seq(4.9375, 50.9375, 80, 48, 0.125, -0.125, 0, 0,
0, 4)
val actualMetadata =
- rasterDf.selectExpr("RS_Metadata(raster) as
metadata").first().getSeq[Double](0)
+ rasterDf.selectExpr("RS_Metadata(raster) as
metadata").first().getStruct(0)
for (i <- expectedMetadata.indices) {
- assertEquals(expectedMetadata(i), actualMetadata(i), 1e-6)
+ assertEquals(expectedMetadata(i), actualMetadata.getDouble(i), 1e-6)
}
val expectedFirstVal = 60.95357131958008
@@ -2518,10 +2594,10 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
val rasterDf = df.selectExpr("RS_FromNetCDF(content, 'O3', 'lon', 'lat')
as raster")
val expectedMetadata = Seq(4.9375, 50.9375, 80, 48, 0.125, -0.125, 0, 0,
0, 4)
val actualMetadata =
- rasterDf.selectExpr("RS_Metadata(raster) as
metadata").first().getSeq[Double](0)
+ rasterDf.selectExpr("RS_Metadata(raster) as
metadata").first().getStruct(0)
for (i <- expectedMetadata.indices) {
- assertEquals(expectedMetadata(i), actualMetadata(i), 1e-6)
+ assertEquals(expectedMetadata(i), actualMetadata.getDouble(i), 1e-6)
}
val expectedFirstVal = 60.95357131958008
@@ -2540,4 +2616,30 @@ class rasteralgebraTest extends TestBaseScala with
BeforeAndAfter with GivenWhen
assertEquals(expectedRecordInfo, recordInfo)
}
}
+
+ private def assertRSMetadata(
+ expectedDf: DataFrame,
+ expectedCol: String,
+ actualDf: DataFrame,
+ actualCol: String) = {
+ // Extract the actual metadata struct
+ val actualMetadataStruct =
actualDf.selectExpr(actualCol).first().getStruct(0)
+
+ // Extract the expected metadata struct
+ val expectedMetadataStruct =
expectedDf.selectExpr(expectedCol).first().getStruct(0)
+
+ // Function to convert Struct to Seq[Double]
+ def structToSeq(struct: Row): Seq[Double] = {
+ (0 until struct.size).map(struct.getDouble)
+ }
+
+ // Convert Structs to Seqs
+ val actualMetadata = structToSeq(actualMetadataStruct).slice(0, 9)
+ val expectedMetadata = structToSeq(expectedMetadataStruct).slice(0, 9)
+
+ // Compare the actual and expected metadata
+ assert(
+ actualMetadata == expectedMetadata,
+ s"Expected: $expectedMetadata, but got: $actualMetadata")
+ }
}
