This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 6029bec540 [GH-2348] Fix STAC tests resilience to external server
failures in Scala (#2350)
6029bec540 is described below
commit 6029bec540e11b52d0a21819e23db6f4001cec94
Author: Feng Zhang <[email protected]>
AuthorDate: Mon Sep 15 21:41:22 2025 -0700
[GH-2348] Fix STAC tests resilience to external server failures in Scala
(#2350)
* Fix the STAC reader scala unit tests to remove dependency on remote
services
* address review comments
---
.../resources/stac/collections/copernicus-dem.json | 63 +++
.../stac/collections/earthview-catalog.json | 60 +++
.../src/test/resources/stac/collections/naip.json | 64 +++
.../stac/collections/sentinel-2-pre-c1-l2a.json | 103 +++++
.../stac/collections/vegetation-collection.json | 63 +++
.../stac/collections/wildfire-collection.json | 63 +++
.../resources/stac/items/sentinel-2-items.json | 445 +++++++++++++++++++++
.../sql/sedona_sql/io/stac/StacBatchTest.scala | 25 +-
.../sedona_sql/io/stac/StacDataSourceTest.scala | 18 +-
.../io/stac/StacPartitionReaderTest.scala | 8 +-
.../sql/sedona_sql/io/stac/StacTestUtils.scala | 35 ++
11 files changed, 918 insertions(+), 29 deletions(-)
diff --git
a/spark/common/src/test/resources/stac/collections/copernicus-dem.json
b/spark/common/src/test/resources/stac/collections/copernicus-dem.json
new file mode 100644
index 0000000000..2373c14ff1
--- /dev/null
+++ b/spark/common/src/test/resources/stac/collections/copernicus-dem.json
@@ -0,0 +1,63 @@
+{
+ "id": "copernicus-dem",
+ "type": "Collection",
+ "stac_extensions": [
+ "https://stac-extensions.github.io/raster/v1.1.0/schema.json";
+ ],
+ "stac_version": "1.0.0",
+ "description": "Mock Copernicus DEM collection for testing",
+ "title": "Mock Copernicus Digital Elevation Model",
+ "keywords": [
+ "copernicus",
+ "dem",
+ "elevation"
+ ],
+ "providers": [
+ {
+ "name": "Mock Earth Data Hub",
+ "description": "Mock Copernicus Earth Data Hub",
+ "roles": [
+ "producer"
+ ],
+ "url": "http://mock-earthdatahub.example.com";
+ }
+ ],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [
+ -180.0,
+ -90.0,
+ 180.0,
+ 90.0
+ ]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ [
+ "2021-01-01T00:00:00Z",
+ "2021-12-31T23:59:59Z"
+ ]
+ ]
+ }
+ },
+ "license": "proprietary",
+ "links": [
+ {
+ "rel": "root",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "self",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-dem-item.json",
+ "type": "application/json"
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/spark/common/src/test/resources/stac/collections/earthview-catalog.json
b/spark/common/src/test/resources/stac/collections/earthview-catalog.json
new file mode 100644
index 0000000000..df6a145ddf
--- /dev/null
+++ b/spark/common/src/test/resources/stac/collections/earthview-catalog.json
@@ -0,0 +1,60 @@
+{
+ "id": "earthview-catalog",
+ "type": "Catalog",
+ "stac_version": "1.0.0",
+ "description": "Mock Satellogic EarthView catalog for testing",
+ "title": "Mock Satellogic EarthView Catalog",
+ "keywords": [
+ "satellogic",
+ "earthview",
+ "satellite"
+ ],
+ "providers": [
+ {
+ "name": "Mock Satellogic",
+ "description": "Mock Satellogic EarthView",
+ "roles": [
+ "producer"
+ ],
+ "url": "http://mock-satellogic.example.com";
+ }
+ ],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [
+ -180.0,
+ -90.0,
+ 180.0,
+ 90.0
+ ]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ [
+ "2020-01-01T00:00:00Z",
+ "2023-12-31T23:59:59Z"
+ ]
+ ]
+ }
+ },
+ "license": "proprietary",
+ "links": [
+ {
+ "rel": "root",
+ "href": "catalog.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "self",
+ "href": "catalog.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-earthview-item.json",
+ "type": "application/json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/spark/common/src/test/resources/stac/collections/naip.json
b/spark/common/src/test/resources/stac/collections/naip.json
new file mode 100644
index 0000000000..d977a9093a
--- /dev/null
+++ b/spark/common/src/test/resources/stac/collections/naip.json
@@ -0,0 +1,64 @@
+{
+ "id": "naip",
+ "type": "Collection",
+ "stac_extensions": [
+ "https://stac-extensions.github.io/eo/v1.0.0/schema.json";,
+ "https://stac-extensions.github.io/raster/v1.1.0/schema.json";
+ ],
+ "stac_version": "1.0.0",
+ "description": "Mock NAIP (National Agriculture Imagery Program) collection
for testing",
+ "title": "Mock NAIP: National Agriculture Imagery Program",
+ "keywords": [
+ "naip",
+ "aerial",
+ "imagery"
+ ],
+ "providers": [
+ {
+ "name": "Mock Planetary Computer",
+ "description": "Mock Microsoft Planetary Computer",
+ "roles": [
+ "host"
+ ],
+ "url": "http://mock-planetarycomputer.example.com";
+ }
+ ],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [
+ -160.0,
+ 18.0,
+ -64.0,
+ 50.0
+ ]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ [
+ "2009-01-01T00:00:00Z",
+ "2022-12-31T23:59:59Z"
+ ]
+ ]
+ }
+ },
+ "license": "proprietary",
+ "links": [
+ {
+ "rel": "root",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "self",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-naip-item.json",
+ "type": "application/json"
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/spark/common/src/test/resources/stac/collections/sentinel-2-pre-c1-l2a.json
b/spark/common/src/test/resources/stac/collections/sentinel-2-pre-c1-l2a.json
new file mode 100644
index 0000000000..c243be556f
--- /dev/null
+++
b/spark/common/src/test/resources/stac/collections/sentinel-2-pre-c1-l2a.json
@@ -0,0 +1,103 @@
+{
+ "id": "sentinel-2-pre-c1-l2a",
+ "type": "Collection",
+ "stac_extensions": [
+ "https://stac-extensions.github.io/eo/v1.0.0/schema.json";,
+ "https://stac-extensions.github.io/projection/v1.0.0/schema.json";,
+ "https://stac-extensions.github.io/view/v1.0.0/schema.json";
+ ],
+ "stac_version": "1.0.0",
+ "description": "Mock Sentinel-2 Pre-Collection 1 Level-2A collection for
testing",
+ "title": "Mock Sentinel-2 Pre-Collection 1 Level-2A",
+ "keywords": [
+ "sentinel",
+ "earth observation",
+ "esa"
+ ],
+ "providers": [
+ {
+ "name": "Mock Earth Search",
+ "description": "Mock provider for testing",
+ "roles": [
+ "producer",
+ "processor"
+ ],
+ "url": "http://mock-earth-search.example.com";
+ }
+ ],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [
+ -180.0,
+ -90.0,
+ 180.0,
+ 90.0
+ ]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ [
+ "2015-06-23T00:00:00Z",
+ "2021-12-31T23:59:59Z"
+ ]
+ ]
+ }
+ },
+ "license": "proprietary",
+ "links": [
+ {
+ "rel": "root",
+ "href": "https://earth-search.aws.element84.com/v1";,
+ "type": "application/json"
+ },
+ {
+ "rel": "parent",
+ "href": "https://earth-search.aws.element84.com/v1";,
+ "type": "application/json"
+ },
+ {
+ "rel": "self",
+ "href":
"https://earth-search.aws.element84.com/v1/collections/sentinel-2-pre-c1-l2a";,
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-item-1.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-item-2.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-item-3.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-item-4.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-item-5.json",
+ "type": "application/json"
+ }
+ ],
+ "summaries": {
+ "platform": [
+ "sentinel-2a",
+ "sentinel-2b"
+ ],
+ "constellation": [
+ "sentinel-2"
+ ],
+ "instruments": [
+ "msi"
+ ]
+ }
+}
\ No newline at end of file
diff --git
a/spark/common/src/test/resources/stac/collections/vegetation-collection.json
b/spark/common/src/test/resources/stac/collections/vegetation-collection.json
new file mode 100644
index 0000000000..fc573d79a9
--- /dev/null
+++
b/spark/common/src/test/resources/stac/collections/vegetation-collection.json
@@ -0,0 +1,63 @@
+{
+ "id": "vegetation-collection",
+ "type": "Collection",
+ "stac_extensions": [
+ "https://stac-extensions.github.io/raster/v1.1.0/schema.json";
+ ],
+ "stac_version": "1.0.0",
+ "description": "Mock vegetation collection for testing",
+ "title": "Mock Vegetation Collection",
+ "keywords": [
+ "vegetation",
+ "canopy",
+ "height"
+ ],
+ "providers": [
+ {
+ "name": "Mock CFO",
+ "description": "Mock California Forest Observatory",
+ "roles": [
+ "producer"
+ ],
+ "url": "http://mock-cfo.example.com";
+ }
+ ],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [
+ -124.4096,
+ 32.5342,
+ -114.1308,
+ 42.0095
+ ]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ [
+ "2016-01-01T00:00:00Z",
+ "2016-12-31T23:59:59Z"
+ ]
+ ]
+ }
+ },
+ "license": "CC-BY-4.0",
+ "links": [
+ {
+ "rel": "root",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "self",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "California-Vegetation-CanopyBaseHeight-2016-Summer-00010m.json",
+ "type": "application/json"
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/spark/common/src/test/resources/stac/collections/wildfire-collection.json
b/spark/common/src/test/resources/stac/collections/wildfire-collection.json
new file mode 100644
index 0000000000..fa00f145c5
--- /dev/null
+++ b/spark/common/src/test/resources/stac/collections/wildfire-collection.json
@@ -0,0 +1,63 @@
+{
+ "id": "wildfire-collection",
+ "type": "Collection",
+ "stac_extensions": [
+ "https://stac-extensions.github.io/raster/v1.1.0/schema.json";
+ ],
+ "stac_version": "1.0.0",
+ "description": "Mock wildfire collection for testing",
+ "title": "Mock Wildfire Collection",
+ "keywords": [
+ "wildfire",
+ "fire",
+ "severity"
+ ],
+ "providers": [
+ {
+ "name": "Mock CFO",
+ "description": "Mock California Forest Observatory",
+ "roles": [
+ "producer"
+ ],
+ "url": "http://mock-cfo.example.com";
+ }
+ ],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [
+ -124.4096,
+ 32.5342,
+ -114.1308,
+ 42.0095
+ ]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ [
+ "2020-01-01T00:00:00Z",
+ "2020-12-31T23:59:59Z"
+ ]
+ ]
+ }
+ },
+ "license": "CC-BY-4.0",
+ "links": [
+ {
+ "rel": "root",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "self",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "item",
+ "href": "mock-wildfire-item.json",
+ "type": "application/json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/spark/common/src/test/resources/stac/items/sentinel-2-items.json
b/spark/common/src/test/resources/stac/items/sentinel-2-items.json
new file mode 100644
index 0000000000..e7548b016e
--- /dev/null
+++ b/spark/common/src/test/resources/stac/items/sentinel-2-items.json
@@ -0,0 +1,445 @@
+{
+ "type": "FeatureCollection",
+ "stac_version": "1.0.0",
+ "stac_extensions": [],
+ "context": {
+ "page": 1,
+ "limit": 10,
+ "matched": 10,
+ "returned": 10
+ },
+ "features": [
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220101T100001_N0400_R122_T33UUP_20220101T100001",
+ "properties": {
+ "datetime": "2022-01-01T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 180.5,
+ "view:sun_elevation": 25.3
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [10.0, 50.0],
+ [11.0, 50.0],
+ [11.0, 51.0],
+ [10.0, 51.0],
+ [10.0, 50.0]
+ ]
+ ]
+ },
+ "bbox": [10.0, 50.0, 11.0, 51.0],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-1.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220102T100001_N0400_R122_T33UUP_20220102T100001",
+ "properties": {
+ "datetime": "2022-01-02T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 181.2,
+ "view:sun_elevation": 25.8
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [10.5, 50.5],
+ [11.5, 50.5],
+ [11.5, 51.5],
+ [10.5, 51.5],
+ [10.5, 50.5]
+ ]
+ ]
+ },
+ "bbox": [10.5, 50.5, 11.5, 51.5],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-2.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-2.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220103T100001_N0400_R122_T33UUP_20220103T100001",
+ "properties": {
+ "datetime": "2022-01-03T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 182.1,
+ "view:sun_elevation": 26.2
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [11.0, 51.0],
+ [12.0, 51.0],
+ [12.0, 52.0],
+ [11.0, 52.0],
+ [11.0, 51.0]
+ ]
+ ]
+ },
+ "bbox": [11.0, 51.0, 12.0, 52.0],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-3.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-3.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220104T100001_N0400_R122_T33UUP_20220104T100001",
+ "properties": {
+ "datetime": "2022-01-04T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 183.0,
+ "view:sun_elevation": 26.7
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [11.5, 51.5],
+ [12.5, 51.5],
+ [12.5, 52.5],
+ [11.5, 52.5],
+ [11.5, 51.5]
+ ]
+ ]
+ },
+ "bbox": [11.5, 51.5, 12.5, 52.5],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-4.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-4.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220105T100001_N0400_R122_T33UUP_20220105T100001",
+ "properties": {
+ "datetime": "2022-01-05T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 183.8,
+ "view:sun_elevation": 27.1
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [12.0, 52.0],
+ [13.0, 52.0],
+ [13.0, 53.0],
+ [12.0, 53.0],
+ [12.0, 52.0]
+ ]
+ ]
+ },
+ "bbox": [12.0, 52.0, 13.0, 53.0],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-5.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-5.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220106T100001_N0400_R122_T33UUP_20220106T100001",
+ "properties": {
+ "datetime": "2022-01-06T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 184.5,
+ "view:sun_elevation": 27.5
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [12.5, 52.5],
+ [13.5, 52.5],
+ [13.5, 53.5],
+ [12.5, 53.5],
+ [12.5, 52.5]
+ ]
+ ]
+ },
+ "bbox": [12.5, 52.5, 13.5, 53.5],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-6.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-6.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220107T100001_N0400_R122_T33UUP_20220107T100001",
+ "properties": {
+ "datetime": "2022-01-07T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 185.2,
+ "view:sun_elevation": 27.9
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [13.0, 53.0],
+ [14.0, 53.0],
+ [14.0, 54.0],
+ [13.0, 54.0],
+ [13.0, 53.0]
+ ]
+ ]
+ },
+ "bbox": [13.0, 53.0, 14.0, 54.0],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-7.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-7.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220108T100001_N0400_R122_T33UUP_20220108T100001",
+ "properties": {
+ "datetime": "2022-01-08T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 185.9,
+ "view:sun_elevation": 28.3
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [13.5, 53.5],
+ [14.5, 53.5],
+ [14.5, 54.5],
+ [13.5, 54.5],
+ [13.5, 53.5]
+ ]
+ ]
+ },
+ "bbox": [13.5, 53.5, 14.5, 54.5],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-8.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-8.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220109T100001_N0400_R122_T33UUP_20220109T100001",
+ "properties": {
+ "datetime": "2022-01-09T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 186.6,
+ "view:sun_elevation": 28.7
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [14.0, 54.0],
+ [15.0, 54.0],
+ [15.0, 55.0],
+ [14.0, 55.0],
+ [14.0, 54.0]
+ ]
+ ]
+ },
+ "bbox": [14.0, 54.0, 15.0, 55.0],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-9.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-9.json",
+ "type": "application/json"
+ }
+ ]
+ },
+ {
+ "type": "Feature",
+ "stac_version": "1.0.0",
+ "id": "S2A_MSIL2A_20220110T100001_N0400_R122_T33UUP_20220110T100001",
+ "properties": {
+ "datetime": "2022-01-10T10:00:01.000Z",
+ "platform": "sentinel-2a",
+ "constellation": "sentinel-2",
+ "instruments": ["msi"],
+ "gsd": 10,
+ "proj:epsg": 32633,
+ "view:sun_azimuth": 187.3,
+ "view:sun_elevation": 29.1
+ },
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [14.5, 54.5],
+ [15.5, 54.5],
+ [15.5, 55.5],
+ [14.5, 55.5],
+ [14.5, 54.5]
+ ]
+ ]
+ },
+ "bbox": [14.5, 54.5, 15.5, 55.5],
+ "assets": {
+ "thumbnail": {
+ "href": "mock-thumbnail-10.jpg",
+ "type": "image/jpeg",
+ "title": "Thumbnail"
+ }
+ },
+ "links": [
+ {
+ "rel": "self",
+ "href": "mock-item-10.json",
+ "type": "application/json"
+ }
+ ]
+ }
+ ],
+ "links": [
+ {
+ "rel": "self",
+ "href": "sentinel-2-items.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "root",
+ "href": "../collections/sentinel-2-pre-c1-l2a.json",
+ "type": "application/json"
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacBatchTest.scala
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacBatchTest.scala
index 612ce2bb52..9430c2eb62 100644
---
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacBatchTest.scala
+++
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacBatchTest.scala
@@ -33,18 +33,9 @@ class StacBatchTest extends TestBaseScala {
Source.fromResource(resourceFilePath).getLines().mkString("\n")
}
- def getAbsolutePathOfResource(resourceFilePath: String): String = {
- val resourceUrl = getClass.getClassLoader.getResource(resourceFilePath)
- if (resourceUrl != null) {
- resourceUrl.getPath
- } else {
- throw new IllegalArgumentException(s"Resource not found:
$resourceFilePath")
- }
- }
-
it("collectItemLinks should collect correct item links") {
val collectionUrl =
-
"https://earth-search.aws.element84.com/v1/collections/sentinel-2-pre-c1-l2a";
+
StacTestUtils.getFileUrlOfResource("stac/collections/sentinel-2-pre-c1-l2a.json")
val stacCollectionJson = StacUtils.loadStacCollectionToJson(collectionUrl)
val opts = mutable
.Map(
@@ -85,15 +76,16 @@ class StacBatchTest extends TestBaseScala {
| "id": "sample-collection",
| "description": "A sample STAC collection",
| "links": [
- | {"rel": "item", "href":
"https://storage.googleapis.com/cfo-public/vegetation/California-Vegetation-CanopyBaseHeight-2016-Summer-00010m.json"},
- | {"rel": "item", "href":
"https://storage.googleapis.com/cfo-public/vegetation/California-Vegetation-CanopyBaseHeight-2016-Summer-00010m.json"},
- | {"rel": "item", "href":
"https://storage.googleapis.com/cfo-public/vegetation/California-Vegetation-CanopyBaseHeight-2016-Summer-00010m.json"}
+ | {"rel": "item", "href": "mock-item-1.json"},
+ | {"rel": "item", "href": "mock-item-2.json"},
+ | {"rel": "item", "href": "mock-item-3.json"}
| ]
|}
""".stripMargin
val opts = mutable.Map("numPartitions" -> "2", "itemsLimitMax" ->
"20").toMap
- val collectionUrl =
"https://storage.googleapis.com/cfo-public/vegetation/collection.json";
+ val collectionUrl =
+
StacTestUtils.getFileUrlOfResource("stac/collections/vegetation-collection.json")
val stacBatch =
StacBatch(
@@ -121,7 +113,8 @@ class StacBatchTest extends TestBaseScala {
""".stripMargin
val opts = mutable.Map("numPartitions" -> "2", "itemsLimitMax" ->
"20").toMap
- val collectionUrl = "https://path/to/collection.json";
+ val collectionUrl =
+
StacTestUtils.getFileUrlOfResource("stac/collections/vegetation-collection.json")
val stacBatch =
StacBatch(
@@ -142,7 +135,7 @@ class StacBatchTest extends TestBaseScala {
val rootJsonFile = "datasource_stac/collection.json"
val stacCollectionJson = loadJsonFromResource(rootJsonFile)
val opts = mutable.Map("numPartitions" -> "3", "itemsLimitMax" ->
"20").toMap
- val collectionUrl = getAbsolutePathOfResource(rootJsonFile)
+ val collectionUrl = StacTestUtils.getFileUrlOfResource(rootJsonFile)
val stacBatch =
StacBatch(
diff --git
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacDataSourceTest.scala
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacDataSourceTest.scala
index da9ff573e3..50051e8258 100644
---
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacDataSourceTest.scala
+++
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacDataSourceTest.scala
@@ -27,13 +27,13 @@ class StacDataSourceTest extends TestBaseScala {
val STAC_COLLECTION_LOCAL: String = resourceFolder +
"datasource_stac/collection.json"
val STAC_ITEM_LOCAL: String = resourceFolder + "geojson/core-item.json"
- val STAC_COLLECTION_REMOTE: List[String] = List(
-
"https://earth-search.aws.element84.com/v1/collections/sentinel-2-pre-c1-l2a";,
- "https://storage.googleapis.com/cfo-public/vegetation/collection.json";,
- "https://storage.googleapis.com/cfo-public/wildfire/collection.json";,
- "https://earthdatahub.destine.eu/api/stac/v1/collections/copernicus-dem";,
- "https://planetarycomputer.microsoft.com/api/stac/v1/collections/naip";,
-
"https://satellogic-earthview.s3.us-west-2.amazonaws.com/stac/catalog.json";)
+ val STAC_COLLECTION_MOCK: List[String] = List(
+
StacTestUtils.getFileUrlOfResource("stac/collections/sentinel-2-pre-c1-l2a.json"),
+
StacTestUtils.getFileUrlOfResource("stac/collections/vegetation-collection.json"),
+
StacTestUtils.getFileUrlOfResource("stac/collections/wildfire-collection.json"),
+ StacTestUtils.getFileUrlOfResource("stac/collections/copernicus-dem.json"),
+ StacTestUtils.getFileUrlOfResource("stac/collections/naip.json"),
+
StacTestUtils.getFileUrlOfResource("stac/collections/earthview-catalog.json"))
it("basic df load from local file should work") {
val dfStac = sparkSession.read.format("stac").load(STAC_COLLECTION_LOCAL)
@@ -49,8 +49,8 @@ class StacDataSourceTest extends TestBaseScala {
assert(rowCount > 0)
}
- it("basic df load from remote service endpoints should work") {
- STAC_COLLECTION_REMOTE.foreach { endpoint =>
+ it("basic df load from mock service endpoints should work") {
+ STAC_COLLECTION_MOCK.foreach { endpoint =>
val dfStac = sparkSession.read.format("stac").load(endpoint)
assertSchema(dfStac.schema)
}
diff --git
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacPartitionReaderTest.scala
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacPartitionReaderTest.scala
index 428327e7cc..31923a0141 100644
---
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacPartitionReaderTest.scala
+++
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacPartitionReaderTest.scala
@@ -33,8 +33,8 @@ class StacPartitionReaderTest extends TestBaseScala {
val JSON_STAC_ITEM_CORE: String = s"file://$TEST_DATA_FOLDER/core-item.json"
val JSON_STAC_ITEM_EXTENDED: String =
s"file://$TEST_DATA_FOLDER/extended-item.json"
val JSON_STAC_ITEM_FEATURES: String =
s"file://$TEST_DATA_FOLDER/collection-items.json"
- val HTTPS_STAC_ITEM_FEATURES: String =
-
"https://earth-search.aws.element84.com/v1/collections/sentinel-2-pre-c1-l2a/items";
+ val MOCK_STAC_ITEM_FEATURES: String =
+ StacTestUtils.getFileUrlOfResource("stac/items/sentinel-2-items.json")
it("StacPartitionReader should read feature files from local files") {
val jsonFiles =
@@ -81,8 +81,8 @@ class StacPartitionReaderTest extends TestBaseScala {
reader.close()
}
- it("StacPartitionReader should read features collection file from https
endpoint") {
- val jsonFiles = Seq(HTTPS_STAC_ITEM_FEATURES).toArray
+ it("StacPartitionReader should read features collection file from mock
endpoint") {
+ val jsonFiles = Seq(MOCK_STAC_ITEM_FEATURES).toArray
val partition = StacPartition(0, jsonFiles, Map.empty[String,
String].asJava)
val reader =
new StacPartitionReader(
diff --git
a/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacTestUtils.scala
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacTestUtils.scala
new file mode 100644
index 0000000000..169aaa8cef
--- /dev/null
+++
b/spark/common/src/test/scala/org/apache/spark/sql/sedona_sql/io/stac/StacTestUtils.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.spark.sql.sedona_sql.io.stac
+
+object StacTestUtils {
+
+ def getAbsolutePathOfResource(resourceFilePath: String): String = {
+ val resourceUrl = getClass.getClassLoader.getResource(resourceFilePath)
+ if (resourceUrl != null) {
+ resourceUrl.getPath
+ } else {
+ throw new IllegalArgumentException(s"Resource not found:
$resourceFilePath")
+ }
+ }
+
+ def getFileUrlOfResource(resourceFilePath: String): String = {
+ "file://" + getAbsolutePathOfResource(resourceFilePath)
+ }
+}
