This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new e6d34a300 [SEDONA-646] Fix DBR compatibility problem for shapefile
reader (#1573)
e6d34a300 is described below
commit e6d34a3009bbde993577cfa61d088d02433f0b90
Author: Kristin Cowalcijk <[email protected]>
AuthorDate: Tue Sep 3 08:59:05 2024 +0800
[SEDONA-646] Fix DBR compatibility problem for shapefile reader (#1573)
* Fix shapefile format reader for DBR 15.4 LTS
* Backport to Spark 3.0 to 3.4
---
.../datasources/shapefile/ShapefilePartition.scala | 27 ++++++++++++++++++++++
.../ShapefilePartitionReaderFactory.scala | 3 +--
.../sql/datasources/shapefile/ShapefileScan.scala | 2 +-
.../datasources/shapefile/ShapefilePartition.scala | 27 ++++++++++++++++++++++
.../ShapefilePartitionReaderFactory.scala | 3 +--
.../sql/datasources/shapefile/ShapefileScan.scala | 2 +-
.../datasources/shapefile/ShapefilePartition.scala | 27 ++++++++++++++++++++++
.../ShapefilePartitionReaderFactory.scala | 3 +--
.../sql/datasources/shapefile/ShapefileScan.scala | 2 +-
.../datasources/shapefile/ShapefilePartition.scala | 27 ++++++++++++++++++++++
.../ShapefilePartitionReaderFactory.scala | 3 +--
.../sql/datasources/shapefile/ShapefileScan.scala | 2 +-
.../datasources/shapefile/ShapefilePartition.scala | 27 ++++++++++++++++++++++
.../ShapefilePartitionReaderFactory.scala | 3 +--
.../sql/datasources/shapefile/ShapefileScan.scala | 2 +-
.../datasources/shapefile/ShapefilePartition.scala | 27 ++++++++++++++++++++++
.../ShapefilePartitionReaderFactory.scala | 3 +--
.../sql/datasources/shapefile/ShapefileScan.scala | 2 +-
18 files changed, 174 insertions(+), 18 deletions(-)
diff --git
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
new file mode 100644
index 000000000..306b1df4f
--- /dev/null
+++
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.shapefile
+
+import org.apache.spark.Partition
+import org.apache.spark.sql.connector.read.InputPartition
+import org.apache.spark.sql.execution.datasources.PartitionedFile
+
+case class ShapefilePartition(index: Int, files: Array[PartitionedFile])
+ extends Partition
+ with InputPartition
diff --git
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
index ba25c92da..5a28af6d6 100644
---
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
+++
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import
org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
-import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
partition match {
- case filePartition: FilePartition => buildReader(filePartition.files)
+ case filePartition: ShapefilePartition =>
buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
diff --git
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
index 081bc623d..e2a2d618b 100644
---
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
+++
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
@@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
- Some(FilePartition(index, group.values.toArray))
+ Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
diff --git
a/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
b/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
new file mode 100644
index 000000000..306b1df4f
--- /dev/null
+++
b/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.shapefile
+
+import org.apache.spark.Partition
+import org.apache.spark.sql.connector.read.InputPartition
+import org.apache.spark.sql.execution.datasources.PartitionedFile
+
+case class ShapefilePartition(index: Int, files: Array[PartitionedFile])
+ extends Partition
+ with InputPartition
diff --git
a/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
b/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
index ba25c92da..5a28af6d6 100644
---
a/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
+++
b/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import
org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
-import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
partition match {
- case filePartition: FilePartition => buildReader(filePartition.files)
+ case filePartition: ShapefilePartition =>
buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
diff --git
a/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
b/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
index 081bc623d..e2a2d618b 100644
---
a/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
+++
b/spark/spark-3.1/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
@@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
- Some(FilePartition(index, group.values.toArray))
+ Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
diff --git
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
new file mode 100644
index 000000000..306b1df4f
--- /dev/null
+++
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.shapefile
+
+import org.apache.spark.Partition
+import org.apache.spark.sql.connector.read.InputPartition
+import org.apache.spark.sql.execution.datasources.PartitionedFile
+
+case class ShapefilePartition(index: Int, files: Array[PartitionedFile])
+ extends Partition
+ with InputPartition
diff --git
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
index ba25c92da..5a28af6d6 100644
---
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
+++
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import
org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
-import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
partition match {
- case filePartition: FilePartition => buildReader(filePartition.files)
+ case filePartition: ShapefilePartition =>
buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
diff --git
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
index 081bc623d..e2a2d618b 100644
---
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
+++
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
@@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
- Some(FilePartition(index, group.values.toArray))
+ Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
diff --git
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
new file mode 100644
index 000000000..306b1df4f
--- /dev/null
+++
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.shapefile
+
+import org.apache.spark.Partition
+import org.apache.spark.sql.connector.read.InputPartition
+import org.apache.spark.sql.execution.datasources.PartitionedFile
+
+case class ShapefilePartition(index: Int, files: Array[PartitionedFile])
+ extends Partition
+ with InputPartition
diff --git
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
index ba25c92da..5a28af6d6 100644
---
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
+++
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import
org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
-import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
partition match {
- case filePartition: FilePartition => buildReader(filePartition.files)
+ case filePartition: ShapefilePartition =>
buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
diff --git
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
index 526b6cbee..283632a0d 100644
---
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
+++
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
@@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
- Some(FilePartition(index, group.values.toArray))
+ Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
diff --git
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
new file mode 100644
index 000000000..306b1df4f
--- /dev/null
+++
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.shapefile
+
+import org.apache.spark.Partition
+import org.apache.spark.sql.connector.read.InputPartition
+import org.apache.spark.sql.execution.datasources.PartitionedFile
+
+case class ShapefilePartition(index: Int, files: Array[PartitionedFile])
+ extends Partition
+ with InputPartition
diff --git
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
index ba25c92da..5a28af6d6 100644
---
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
+++
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import
org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
-import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
partition match {
- case filePartition: FilePartition => buildReader(filePartition.files)
+ case filePartition: ShapefilePartition =>
buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
diff --git
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
index f8f4cac2f..afdface89 100644
---
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
+++
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
@@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
- Some(FilePartition(index, group.values.toArray))
+ Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
diff --git
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
new file mode 100644
index 000000000..306b1df4f
--- /dev/null
+++
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartition.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.shapefile
+
+import org.apache.spark.Partition
+import org.apache.spark.sql.connector.read.InputPartition
+import org.apache.spark.sql.execution.datasources.PartitionedFile
+
+case class ShapefilePartition(index: Int, files: Array[PartitionedFile])
+ extends Partition
+ with InputPartition
diff --git
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
index ba25c92da..5a28af6d6 100644
---
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
+++
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefilePartitionReaderFactory.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.connector.read.PartitionReader
import org.apache.spark.sql.connector.read.PartitionReaderFactory
import org.apache.spark.sql.execution.datasources.PartitionedFile
import
org.apache.spark.sql.execution.datasources.v2.PartitionReaderWithPartitionValues
-import org.apache.spark.sql.execution.datasources.FilePartition
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -58,7 +57,7 @@ case class ShapefilePartitionReaderFactory(
override def createReader(partition: InputPartition):
PartitionReader[InternalRow] = {
partition match {
- case filePartition: FilePartition => buildReader(filePartition.files)
+ case filePartition: ShapefilePartition =>
buildReader(filePartition.files)
case _ =>
throw new IllegalArgumentException(
s"Unexpected partition type: ${partition.getClass.getCanonicalName}")
diff --git
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
index f8f4cac2f..afdface89 100644
---
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
+++
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/shapefile/ShapefileScan.scala
@@ -105,7 +105,7 @@ case class ShapefileScan(
} else false
}
if (!hasMissingFiles) {
- Some(FilePartition(index, group.values.toArray))
+ Some(ShapefilePartition(index, group.values.toArray))
} else {
None
}
