This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 46229b4798 [VL] Remove resolving ViewFs file path from scan validation
(#8829)
46229b4798 is described below
commit 46229b4798a8fc584cb89d96abfa257b62d1760a
Author: PHILO-HE <[email protected]>
AuthorDate: Wed Feb 26 17:53:34 2025 +0800
[VL] Remove resolving ViewFs file path from scan validation (#8829)
---
.../gluten/backendsapi/velox/VeloxBackend.scala | 27 +++++-----------------
1 file changed, 6 insertions(+), 21 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
index 99cba7ea2f..25655c72db 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
@@ -51,9 +51,7 @@ import org.apache.spark.sql.types._
import org.apache.spark.util.SerializableConfiguration
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.viewfs.ViewFileSystemUtils
-import scala.collection.mutable
import scala.util.control.Breaks.breakable
class VeloxBackend extends SubstraitBackend {
@@ -109,25 +107,12 @@ object VeloxBackendSettings extends BackendSettingsApi {
def validateScheme(): Option[String] = {
val filteredRootPaths = distinctRootPaths(rootPaths)
- if (filteredRootPaths.nonEmpty) {
- val resolvedPaths =
- if (GlutenConfig.get.enableHdfsViewfs) {
- ViewFileSystemUtils.convertViewfsToHdfs(
- filteredRootPaths,
- mutable.Map.empty[String, String],
- serializableHadoopConf.get.value)
- } else {
- filteredRootPaths
- }
-
- if (
-
!VeloxFileSystemValidationJniWrapper.allSupportedByRegisteredFileSystems(
- resolvedPaths.toArray)
- ) {
- Some(s"Scheme of [$filteredRootPaths] is not supported by registered
file systems.")
- } else {
- None
- }
+ if (
+ filteredRootPaths.nonEmpty &&
+
!VeloxFileSystemValidationJniWrapper.allSupportedByRegisteredFileSystems(
+ filteredRootPaths.toArray)
+ ) {
+ Some(s"Scheme of [$filteredRootPaths] is not supported by registered
file systems.")
} else {
None
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
