This is an automated email from the ASF dual-hosted git repository.
jackylee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7a564d8153 [VL] Fallback unsupported orc write for spark32 and spark33
(#7996)
7a564d8153 is described below
commit 7a564d81536251ff361a3c0a3b9ea5f44ec02a00
Author: jackylee <[email protected]>
AuthorDate: Thu Nov 21 10:09:59 2024 +0800
[VL] Fallback unsupported orc write for spark32 and spark33 (#7996)
---
.../backendsapi/velox/VeloxListenerApi.scala | 4 +--
.../datasources/velox/VeloxOrcWriterInjects.scala | 32 ----------------------
2 files changed, 2 insertions(+), 34 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 85de0a8889..d29d302970 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -32,7 +32,7 @@ import org.apache.spark.api.plugin.PluginContext
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.ByteUnit
import org.apache.spark.sql.execution.datasources.GlutenWriterColumnarRules
-import
org.apache.spark.sql.execution.datasources.velox.{VeloxOrcWriterInjects,
VeloxParquetWriterInjects, VeloxRowSplitter}
+import
org.apache.spark.sql.execution.datasources.velox.{VeloxParquetWriterInjects,
VeloxRowSplitter}
import org.apache.spark.sql.expression.UDFResolver
import org.apache.spark.sql.internal.{GlutenConfigUtil, StaticSQLConf}
import org.apache.spark.util.{SparkDirectoryUtil, SparkResourceUtil}
@@ -166,7 +166,7 @@ class VeloxListenerApi extends ListenerApi with Logging {
NativeBackendInitializer.initializeBackend(parsed)
// Inject backend-specific implementations to override spark classes.
- GlutenFormatFactory.register(new VeloxParquetWriterInjects, new
VeloxOrcWriterInjects)
+ GlutenFormatFactory.register(new VeloxParquetWriterInjects)
GlutenFormatFactory.injectPostRuleFactory(
session => GlutenWriterColumnarRules.NativeWritePostRule(session))
GlutenFormatFactory.register(new VeloxRowSplitter())
diff --git
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxOrcWriterInjects.scala
b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxOrcWriterInjects.scala
deleted file mode 100644
index 0999121dd8..0000000000
---
a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxOrcWriterInjects.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.velox
-
-import scala.collection.JavaConverters.mapAsJavaMapConverter
-import scala.collection.mutable
-
-class VeloxOrcWriterInjects extends VeloxFormatWriterInjects {
- override def nativeConf(
- options: Map[String, String],
- compressionCodec: String): java.util.Map[String, String] = {
- val sparkOptions = new mutable.HashMap[String, String]()
- // TODO: implement it
- sparkOptions.asJava
- }
-
- override val formatName: String = "orc"
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
