JkSelf commented on code in PR #8722:
URL: https://github.com/apache/incubator-gluten/pull/8722#discussion_r1959424368
##########
shims/spark32/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala:
##########
@@ -103,17 +104,17 @@ class HiveFileFormat(fileSinkConf: FileSinkDesc)
if ("true" ==
sparkSession.sparkContext.getLocalProperty("isNativeApplicable")) {
val nativeFormat =
sparkSession.sparkContext.getLocalProperty("nativeFormat")
val tableOptions = tableDesc.getProperties.asScala.toMap
- val isParquetFormat = nativeFormat == "parquet"
val compressionCodec = if (fileSinkConf.compressed) {
- // hive related configurations
- fileSinkConf.compressCodec
- } else if (isParquetFormat) {
+ // MapredParquetOutputFormat use the `ParquetOutputFormat.COMPRESSION`
as
+ // the compression codec. Currently, we only support the parquet native
+ // writer for Spark3.2/3.3 code path.
+ tableOptions.getOrElse(
+ ParquetOutputFormat.COMPRESSION,
+ conf.get(ParquetOutputFormat.COMPRESSION,
CompressionCodecName.UNCOMPRESSED.name))
+ } else {
val parquetOptions =
new ParquetOptions(tableOptions, sparkSession.sessionState.conf)
parquetOptions.compressionCodecClassName
- } else {
- val orcOptions = new OrcOptions(tableOptions,
sparkSession.sessionState.conf)
Review Comment:
Why was the ORC code removed here? It appears that the CK backend supports
the ORC format and requires this code. The CI failure in the CK backend might
be related to this change. Could you please investigate?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
