jinchengchenghh commented on code in PR #8722:
URL: https://github.com/apache/incubator-gluten/pull/8722#discussion_r1959966443
##########
shims/spark33/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala:
##########
@@ -99,17 +101,25 @@ class HiveFileFormat(fileSinkConf: FileSinkDesc)
if ("true" ==
sparkSession.sparkContext.getLocalProperty("isNativeApplicable")) {
val nativeFormat =
sparkSession.sparkContext.getLocalProperty("nativeFormat")
val tableOptions = tableDesc.getProperties.asScala.toMap
- val isParquetFormat = nativeFormat == "parquet"
- val compressionCodec = if (fileSinkConf.compressed) {
- // hive related configurations
- fileSinkConf.compressCodec
- } else if (isParquetFormat) {
- val parquetOptions =
- new ParquetOptions(tableOptions, sparkSession.sessionState.conf)
- parquetOptions.compressionCodecClassName
- } else {
- val orcOptions = new OrcOptions(tableOptions,
sparkSession.sessionState.conf)
- orcOptions.compressionCodec
+ val compressionCodec = nativeFormat match {
+ case "parquet" if fileSinkConf.compressed =>
+ // MapredParquetOutputFormat use the
`ParquetOutputFormat.COMPRESSION` as
+ // the compression codec. Currently, we only support the parquet
native
+ // writer for Spark3.2/3.3 code path.
Review Comment:
Is the comment right? @JkSelf
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
