[ https://issues.apache.org/jira/browse/SPARK-39387?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
dzcxzl updated SPARK-39387: --------------------------- Description: HIVE-25190: Fix many small allocations in BytesColumnVector {code:java} Caused by: java.lang.RuntimeException: Overflow of newLength. smallBuffer.length=1073741824, nextElemLength=408101 at org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector.increaseBufferSpace(BytesColumnVector.java:311) at org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector.setVal(BytesColumnVector.java:182) at org.apache.hadoop.hive.ql.io.orc.WriterImpl.setColumn(WriterImpl.java:179) at org.apache.hadoop.hive.ql.io.orc.WriterImpl.setColumn(WriterImpl.java:268) at org.apache.hadoop.hive.ql.io.orc.WriterImpl.setColumn(WriterImpl.java:223) at org.apache.hadoop.hive.ql.io.orc.WriterImpl.addRow(WriterImpl.java:294) at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat$OrcRecordWriter.write(OrcOutputFormat.java:105) at org.apache.spark.sql.hive.execution.HiveOutputWriter.write(HiveFileFormat.scala:157) at org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.write(FileFormatDataWriter.scala:176) at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.writeWithMetrics(FileFormatDataWriter.scala:86) at org.apache.spark.sql.execution.datasources.FileFormatDataWriter.writeWithIterator(FileFormatDataWriter.scala:93) at org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:312) at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1534) at org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:319) {code} was:[HIVE-25190|https://issues.apache.org/jira/browse/HIVE-25190]: Fix many small allocations in BytesColumnVector > Upgrade hive-storage-api to 2.7.3 > --------------------------------- > > Key: SPARK-39387 > URL: https://issues.apache.org/jira/browse/SPARK-39387 > Project: Spark > Issue Type: Improvement > Components: Build > Affects Versions: 3.2.1 > Reporter: dzcxzl > Priority: Minor > > HIVE-25190: Fix many small allocations in BytesColumnVector > > {code:java} > Caused by: java.lang.RuntimeException: Overflow of newLength. > smallBuffer.length=1073741824, nextElemLength=408101 > at > org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector.increaseBufferSpace(BytesColumnVector.java:311) > at > org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector.setVal(BytesColumnVector.java:182) > at > org.apache.hadoop.hive.ql.io.orc.WriterImpl.setColumn(WriterImpl.java:179) > at > org.apache.hadoop.hive.ql.io.orc.WriterImpl.setColumn(WriterImpl.java:268) > at > org.apache.hadoop.hive.ql.io.orc.WriterImpl.setColumn(WriterImpl.java:223) > at > org.apache.hadoop.hive.ql.io.orc.WriterImpl.addRow(WriterImpl.java:294) > at > org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat$OrcRecordWriter.write(OrcOutputFormat.java:105) > at > org.apache.spark.sql.hive.execution.HiveOutputWriter.write(HiveFileFormat.scala:157) > at > org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.write(FileFormatDataWriter.scala:176) > at > org.apache.spark.sql.execution.datasources.FileFormatDataWriter.writeWithMetrics(FileFormatDataWriter.scala:86) > at > org.apache.spark.sql.execution.datasources.FileFormatDataWriter.writeWithIterator(FileFormatDataWriter.scala:93) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$.$anonfun$executeTask$1(FileFormatWriter.scala:312) > at > org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1534) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$.executeTask(FileFormatWriter.scala:319) > {code} -- This message was sent by Atlassian Jira (v8.20.7#820007) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org