[
https://issues.apache.org/jira/browse/HUDI-8644?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan updated HUDI-8644:
--------------------------------------
Summary: Compaction based on new FG reader hits Concurrent modification
exception (was: Clustering based on new FG reader hits Concurrent modification
exception)
> Compaction based on new FG reader hits Concurrent modification exception
> ------------------------------------------------------------------------
>
> Key: HUDI-8644
> URL: https://issues.apache.org/jira/browse/HUDI-8644
> Project: Apache Hudi
> Issue Type: Bug
> Components: clustering
> Reporter: sivabalan narayanan
> Priority: Blocker
> Fix For: 1.0.0
>
>
>
>
> {code:java}
> 24/12/04 01:53:19 WARN TaskSetManager: Lost task 3.0 in stage 175.0 (TID
> 7640) (ip-10-0-106-23.us-west-2.compute.internal executor 7):
> com.esotericsoftware.kryo.KryoException:
> java.lang.UnsupportedOperationException
> Serialization trace:
> reserved (org.apache.avro.Schema$Field)
> fieldMap (org.apache.avro.Schema$RecordSchema)
> right (org.apache.hudi.common.util.collection.ImmutablePair)
> at
> com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:144)
> at
> com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:543)
> at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
> at
> com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:161)
> at
> com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:39)
> at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731)
> at
> com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125)
> at
> com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:543)
> at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
> at
> com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:161)
> at
> com.esotericsoftware.kryo.serializers.MapSerializer.read(MapSerializer.java:39)
> at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731)
> at
> com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125)
> at
> com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:543)
> at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:813)
> at
> org.apache.hudi.common.util.SerializationUtils$KryoSerializerInstance.deserialize(SerializationUtils.java:103){code}
> {code:java}
> at
> org.apache.hudi.common.util.SerializationUtils.deserialize(SerializationUtils.java:77)
> at
> org.apache.hudi.common.util.collection.BitCaskDiskMap.get(BitCaskDiskMap.java:209)
> at
> org.apache.hudi.common.util.collection.BitCaskDiskMap.get(BitCaskDiskMap.java:202)
> at
> org.apache.hudi.common.util.collection.BitCaskDiskMap.get(BitCaskDiskMap.java:198)
> at
> org.apache.hudi.common.util.collection.BitCaskDiskMap.remove(BitCaskDiskMap.java:245)
> at
> org.apache.hudi.common.util.collection.BitCaskDiskMap.remove(BitCaskDiskMap.java:67)
> at
> org.apache.hudi.common.util.collection.ExternalSpillableMap.remove(ExternalSpillableMap.java:240)
> at
> org.apache.hudi.common.table.read.HoodiePositionBasedFileGroupRecordBuffer.hasNextBaseRecord(HoodiePositionBasedFileGroupRecordBuffer.java:221)
> at
> org.apache.hudi.common.table.read.HoodieKeyBasedFileGroupRecordBuffer.doHasNext(HoodieKeyBasedFileGroupRecordBuffer.java:135)
> at
> org.apache.hudi.common.table.read.HoodieBaseFileGroupRecordBuffer.hasNext(HoodieBaseFileGroupRecordBuffer.java:156)
> at
> org.apache.hudi.common.table.read.HoodieFileGroupReader.hasNext(HoodieFileGroupReader.java:236)
> at
> org.apache.hudi.common.table.read.HoodieFileGroupReader$HoodieFileGroupReaderIterator.hasNext(HoodieFileGroupReader.java:296)
> at
> org.apache.hudi.io.HoodieSparkMergeHandleV2.write(HoodieSparkMergeHandleV2.java:230)
> at
> org.apache.hudi.table.HoodieSparkCopyOnWriteTable.compactUsingFileGroupReader(HoodieSparkCopyOnWriteTable.java:285)
> at
> org.apache.hudi.table.action.compact.HoodieCompactor.compact(HoodieCompactor.java:295)
> at
> org.apache.hudi.table.action.compact.HoodieCompactor.lambda$compact$41887b12$1(HoodieCompactor.java:150)
> at
> org.apache.spark.api.java.JavaPairRDD$.$anonfun$toScalaFunction$1(JavaPairRDD.scala:1070)
> at scala.collection.Iterator$$anon$10.next(Iterator.scala:461)
> at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
> at
> org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223){code}
> {code:java}
> at
> org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352)
> at
> org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1614)
> at
> org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1524)
> at
> org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1588)
> at
> org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1389)
> at
> org.apache.spark.storage.BlockManager.getOrElseUpdateRDDBlock(BlockManager.scala:1343)
> at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:379)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
> at
> org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:367)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:331)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
> at
> org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:166)
> at org.apache.spark.scheduler.Task.run(Task.scala:141)
> at
> org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$4(Executor.scala:620)
> at
> org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:64)
> at
> org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:61)
> at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:94)
> at
> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:623)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:750)
> Caused by: java.lang.UnsupportedOperationException
> at
> java.util.Collections$UnmodifiableCollection.add(Collections.java:1057)
> at
> com.esotericsoftware.kryo.serializers.CollectionSerializer.read(CollectionSerializer.java:134)
> at
> com.esotericsoftware.kryo.serializers.CollectionSerializer.read(CollectionSerializer.java:40)
> at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731){code}
> {code:java}
> at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:731)
> at
> com.esotericsoftware.kryo.serializers.ObjectField.read(ObjectField.java:125)
> ... 58 more
> 24/12/04 01:53:19 INFO TaskSetManager: Starting task 3.1 in stage 175.0 (TID
> 9109) (ip-10-0-99-46.us-west-2.compute.internal, executor 6, partition 3,
> PROCESS_LOCAL, 9798 bytes) {code}
> {code:java}
> 24/12/04 01:53:20 INFO TaskSetManager: Lost task 4.0 in stage 175.0 (TID
> 7641) on ip-10-0-99-46.us-west-2.compute.internal, executor 6:
> com.esotericsoftware.kryo.KryoException
> (java.lang.UnsupportedOperationException
> Serialization trace:
> reserved (org.apache.avro.Schema$Field)
> fieldMap (org.apache.avro.Schema$RecordSchema)
> right (org.apache.hudi.common.util.collection.ImmutablePair)) [duplicate 1]
> 24/12/04 01:53:20 INFO TaskSetManager: Starting task 4.1 in stage 175.0 (TID
> 9110) (ip-10-0-90-216.us-west-2.compute.internal, executor 4, partition 4,
> PROCESS_LOCAL, 9798 bytes)
> 24/12/04 01:53:21 INFO BlockManagerInfo: Added rdd_359_37 in memory on
> ip-10-0-66-62.us-west-2.compute.internal:40389 (size: 363.0 B, free: 29.4 GiB)
> 24/12/04 01:53:21 INFO TaskSetManager: Finished task 37.0 in stage 175.0 (TID
> 7674) in 43143 ms on ip-10-0-66-62.us-west-2.compute.internal (executor 10)
> (43/79)
> 24/12/04 01:53:21 INFO TaskSetManager: Lost task 9.0 in stage 175.0 (TID
> 7646) on ip-10-0-118-204.us-west-2.compute.internal, executor 9:
> com.esotericsoftware.kryo.KryoException
> (java.lang.UnsupportedOperationException
> Serialization trace:
> reserved (org.apache.avro.Schema$Field)
> fieldMap (org.apache.avro.Schema$RecordSchema)
> right (org.apache.hudi.common.util.collection.ImmutablePair)) [duplicate 2]
> 24/12/04 01:53:21 INFO TaskSetManager: Starting task 9.1 in stage 175.0 (TID
> 9111) (ip-10-0-88-53.us-west-2.compute.internal, executor 5, partition 9,
> PROCESS_LOCAL, 9798 bytes)
> 24/12/04 01:53:21 INFO BlockManagerInfo: Added rdd_359_23 in memory on
> ip-10-0-106-23.us-west-2.compute.internal:39533 (size: 362.0 B, free: 29.6
> GiB)
> 24/12/04 01:53:21 INFO TaskSetManager: Finished task 23.0 in stage 175.0 (TID
> 7660) in 43215 ms on ip-10-0-106-23.us-west-2.compute.internal (executor 7)
> (44/79)
> 24/12/04 01:53:21 INFO TaskSetManager: Lost task 12.0 in stage 175.0 (TID
> 7649) on ip-10-0-88-53.us-west-2.compute.internal, executor 5:
> com.esotericsoftware.kryo.KryoException
> (java.lang.UnsupportedOperationException {code}
>
>
>
>
>
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
