jackwang2 opened a new issue #676: Global dedup throws exception: Connection pool shut down URL: https://github.com/apache/incubator-hudi/issues/676 19/05/13 12:55:15 INFO DAGScheduler: ResultStage 30 (count at HoodieSparkSqlWriter.scala:151) failed in 1.379 s due to Job aborted due to stage failure: Task 52 in stage 30.0 failed 4 times, most recent failure: Lost task 52.3 in stage 30.0 (TID 13983, ip-172-19-104-207, executor 32): java.lang.RuntimeException: com.uber.hoodie.exception.HoodieException: com.uber.hoodie.exception.HoodieException: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Connection pool shut down at com.uber.hoodie.func.LazyIterableIterator.next(LazyIterableIterator.java:121) at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43) at scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440) at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:378) at org.apache.spark.storage.BlockManager$$anonfun$doPutIterator$1.apply(BlockManager.scala:1109) at org.apache.spark.storage.BlockManager$$anonfun$doPutIterator$1.apply(BlockManager.scala:1083) at org.apache.spark.storage.BlockManager.doPut(BlockManager.scala:1018) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1083) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:809) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:335) at org.apache.spark.rdd.RDD.iterator(RDD.scala:286) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at org.apache.spark.scheduler.Task.run(Task.scala:109) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused by: com.uber.hoodie.exception.HoodieException: com.uber.hoodie.exception.HoodieException: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Connection pool shut down at com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.computeNext(CopyOnWriteLazyInsertIterable.java:106) at com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.computeNext(CopyOnWriteLazyInsertIterable.java:45) at com.uber.hoodie.func.LazyIterableIterator.next(LazyIterableIterator.java:119) ... 20 more Caused by: com.uber.hoodie.exception.HoodieException: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Connection pool shut down at com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:146) at com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.computeNext(CopyOnWriteLazyInsertIterable.java:102) ... 22 more Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Connection pool shut down at java.util.concurrent.FutureTask.report(FutureTask.java:122) at java.util.concurrent.FutureTask.get(FutureTask.java:192) at com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:144) ... 23 more Caused by: java.lang.IllegalStateException: Connection pool shut down at org.apache.http.util.Asserts.check(Asserts.java:34) at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:191) at org.apache.http.pool.AbstractConnPool.lease(AbstractConnPool.java:294) at org.apache.http.impl.conn.PoolingClientConnectionManager.requestConnection(PoolingClientConnectionManager.java:187) at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:72) at com.amazonaws.http.conn.$Proxy9.requestConnection(Unknown Source) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:414) at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:835) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:384) at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:232) at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3528) at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:976) at com.amazonaws.services.s3.AmazonS3Client.getObjectMetadata(AmazonS3Client.java:956) at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:892) at org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:77) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426) at org.apache.hadoop.fs.s3a.S3AFileSystem.create(S3AFileSystem.java:400) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:911) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:892) at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:789) at com.uber.hoodie.io.storage.HoodieWrapperFileSystem.create(HoodieWrapperFileSystem.java:150) at org.apache.parquet.hadoop.ParquetFileWriter.<init>(ParquetFileWriter.java:241) at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:273) at org.apache.parquet.hadoop.ParquetWriter.<init>(ParquetWriter.java:222) at com.uber.hoodie.io.storage.HoodieParquetWriter.<init>(HoodieParquetWriter.java:54) at com.uber.hoodie.io.storage.HoodieStorageWriterFactory.newParquetStorageWriter(HoodieStorageWriterFactory.java:55) at com.uber.hoodie.io.storage.HoodieStorageWriterFactory.getStorageWriter(HoodieStorageWriterFactory.java:38) at com.uber.hoodie.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:71) at com.uber.hoodie.func.CopyOnWriteLazyInsertIterable$CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteLazyInsertIterable.java:149) at com.uber.hoodie.func.CopyOnWriteLazyInsertIterable$CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteLazyInsertIterable.java:127) at com.uber.hoodie.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:38) at com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:124) at java.util.concurrent.FutureTask.run(FutureTask.java:266)
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
