[
https://issues.apache.org/jira/browse/HADOOP-20053?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18136339#comment-18136339
]
ASF GitHub Bot commented on HADOOP-20053:
-----------------------------------------
wangyum commented on PR #8908:
URL: https://github.com/apache/hadoop/pull/8908#issuecomment-6572285422
Spark executor log:
```
26/11/02 00:04:16 DEBUG PerformanceAdvisory: Unable to initialize NativeIO
libraries
java.lang.UnsatisfiedLinkError: 'void
org.apache.hadoop.io.nativeio.NativeIO$POSIX.initNativePosix(boolean)'
at org.apache.hadoop.io.nativeio.NativeIO$POSIX.initNativePosix(Native
Method)
at
org.apache.hadoop.io.nativeio.NativeIO$POSIX.<clinit>(NativeIO.java:341)
at org.apache.hadoop.io.nativeio.NativeIO.initNative(Native Method)
at org.apache.hadoop.io.nativeio.NativeIO.<clinit>(NativeIO.java:861)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.<init>(ShortCircuitShm.java:469)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShm.<init>(DfsClientShm.java:70)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager.requestNewShm(DfsClientShmManager.java:181)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager.allocSlot(DfsClientShmManager.java:251)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager.allocSlot(DfsClientShmManager.java:417)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.allocShmSlot(ShortCircuitCache.java:1020)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.createShortCircuitReplicaInfo(BlockReaderFactory.java:535)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.create(ShortCircuitCache.java:800)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.fetchOrCreate(ShortCircuitCache.java:737)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getBlockReaderLocal(BlockReaderFactory.java:483)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:360)
at
org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:723)
at
org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:653)
at
org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:887)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:960)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:773)
at java.base/java.io.FilterInputStream.read(FilterInputStream.java:71)
at
org.apache.parquet.io.DelegatingSeekableInputStream.read(DelegatingSeekableInputStream.java:61)
at
org.apache.parquet.bytes.BytesUtils.readIntLittleEndian(BytesUtils.java:87)
at
org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:607)
at
org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:578)
at
org.apache.parquet.hadoop.ParquetFileReader.<init>(ParquetFileReader.java:971)
at
org.apache.parquet.hadoop.ParquetFileReader.open(ParquetFileReader.java:744)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReader.openFileAndReadFooter(ParquetFooterReader.java:102)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.$anonfun$buildParquetReader$4(ParquetFileFormat.scala:331)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.$anonfun$buildParquetReader$13(ParquetFileFormat.scala:424)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.org$apache$spark$sql$execution$datasources$FileScanRDD$$anon$$readCurrentFile(FileScanRDD.scala:242)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:302)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext0(FileScanRDD.scala:142)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:151)
at
org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:877)
at
org.apache.spark.sql.execution.ColumnarToRowExec$.advanceBatch(Columnar.scala:232)
at
org.apache.spark.sql.execution.ColumnarToRowExec.advanceBatch(Columnar.scala)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown
Source)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown
Source)
at
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:55)
at
org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$2(SparkPlan.scala:419)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:960)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:960)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:57)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:338)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at
org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:223)
at org.apache.spark.scheduler.Task.run(Task.scala:147)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:912)
at
org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:86)
at
org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:83)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:97)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:915)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1474)
26/11/02 00:04:16 DEBUG PerformanceAdvisory: Unable to initialize NativeIO
libraries
java.lang.NoSuchFieldError: workaroundNonThreadSafePasswdCalls
at org.apache.hadoop.io.nativeio.NativeIO.initNative(Native Method)
at org.apache.hadoop.io.nativeio.NativeIO.<clinit>(NativeIO.java:861)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.<init>(ShortCircuitShm.java:469)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShm.<init>(DfsClientShm.java:70)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager.requestNewShm(DfsClientShmManager.java:181)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager.allocSlot(DfsClientShmManager.java:251)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager.allocSlot(DfsClientShmManager.java:417)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.allocShmSlot(ShortCircuitCache.java:1020)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.createShortCircuitReplicaInfo(BlockReaderFactory.java:535)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.create(ShortCircuitCache.java:800)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.fetchOrCreate(ShortCircuitCache.java:737)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getBlockReaderLocal(BlockReaderFactory.java:483)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:360)
at
org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:723)
at
org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:653)
at
org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:887)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:960)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:773)
at java.base/java.io.FilterInputStream.read(FilterInputStream.java:71)
at
org.apache.parquet.io.DelegatingSeekableInputStream.read(DelegatingSeekableInputStream.java:61)
at
org.apache.parquet.bytes.BytesUtils.readIntLittleEndian(BytesUtils.java:87)
at
org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:607)
at
org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:578)
at
org.apache.parquet.hadoop.ParquetFileReader.<init>(ParquetFileReader.java:971)
at
org.apache.parquet.hadoop.ParquetFileReader.open(ParquetFileReader.java:744)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReader.openFileAndReadFooter(ParquetFooterReader.java:102)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.$anonfun$buildParquetReader$4(ParquetFileFormat.scala:331)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.$anonfun$buildParquetReader$13(ParquetFileFormat.scala:424)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.org$apache$spark$sql$execution$datasources$FileScanRDD$$anon$$readCurrentFile(FileScanRDD.scala:242)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:302)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext0(FileScanRDD.scala:142)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:151)
at
org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:877)
at
org.apache.spark.sql.execution.ColumnarToRowExec$.advanceBatch(Columnar.scala:232)
at
org.apache.spark.sql.execution.ColumnarToRowExec.advanceBatch(Columnar.scala)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown
Source)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown
Source)
at
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:55)
at
org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$2(SparkPlan.scala:419)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:960)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:960)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:57)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:338)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at
org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:223)
at org.apache.spark.scheduler.Task.run(Task.scala:147)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:912)
at
org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:86)
at
org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:83)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:97)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:915)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1474)
26/11/02 00:04:16 WARN ShortCircuitCache: ShortCircuitCache(0x1e3dd5c2):
failed to load 3421868755_BP-1855921606-10.162.206.182-1754384840656
java.lang.UnsupportedOperationException: NativeIO is not available.
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.<init>(ShortCircuitShm.java:470)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShm.<init>(DfsClientShm.java:70)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager.requestNewShm(DfsClientShmManager.java:181)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager.allocSlot(DfsClientShmManager.java:251)
at
org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager.allocSlot(DfsClientShmManager.java:417)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.allocShmSlot(ShortCircuitCache.java:1020)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.createShortCircuitReplicaInfo(BlockReaderFactory.java:535)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.create(ShortCircuitCache.java:800)
at
org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.fetchOrCreate(ShortCircuitCache.java:737)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.getBlockReaderLocal(BlockReaderFactory.java:483)
at
org.apache.hadoop.hdfs.client.impl.BlockReaderFactory.build(BlockReaderFactory.java:360)
at
org.apache.hadoop.hdfs.DFSInputStream.getBlockReader(DFSInputStream.java:723)
at
org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:653)
at
org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:887)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:960)
at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:773)
at java.base/java.io.FilterInputStream.read(FilterInputStream.java:71)
at
org.apache.parquet.io.DelegatingSeekableInputStream.read(DelegatingSeekableInputStream.java:61)
at
org.apache.parquet.bytes.BytesUtils.readIntLittleEndian(BytesUtils.java:87)
at
org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:607)
at
org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:578)
at
org.apache.parquet.hadoop.ParquetFileReader.<init>(ParquetFileReader.java:971)
at
org.apache.parquet.hadoop.ParquetFileReader.open(ParquetFileReader.java:744)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFooterReader.openFileAndReadFooter(ParquetFooterReader.java:102)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.$anonfun$buildParquetReader$4(ParquetFileFormat.scala:331)
at
org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat.$anonfun$buildParquetReader$13(ParquetFileFormat.scala:424)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.org$apache$spark$sql$execution$datasources$FileScanRDD$$anon$$readCurrentFile(FileScanRDD.scala:242)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:302)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext0(FileScanRDD.scala:142)
at
org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:151)
at
org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:877)
at
org.apache.spark.sql.execution.ColumnarToRowExec$.advanceBatch(Columnar.scala:232)
at
org.apache.spark.sql.execution.ColumnarToRowExec.advanceBatch(Columnar.scala)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown
Source)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown
Source)
at
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:55)
at
org.apache.spark.sql.execution.WholeStageCodegenEvaluatorFactory$WholeStageCodegenPartitionEvaluator$$anon$1.hasNext(WholeStageCodegenEvaluatorFactory.scala:50)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$2(SparkPlan.scala:419)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:960)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:960)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:57)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:374)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:338)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:93)
at
org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:223)
at org.apache.spark.scheduler.Task.run(Task.scala:147)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$5(Executor.scala:912)
at
org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally(SparkErrorUtils.scala:86)
at
org.apache.spark.util.SparkErrorUtils.tryWithSafeFinally$(SparkErrorUtils.scala:83)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:97)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:915)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1474)
```
> SIGSEGV in jni_NewObject after a failed NativeIO initialization poisons the
> shared FileDescriptor natives cache
> ---------------------------------------------------------------------------------------------------------------
>
> Key: HADOOP-20053
> URL: https://issues.apache.org/jira/browse/HADOOP-20053
> Project: Hadoop Common
> Issue Type: Bug
> Affects Versions: 3.5.0
> Reporter: Yuming Wang
> Priority: Major
> Labels: pull-request-available
>
> h2. Symptom
> Spark executors with HDFS short-circuit reads enabled die with SIGSEGV. Two
> production hs_err files show the *identical* crash site:
> {code}
> # Problematic frame:
> # V [libjvm.so+0x8932b7] jni_NewObject+0x107
> siginfo: si_addr: 0x0 (SEGV_MAPERR)
> Registers: RSI=0x0, RDX=0x0 <- jclass and jmethodID arguments are NULL
> {code}
> Java stack of the crashing thread:
> {code}
> DomainSocket.receiveFileDescriptors0
> DomainSocket.recvFileInputStreams
> DfsClientShmManager$EndpointShmManager.requestNewShm
> ... ShortCircuitCache.allocShmSlot ... BlockReaderFactory.build
> ... ParquetFileReader.readFooter ... (Spark task reading HDFS)
> {code}
> Sibling executors that do not crash instead log:
> {code}
> java.lang.UnsupportedOperationException: NativeIO is not available.
> at
> org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.<init>(ShortCircuitShm.java:470)
> {code}
> h2. Root cause chain (all links verified against production logs)
> *1)* A version skew (newer Java code, older {{libhadoop.so}} on the nodes)
> makes {{NativeIO.POSIX}} initialization fail. The failure is only logged at
> DEBUG via PerformanceAdvisory, so production logs show nothing. With DEBUG
> enabled we captured both directions of the skew:
> {code}
> DEBUG PerformanceAdvisory: Unable to initialize NativeIO libraries
> java.lang.UnsatisfiedLinkError: 'void
> ...NativeIO$POSIX.initNativePosix(boolean)'
> ...
> DEBUG PerformanceAdvisory: Unable to initialize NativeIO libraries
> java.lang.NoSuchFieldError: workaroundNonThreadSafePasswdCalls
> {code}
> *2)* The native {{initNative*}} *error paths call {{fd_deinit(env)}}* ("these
> are all idempotent and safe to call even if the class wasn't inited yet").
> That comment is wrong for the FileDescriptor cache:
> {{fd_class}}/{{fd_constructor}} are *shared with DomainSocket*, which may
> have initialized them already through {{DomainSocket.anchorNative()}} (a
> once-per-JVM call). After the teardown they are NULL forever - {{fd_init()}}
> is never re-invoked.
> *3)* Whether a given JVM crashes or only warns depends on *class
> initialization order*: if {{DomainSocket.<clinit>}} ran before the NativeIO
> failure, the cache is poisoned and the next {{receiveFileDescriptors0}}
> executes {{fd_create()}} -> {{NewObject(env, NULL, NULL)}} -> SIGSEGV. If
> {{DomainSocket.<clinit>}} runs after the failure, its {{fd_init()}}
> re-populates the cache and the same request fails gracefully with "NativeIO
> is not available".
> *4)* {{fd_create()}}/{{fd_get()}} had no NULL guards, so any JVM crashes at
> the same PC deterministically. This is JVM-independent.
> Note: we initially suspected JVM-side GC bugs (G1 pinning) and ruled them out
> - the registers show clean NULLs, not heap corruption.
> h2. Proposed fix
> * {{fd_init()}}: resolve all members into locals first and publish the
> statics atomically; failures leave the statics NULL and are retry-friendly.
> * {{fd_get()}}/{{fd_create()}} (unix and windows): lazily re-initialize the
> cache, then throw IllegalStateException instead of calling JNI with NULL
> members. The existing {{receiveFileDescriptors0}} error path propagates it
> and the client falls back to a regular block reader.
> * {{NativeIO.c}}: remove {{fd_deinit(env)}} from the three init error paths
> (the FileDescriptor cache must not be torn down by unrelated NativeIO
> failures). {{fd_deinit()}} itself is retained for symmetry / explicit
> shutdown use.
> * {{NativeIO.java}}: log the init failure at WARN instead of DEBUG through a
> shared constant message (kept a constant expression so javac folds it into
> the inner classes whose static initializers can run re-entrantly).
> Known tradeoff (documented in source and commit message): {{fd_init()}} is
> intentionally not thread-safe; a cold-cache race can leak at most one global
> ref per racing thread and is caught by the NULL checks, so the worst outcome
> is a spurious IllegalStateException. fd_create() is only reached from
> short-circuit shm setup which is near-serialized (DfsClientShmManager loading
> lock).
> h2. Testing
> No unit test added: reproducing requires a version-skewed libhadoop.so, which
> cannot be constructed in a unit test. Failure is evidenced by the production
> hs_err files above; post-fix behavior (graceful IllegalStateException through
> the existing DomainSocket error path) is verified by inspection; end-to-end
> verification plan: deploy a rebuilt libhadoop.so to the skewed environment
> and observe the WARN plus read fallback instead of the SIGSEGV. Existing
> TestNativeIO / TestDomainSocket suites continue to pass.
> h2. Mitigation for affected deployments
> * {{dfs.client.read.shortcircuit=false}}, or
> * align the node-side {{libhadoop.so}} with the Java code version (the root
> trigger in this incident).
--
This message was sent by Atlassian Jira
(v10.3.26#10030026)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
