This is an automated email from the ASF dual-hosted git repository.
jtuglu1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 177cada0f34 fix: OrcInputFormat concurrent FileSystem init race
condition (#19491) (#19497)
177cada0f34 is described below
commit 177cada0f3470a1db64ff781fc89bde2764b14b3
Author: Shekhar Prasad Rajak <[email protected]>
AuthorDate: Tue May 26 00:34:27 2026 +0530
fix: OrcInputFormat concurrent FileSystem init race condition (#19491)
(#19497)
OrcInputFormat.initialize() — which swaps
Thread.currentThread().setContextClassLoader() and calls FileSystem.get(conf) —
was invoked on every createReader() call. When a ParallelIndexTask runs
multiple ORC subtasks concurrently in the same JVM (as in embedded tests)
---
.../apache/druid/data/input/orc/OrcInputFormat.java | 19 +++++++++++--------
.../druid/data/input/orc/OrcInputFormatTest.java | 1 +
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git
a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
index 7474a79a15e..96bfc03214d 100644
---
a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
+++
b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
@@ -36,12 +36,14 @@ import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
public class OrcInputFormat extends NestedInputFormat
{
static final long SCALE_FACTOR = 8L;
private final boolean binaryAsString;
private final Configuration conf;
+ private final AtomicBoolean fileSystemInitialized = new AtomicBoolean(false);
@JsonCreator
public OrcInputFormat(
@@ -55,19 +57,20 @@ public class OrcInputFormat extends NestedInputFormat
this.conf = conf;
}
- private void initialize(Configuration conf)
+ // Init FileSystem once under this class's classloader to avoid concurrent
setContextClassLoader races.
+ private void ensureFileSystemInitialized()
{
- //Initializing seperately since during eager initialization, resolving
- //namenode hostname throws an error if nodes are ephemeral
-
- // Ensure that FileSystem class level initialization happens with correct
CL
- // See https://github.com/apache/druid/issues/1714
- ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader();
+ if (!fileSystemInitialized.compareAndSet(false, true)) {
+ return;
+ }
+ final ClassLoader currCtxCl =
Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
FileSystem.get(conf);
}
catch (IOException ex) {
+ // Reset so a subsequent createReader can retry init instead of skipping
it.
+ fileSystemInitialized.set(false);
throw new RuntimeException(ex);
}
finally {
@@ -91,7 +94,7 @@ public class OrcInputFormat extends NestedInputFormat
@Override
public InputEntityReader createReader(InputRowSchema inputRowSchema,
InputEntity source, File temporaryDirectory)
{
- initialize(conf);
+ ensureFileSystemInitialized();
return new OrcReader(conf, inputRowSchema, source, temporaryDirectory,
getFlattenSpec(), binaryAsString);
}
diff --git
a/extensions-core/orc-extensions/src/test/java/org/apache/druid/data/input/orc/OrcInputFormatTest.java
b/extensions-core/orc-extensions/src/test/java/org/apache/druid/data/input/orc/OrcInputFormatTest.java
index a7f6e5131c3..555d1de2c99 100644
---
a/extensions-core/orc-extensions/src/test/java/org/apache/druid/data/input/orc/OrcInputFormatTest.java
+++
b/extensions-core/orc-extensions/src/test/java/org/apache/druid/data/input/orc/OrcInputFormatTest.java
@@ -73,6 +73,7 @@ public class OrcInputFormatTest
{
EqualsVerifier.forClass(OrcInputFormat.class)
.withPrefabValues(Configuration.class, new Configuration(),
new Configuration())
+ .withIgnoredFields("fileSystemInitialized")
.usingGetClass()
.verify();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
