This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new a85b1d8985 Lazy Initialisation of Orc extensions module (#12663)
a85b1d8985 is described below
commit a85b1d898557490ecc4ef1c4d2b8ea813e811389
Author: Tejaswini Bandlamudi <[email protected]>
AuthorDate: Tue Jun 21 11:13:10 2022 +0530
Lazy Initialisation of Orc extensions module (#12663)
* Lazy initialization of Orc extension
* nit
* moving intialize method to OrcInputFormat
---
.../druid/data/input/orc/OrcExtensionsModule.java | 20 -------------------
.../druid/data/input/orc/OrcInputFormat.java | 23 ++++++++++++++++++++++
2 files changed, 23 insertions(+), 20 deletions(-)
diff --git
a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java
b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java
index 77997eaea9..110ac82dc0 100644
---
a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java
+++
b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java
@@ -27,9 +27,7 @@ import com.google.inject.Inject;
import org.apache.druid.data.input.orc.guice.Orc;
import org.apache.druid.initialization.DruidModule;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
@@ -64,24 +62,6 @@ public class OrcExtensionsModule implements DruidModule
// to properly initialize everything
final Configuration conf = new Configuration();
-
- // Set explicit CL. Otherwise it'll try to use thread context CL, which
may not have all of our dependencies.
- conf.setClassLoader(getClass().getClassLoader());
-
- // Ensure that FileSystem class level initialization happens with correct
CL
- // See https://github.com/apache/druid/issues/1714
- ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader();
- try {
-
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
- FileSystem.get(conf);
- }
- catch (IOException ex) {
- throw new RuntimeException(ex);
- }
- finally {
- Thread.currentThread().setContextClassLoader(currCtxCl);
- }
-
if (props != null) {
for (String propName : props.stringPropertyNames()) {
if (propName.startsWith("hadoop.")) {
diff --git
a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
index 1744fef9f6..701d6a86fa 100644
---
a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
+++
b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
@@ -29,9 +29,11 @@ import org.apache.druid.data.input.impl.NestedInputFormat;
import org.apache.druid.data.input.orc.guice.Orc;
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import javax.annotation.Nullable;
import java.io.File;
+import java.io.IOException;
import java.util.Objects;
public class OrcInputFormat extends NestedInputFormat
@@ -51,6 +53,26 @@ public class OrcInputFormat extends NestedInputFormat
this.conf = conf;
}
+ private void initialize(Configuration conf)
+ {
+ //Initializing seperately since during eager initialization, resolving
+ //namenode hostname throws an error if nodes are ephemeral
+
+ // Ensure that FileSystem class level initialization happens with correct
CL
+ // See https://github.com/apache/druid/issues/1714
+ ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader();
+ try {
+
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ FileSystem.get(conf);
+ }
+ catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(currCtxCl);
+ }
+ }
+
@Override
public boolean isSplittable()
{
@@ -60,6 +82,7 @@ public class OrcInputFormat extends NestedInputFormat
@Override
public InputEntityReader createReader(InputRowSchema inputRowSchema,
InputEntity source, File temporaryDirectory)
{
+ initialize(conf);
return new OrcReader(conf, inputRowSchema, source, temporaryDirectory,
getFlattenSpec(), binaryAsString);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
