This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e633ff8 HIVE-23582: LLAP: Make SplitLocationProvider impl pluggable
(Prasanth Jayachandran reviewed by Gopal V)
e633ff8 is described below
commit e633ff8186c14377443881d8eb02f2644e20b510
Author: Prasanth Jayachandran <[email protected]>
AuthorDate: Fri Jun 5 16:46:27 2020 -0700
HIVE-23582: LLAP: Make SplitLocationProvider impl pluggable (Prasanth
Jayachandran reviewed by Gopal V)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 4 ++
.../org/apache/hadoop/hive/ql/exec/tez/Utils.java | 27 ++++++++++-
.../apache/hadoop/hive/ql/exec/tez/TestUtils.java | 53 ++++++++++++++++++++++
3 files changed, 82 insertions(+), 2 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index abd12c9..1464d6a 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4704,6 +4704,10 @@ public class HiveConf extends Configuration {
"instead of using the locations provided by the split itself. If there
is no llap daemon " +
"running, fall back to locations provided by the split. This is
effective only if " +
"hive.execution.mode is llap"),
+
LLAP_SPLIT_LOCATION_PROVIDER_CLASS("hive.llap.split.location.provider.class",
+ "org.apache.hadoop.hive.ql.exec.tez.HostAffinitySplitLocationProvider",
+ "Split location provider class to use during split generation for LLAP.
This class should implement\n" +
+ "org.apache.hadoop.mapred.split.SplitLocationProvider interface"),
LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true,
"Whether LLAP should reject permissive ACLs in some cases (e.g. its
own management\n" +
"protocol or ZK paths), similar to how ssh refuses a key with bad
access permissions."),
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
index d266bb1..92d64df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.split.SplitLocationProvider;
import org.apache.tez.common.counters.TezCounters;
@@ -48,8 +50,29 @@ public class Utils {
&& HiveConf.getBoolVar(conf,
HiveConf.ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS)
&& useCacheAffinity;
SplitLocationProvider splitLocationProvider;
- LOG.info("SplitGenerator using llap affinitized locations: " +
useCustomLocations);
- if (useCustomLocations) {
+ final String locationProviderClass = HiveConf.getVar(conf,
HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS);
+ final boolean customLocationProvider =
+
!HostAffinitySplitLocationProvider.class.getName().equals(locationProviderClass);
+ LOG.info("SplitGenerator using llap affinitized locations: {}
locationProviderClass: {}", useCustomLocations, locationProviderClass);
+ if (customLocationProvider) {
+ SplitLocationProvider locationProviderImpl;
+ try {
+ // the implementation of SplitLocationProvider may have Configuration
as a single arg constructor, so we try
+ // invoking that constructor first. If that does not exist, the
fallback will use no-arg constructor.
+ locationProviderImpl = JavaUtils
+ .newInstance(JavaUtils.getClass(locationProviderClass,
SplitLocationProvider.class),
+ new Class<?>[]{Configuration.class}, new Object[]{conf});
+ } catch (Exception e) {
+ LOG.warn("Unable to instantiate {} class. Will try no-arg constructor
invocation..", locationProviderClass, e);
+ try {
+ locationProviderImpl =
JavaUtils.newInstance(JavaUtils.getClass(locationProviderClass,
+ SplitLocationProvider.class));
+ } catch (Exception ex) {
+ throw new IOException(ex);
+ }
+ }
+ return locationProviderImpl;
+ } else if (useCustomLocations) {
LlapRegistryService serviceRegistry =
LlapRegistryService.getClient(conf);
return getCustomSplitLocationProvider(serviceRegistry, LOG);
} else {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
index d9aa032..71ce812 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.tez;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet;
@@ -50,6 +51,7 @@ import java.util.List;
import java.util.Set;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
@@ -156,4 +158,55 @@ public class TestUtils {
knownLocations.remove(null);
assertArrayEquals(expectedLocations, knownLocations.toArray(new String[]
{}));
}
+
+ public static class NoConstructorSplitLocationProvider implements
SplitLocationProvider {
+
+ @Override
+ public String[] getLocations(final InputSplit inputSplit) throws
IOException {
+ return new String[0];
+ }
+ }
+
+ public static class DefaultConstructorSplitLocationProvider implements
SplitLocationProvider {
+
+ public DefaultConstructorSplitLocationProvider() {
+
+ }
+
+ @Override
+ public String[] getLocations(final InputSplit inputSplit) throws
IOException {
+ return new String[0];
+ }
+ }
+
+ public static class ConfConstructorSplitLocationProvider implements
SplitLocationProvider {
+ private Configuration configuration;
+
+ public ConfConstructorSplitLocationProvider(Configuration conf) {
+ this.configuration = conf;
+ }
+
+ @Override
+ public String[] getLocations(final InputSplit inputSplit) throws
IOException {
+ return new String[0];
+ }
+ }
+
+ @Test
+ public void testCustomSplitLocationProvider() throws IOException {
+ HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_MODE, "llap");
+
+ conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS,
NoConstructorSplitLocationProvider.class.getName());
+ SplitLocationProvider splitLocationProvider =
Utils.getSplitLocationProvider(conf, LOG);
+ assertTrue(splitLocationProvider instanceof
NoConstructorSplitLocationProvider);
+
+ conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS,
DefaultConstructorSplitLocationProvider.class.getName());
+ splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG);
+ assertTrue(splitLocationProvider instanceof
DefaultConstructorSplitLocationProvider);
+
+ conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS,
ConfConstructorSplitLocationProvider.class.getName());
+ splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG);
+ assertTrue(splitLocationProvider instanceof
ConfConstructorSplitLocationProvider);
+ }
}
