This is an automated email from the ASF dual-hosted git repository. prasanthj pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new e633ff8 HIVE-23582: LLAP: Make SplitLocationProvider impl pluggable (Prasanth Jayachandran reviewed by Gopal V) e633ff8 is described below commit e633ff8186c14377443881d8eb02f2644e20b510 Author: Prasanth Jayachandran <prasan...@apache.org> AuthorDate: Fri Jun 5 16:46:27 2020 -0700 HIVE-23582: LLAP: Make SplitLocationProvider impl pluggable (Prasanth Jayachandran reviewed by Gopal V) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 4 ++ .../org/apache/hadoop/hive/ql/exec/tez/Utils.java | 27 ++++++++++- .../apache/hadoop/hive/ql/exec/tez/TestUtils.java | 53 ++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index abd12c9..1464d6a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4704,6 +4704,10 @@ public class HiveConf extends Configuration { "instead of using the locations provided by the split itself. If there is no llap daemon " + "running, fall back to locations provided by the split. This is effective only if " + "hive.execution.mode is llap"), + LLAP_SPLIT_LOCATION_PROVIDER_CLASS("hive.llap.split.location.provider.class", + "org.apache.hadoop.hive.ql.exec.tez.HostAffinitySplitLocationProvider", + "Split location provider class to use during split generation for LLAP. This class should implement\n" + + "org.apache.hadoop.mapred.split.SplitLocationProvider interface"), LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true, "Whether LLAP should reject permissive ACLs in some cases (e.g. its own management\n" + "protocol or ZK paths), similar to how ssh refuses a key with bad access permissions."), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java index d266bb1..92d64df 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/Utils.java @@ -28,6 +28,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.split.SplitLocationProvider; import org.apache.tez.common.counters.TezCounters; @@ -48,8 +50,29 @@ public class Utils { && HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS) && useCacheAffinity; SplitLocationProvider splitLocationProvider; - LOG.info("SplitGenerator using llap affinitized locations: " + useCustomLocations); - if (useCustomLocations) { + final String locationProviderClass = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS); + final boolean customLocationProvider = + !HostAffinitySplitLocationProvider.class.getName().equals(locationProviderClass); + LOG.info("SplitGenerator using llap affinitized locations: {} locationProviderClass: {}", useCustomLocations, locationProviderClass); + if (customLocationProvider) { + SplitLocationProvider locationProviderImpl; + try { + // the implementation of SplitLocationProvider may have Configuration as a single arg constructor, so we try + // invoking that constructor first. If that does not exist, the fallback will use no-arg constructor. + locationProviderImpl = JavaUtils + .newInstance(JavaUtils.getClass(locationProviderClass, SplitLocationProvider.class), + new Class<?>[]{Configuration.class}, new Object[]{conf}); + } catch (Exception e) { + LOG.warn("Unable to instantiate {} class. Will try no-arg constructor invocation..", locationProviderClass, e); + try { + locationProviderImpl = JavaUtils.newInstance(JavaUtils.getClass(locationProviderClass, + SplitLocationProvider.class)); + } catch (Exception ex) { + throw new IOException(ex); + } + } + return locationProviderImpl; + } else if (useCustomLocations) { LlapRegistryService serviceRegistry = LlapRegistryService.getClient(conf); return getCustomSplitLocationProvider(serviceRegistry, LOG); } else { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java index d9aa032..71ce812 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.tez; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.registry.LlapServiceInstance; import org.apache.hadoop.hive.llap.registry.LlapServiceInstanceSet; @@ -50,6 +51,7 @@ import java.util.List; import java.util.Set; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -156,4 +158,55 @@ public class TestUtils { knownLocations.remove(null); assertArrayEquals(expectedLocations, knownLocations.toArray(new String[] {})); } + + public static class NoConstructorSplitLocationProvider implements SplitLocationProvider { + + @Override + public String[] getLocations(final InputSplit inputSplit) throws IOException { + return new String[0]; + } + } + + public static class DefaultConstructorSplitLocationProvider implements SplitLocationProvider { + + public DefaultConstructorSplitLocationProvider() { + + } + + @Override + public String[] getLocations(final InputSplit inputSplit) throws IOException { + return new String[0]; + } + } + + public static class ConfConstructorSplitLocationProvider implements SplitLocationProvider { + private Configuration configuration; + + public ConfConstructorSplitLocationProvider(Configuration conf) { + this.configuration = conf; + } + + @Override + public String[] getLocations(final InputSplit inputSplit) throws IOException { + return new String[0]; + } + } + + @Test + public void testCustomSplitLocationProvider() throws IOException { + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_MODE, "llap"); + + conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS, NoConstructorSplitLocationProvider.class.getName()); + SplitLocationProvider splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG); + assertTrue(splitLocationProvider instanceof NoConstructorSplitLocationProvider); + + conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS, DefaultConstructorSplitLocationProvider.class.getName()); + splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG); + assertTrue(splitLocationProvider instanceof DefaultConstructorSplitLocationProvider); + + conf.setVar(HiveConf.ConfVars.LLAP_SPLIT_LOCATION_PROVIDER_CLASS, ConfConstructorSplitLocationProvider.class.getName()); + splitLocationProvider = Utils.getSplitLocationProvider(conf, LOG); + assertTrue(splitLocationProvider instanceof ConfConstructorSplitLocationProvider); + } }