HIVE-13673: LLAP: handle case where no service instance is found on the host specified in the input split (Jason Dere, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fa59d475 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fa59d475 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fa59d475 Branch: refs/heads/hive-14535 Commit: fa59d47531edd3c7357c239046e9f72f3f3e0550 Parents: 04246db Author: Jason Dere <[email protected]> Authored: Fri May 19 10:46:13 2017 -0700 Committer: Jason Dere <[email protected]> Committed: Fri May 19 10:46:13 2017 -0700 ---------------------------------------------------------------------- .../hadoop/hive/llap/LlapBaseInputFormat.java | 24 +++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/fa59d475/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java ---------------------------------------------------------------------- diff --git a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java index 15a81da..eb93241 100644 --- a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java +++ b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java @@ -29,7 +29,9 @@ import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; @@ -94,6 +96,7 @@ public class LlapBaseInputFormat<V extends WritableComparable<?>> private String user; // "hive", private String pwd; // "" private String query; + private final Random rand = new Random(); public static final String URL_KEY = "llap.if.hs2.connection"; public static final String QUERY_KEY = "llap.if.query"; @@ -101,6 +104,7 @@ public class LlapBaseInputFormat<V extends WritableComparable<?>> public static final String PWD_KEY = "llap.if.pwd"; public final String SPLIT_QUERY = "select get_splits(\"%s\",%d)"; + public static final ServiceInstance[] serviceInstanceArray = new ServiceInstance[0]; public LlapBaseInputFormat(String url, String user, String pwd, String query) { this.url = url; @@ -234,7 +238,11 @@ public class LlapBaseInputFormat<V extends WritableComparable<?>> ServiceInstance serviceInstance = getServiceInstanceForHost(registryService, host); if (serviceInstance == null) { - throw new IOException("No service instances found for " + host + " in registry"); + LOG.info("No service instances found for " + host + " in registry."); + serviceInstance = getServiceInstanceRandom(registryService); + if (serviceInstance == null) { + throw new IOException("No service instances found in registry"); + } } return serviceInstance; @@ -272,6 +280,20 @@ public class LlapBaseInputFormat<V extends WritableComparable<?>> return serviceInstance; } + + private ServiceInstance getServiceInstanceRandom(LlapRegistryService registryService) throws IOException { + ServiceInstanceSet instanceSet = registryService.getInstances(); + ServiceInstance serviceInstance = null; + + LOG.info("Finding random live service instance"); + Collection<ServiceInstance> allInstances = instanceSet.getAll(); + if (allInstances.size() > 0) { + int randIdx = rand.nextInt() % allInstances.size(); + serviceInstance = allInstances.toArray(serviceInstanceArray)[randIdx]; + } + return serviceInstance; + } + private ServiceInstance selectServiceInstance(Set<ServiceInstance> serviceInstances) { if (serviceInstances == null || serviceInstances.isEmpty()) { return null;
