hive git commit: HIVE-14793. Allow ptest branch to be specified, PROFILE override. (Siddharth Seth, reviewed by Sergio Peña)

2016-09-20 Thread sseth 
Repository: hive
Updated Branches:
  refs/heads/master 19774029c -> 62c45de1c


HIVE-14793. Allow ptest branch to be specified, PROFILE override. (Siddharth 
Seth, reviewed by Sergio Peña)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/62c45de1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/62c45de1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/62c45de1

Branch: refs/heads/master
Commit: 62c45de1c12b2f8db6d726462cc93382aedc0905
Parents: 1977402
Author: Siddharth Seth 
Authored: Tue Sep 20 15:37:49 2016 -0700
Committer: Siddharth Seth 
Committed: Tue Sep 20 15:37:49 2016 -0700

--
 dev-support/jenkins-execute-build.sh | 28 +---
 1 file changed, 21 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/62c45de1/dev-support/jenkins-execute-build.sh
--
diff --git a/dev-support/jenkins-execute-build.sh 
b/dev-support/jenkins-execute-build.sh
index b2ba8e9..2142942 100644
--- a/dev-support/jenkins-execute-build.sh
+++ b/dev-support/jenkins-execute-build.sh
@@ -26,7 +26,16 @@ build_ptest_client() {
test -d $PTEST_BUILD_DIR || mkdir -p $PTEST_BUILD_DIR
cd $PTEST_BUILD_DIR &&  rm -rf hive
 
-   git clone --depth 1 https://github.com/apache/hive.git
+  unset GIT_CLONE_ARGS
+  if [ -n "${PTEST_GIT_BRANCH}" ]; then
+GIT_CLONE_ARGS=" -b ${PTEST_GIT_BRANCH}"
+  fi
+  if [ -z "${PTEST_GIT_REPO}" ]; then
+PTEST_GIT_REPO=https://github.com/apache/hive.git
+  fi
+  GIT_CLONE_ARGS=${GIT_CLONE_ARGS}" ${PTEST_GIT_REPO} hive"
+
+   git clone --depth 1 ${GIT_CLONE_ARGS}
cd hive/testutils/ptest2
mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 
-Dmaven.repo.local=$MVN_REPO_LOCAL
 }
@@ -99,10 +108,13 @@ if [ -n "$JIRA_ISSUE" ]; then
fail "attachment $attachment_id is already tested for 
$JIRA_ISSUE"
fi
 
-   BUILD_PROFILE=`get_branch_profile $JIRA_PATCH_URL $JIRA_INFO_FILE`
-   if [ -z "$BUILD_PROFILE" ]; then
-   BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
-   fi
+  # Use the BUILD_PROFILE if it is provided. 
+  if [ -z ${BUILD_PROFILE} ]; then
+ BUILD_PROFILE=`get_branch_profile $JIRA_PATCH_URL $JIRA_INFO_FILE`
+ if [ -z "$BUILD_PROFILE" ]; then
+   BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+ fi
+  fi
 
if is_clear_cache_set $JIRA_INFO_FILE; then
optionalArgs+=(--clearLibraryCache)
@@ -112,8 +124,10 @@ if [ -n "$JIRA_ISSUE" ]; then
 
echo "ISSUE: $JIRA_ISSUE PROFILE: $BUILD_PROFILE"
 else
-   # If not JIRA is specified, then use a default profile
-   BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+   # If not JIRA is specified, and no BUILD_PROFILE provided, then use a 
default profile
+  if [ -z ${BUILD_PROFILE} ]; then
+   BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+  fi
 
echo "ISSUE: unspecified PROFILE: $BUILD_PROFILE"
 fi

hive git commit: HIVE-14624 : LLAP: Use FQDN when submitting work to LLAP (Sergey Shelukhin, reviewed by Siddharth Seth)

2016-09-20 Thread sershe 
Repository: hive
Updated Branches:
  refs/heads/master e297a157c -> 19774029c


HIVE-14624 : LLAP: Use FQDN when submitting work to LLAP (Sergey Shelukhin, 
reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/19774029
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/19774029
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/19774029

Branch: refs/heads/master
Commit: 19774029c4c1d90982354c36840bb485d74faaf1
Parents: e297a15
Author: Sergey Shelukhin 
Authored: Tue Sep 20 11:30:49 2016 -0700
Committer: Sergey Shelukhin 
Committed: Tue Sep 20 11:30:59 2016 -0700

--
 .../java/org/apache/hadoop/hive/conf/HiveConf.java|  3 +++
 .../java/org/apache/hadoop/hive/llap/LlapUtil.java| 12 
 .../apache/hadoop/hive/llap/LlapBaseInputFormat.java  |  6 +++---
 .../hive/llap/tezplugins/LlapTaskCommunicator.java| 14 ++
 .../llap/tezplugins/TestLlapTaskCommunicator.java |  5 +
 5 files changed, 33 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/19774029/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 301159e..ccdfca6 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -393,6 +393,7 @@ public class HiveConf extends Configuration {
 llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_CONTAINER_ID.varname);
 llapDaemonVarsSetLocal.add(ConfVars.LLAP_VALIDATE_ACLS.varname);
 llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_LOGGER.varname);
+llapDaemonVarsSetLocal.add(ConfVars.LLAP_DAEMON_AM_USE_FQDN.varname);
   }
 
   /**
@@ -2909,6 +2910,8 @@ public class HiveConf extends Configuration {
   new TimeValidator(TimeUnit.MILLISECONDS),
   "Amount of time to wait on connection failures to the AM from an LLAP 
daemon before\n" +
   "considering the AM to be dead.", 
"llap.am.liveness.connection.timeout-millis"),
+LLAP_DAEMON_AM_USE_FQDN("hive.llap.am.use.fqdn", false,
+"Whether to use FQDN of the AM machine when submitting work to LLAP."),
 // Not used yet - since the Writable RPC engine does not support this 
policy.
 LLAP_DAEMON_AM_LIVENESS_CONNECTION_SLEEP_BETWEEN_RETRIES_MS(
   "hive.llap.am.liveness.connection.sleep.between.retries.ms", "2000ms",

http://git-wip-us.apache.org/repos/asf/hive/blob/19774029/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
--
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java 
b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
index 0c04d9d..8352943 100644
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapUtil.java
@@ -14,6 +14,8 @@
 package org.apache.hadoop.hive.llap;
 
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -25,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import 
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto.Builder;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
@@ -180,4 +183,13 @@ public class LlapUtil {
   return sb.toString();
 }
   }
+
+  public static String getAmHostNameFromAddress(InetSocketAddress address, 
Configuration conf) {
+if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_DAEMON_AM_USE_FQDN)) 
{
+  return address.getHostName();
+}
+InetAddress ia = address.getAddress();
+// getCanonicalHostName would either return FQDN, or an IP.
+return (ia == null) ? address.getHostName() : ia.getCanonicalHostName();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/19774029/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
--
diff --git 
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java 
b/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
index 7dae4fc..288a8eb 100644
--- 
a/llap-ext-client/src/java/org/apache/hadoop/hive/llap/LlapBaseInputFormat.java
+++

hive git commit: HIVE-14651. Add a local cluster for Tez and LLAP. (Siddharth Seth, reviewed by Prasanth Jayachandran, Sergey Shelukhin)

2016-09-20 Thread sseth 
Repository: hive
Updated Branches:
  refs/heads/master 3c55115b6 -> e297a157c


HIVE-14651. Add a local cluster for Tez and LLAP. (Siddharth Seth,reviewed by 
Prasanth Jayachandran, Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e297a157
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e297a157
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e297a157

Branch: refs/heads/master
Commit: e297a157cfa57f0bd08843bf770856b2f168da75
Parents: 3c55115
Author: Siddharth Seth 
Authored: Tue Sep 20 11:09:11 2016 -0700
Committer: Siddharth Seth 
Committed: Tue Sep 20 11:09:11 2016 -0700

--
 data/conf/llap/tez-site.xml |  12 +
 data/conf/tez/hive-site.xml |  10 -
 .../org/apache/hadoop/hive/ql/QTestUtil.java| 281 ++-
 .../tezplugins/LlapTaskSchedulerService.java|   2 +-
 .../apache/hadoop/hive/shims/Hadoop23Shims.java |  73 +++--
 .../apache/hadoop/hive/shims/HadoopShims.java   |   2 +
 6 files changed, 274 insertions(+), 106 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hive/blob/e297a157/data/conf/llap/tez-site.xml
--
diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml
index 940f390..6f1b9d2 100644
--- a/data/conf/llap/tez-site.xml
+++ b/data/conf/llap/tez-site.xml
@@ -1,6 +1,18 @@
 
+
   
 tez.am.dag.scheduler.class
 
org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled
   
+
+  
+  
+tez.am.task.max.failed.attempts
+2
+  
+  
+tez.runtime.shuffle.connect.timeout
+2
+  
+
 

http://git-wip-us.apache.org/repos/asf/hive/blob/e297a157/data/conf/tez/hive-site.xml
--
diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml
index f3e4dae..dbff10c 100644
--- a/data/conf/tez/hive-site.xml
+++ b/data/conf/tez/hive-site.xml
@@ -269,16 +269,6 @@
 
 
 
-  hive.metastore.fastpath
-  true
-
-
-
-  hive.metastore.rawstore.impl
-  org.apache.hadoop.hive.metastore.hbase.HBaseStore
-
-
-
   hive.orc.splits.ms.footer.cache.enabled
   true
 

http://git-wip-us.apache.org/repos/asf/hive/blob/e297a157/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
--
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java 
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f456dfb..0dfd727 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -63,6 +63,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
@@ -91,6 +92,7 @@ import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
 import org.apache.hadoop.hive.llap.io.api.LlapProxy;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Index;
+import org.apache.hadoop.hive.metastore.hbase.HBaseStore;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -117,6 +119,7 @@ import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.util.Shell;
 import org.apache.hive.common.util.StreamPrinter;
+import org.apache.logging.log4j.util.Strings;
 import org.apache.tools.ant.BuildException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
@@ -147,8 +150,13 @@ public class QTestUtil {
   private final static String defaultCleanupScript = "q_test_cleanup.sql";
   private final String[] testOnlyCommands = new String[]{"crypto"};
 
+  private static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // 
typically target/tmp
+  private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically 
target
+
   private String testWarehouse;
   private final String testFiles;
+  private final boolean useLocalFs;
+  private final boolean localMode;
   protected final String outDir;
   protected final String logDir;
   private final TreeMap qMap;
@@ -161,7 +169,7 @@ public class QTestUtil {
   private final Set qJavaVersionSpecificOutput;
   private static final String SORT_SUFFIX = ".sorted";
   private final HashSet srcTables;
-  private static MiniClusterType clusterType = MiniClusterType.none;
+  private final MiniClusterType clusterType;
   private