Author: vikram Date: Tue Sep 30 18:36:56 2014 New Revision: 1628512 URL: http://svn.apache.org/r1628512 Log: HIVE-7857: Hive query fails after Tez session times out (Vikram Dixit K, reviewed by Gunther Hagleitner)
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1628512&r1=1628511&r2=1628512&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java Tue Sep 30 18:36:56 2014 @@ -78,7 +78,7 @@ public class TezJobMonitor { try { for (TezSessionState s: TezSessionState.getOpenSessions()) { System.err.println("Shutting down tez session."); - TezSessionPoolManager.getInstance().close(s); + TezSessionPoolManager.getInstance().close(s, false); } } catch (Exception e) { // ignore Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1628512&r1=1628511&r2=1628512&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java Tue Sep 30 18:36:56 2014 @@ -168,10 +168,10 @@ public class TezSessionPoolManager { // session in the SessionState } - public void close(TezSessionState tezSessionState) throws Exception { + public void close(TezSessionState tezSessionState, boolean keepTmpDir) throws Exception { LOG.info("Closing tez session default? " + tezSessionState.isDefault()); if (!tezSessionState.isDefault()) { - tezSessionState.close(false); + tezSessionState.close(keepTmpDir); } } @@ -262,24 +262,24 @@ public class TezSessionPoolManager { } if (session != null) { - close(session); + close(session, false); } return getSession(conf, doOpen, forceCreate); } - public void closeAndOpen(TezSessionState sessionState, HiveConf conf) + public void closeAndOpen(TezSessionState sessionState, HiveConf conf, boolean keepTmpDir) throws Exception { - closeAndOpen(sessionState, conf, null); + closeAndOpen(sessionState, conf, null, keepTmpDir); } public void closeAndOpen(TezSessionState sessionState, HiveConf conf, - String[] additionalFiles) throws Exception { + String[] additionalFiles, boolean keepTmpDir) throws Exception { HiveConf sessionConf = sessionState.getConf(); if (sessionConf != null && sessionConf.get("tez.queue.name") != null) { conf.set("tez.queue.name", sessionConf.get("tez.queue.name")); } - close(sessionState); + close(sessionState, keepTmpDir); sessionState.open(conf, additionalFiles); } } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1628512&r1=1628511&r2=1628512&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Tue Sep 30 18:36:56 2014 @@ -370,7 +370,7 @@ public class TezTask extends Task<TezWor console.printInfo("Tez session was closed. Reopening..."); // close the old one, but keep the tmp files around - TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars); + TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, true); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1628512&r1=1628511&r2=1628512&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Tue Sep 30 18:36:56 2014 @@ -1245,7 +1245,7 @@ public class SessionState { try { if (tezSessionState != null) { - TezSessionPoolManager.getInstance().close(tezSessionState); + TezSessionPoolManager.getInstance().close(tezSessionState, false); } } catch (Exception e) { LOG.info("Error closing tez session", e); Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java?rev=1628512&r1=1628511&r2=1628512&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java Tue Sep 30 18:36:56 2014 @@ -165,7 +165,7 @@ public class TestTezSessionPool { TezSessionState session = Mockito.mock(TezSessionState.class); Mockito.when(session.isDefault()).thenReturn(false); - poolManager.closeAndOpen(session, conf); + poolManager.closeAndOpen(session, conf, false); Mockito.verify(session).close(false); Mockito.verify(session).open(conf, null); @@ -178,7 +178,7 @@ public class TestTezSessionPool { Mockito.when(session.isDefault()).thenReturn(false); String[] extraResources = new String[] { "file:///tmp/foo.jar" }; - poolManager.closeAndOpen(session, conf, extraResources); + poolManager.closeAndOpen(session, conf, extraResources, false); Mockito.verify(session).close(false); Mockito.verify(session).open(conf, extraResources); Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java?rev=1628512&r1=1628511&r2=1628512&view=diff ============================================================================== --- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java (original) +++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java Tue Sep 30 18:36:56 2014 @@ -214,7 +214,7 @@ public class TestTezTask { new String[0], Collections.<String,LocalResource> emptyMap()); // validate close/reopen verify(sessionState, times(1)).open(any(HiveConf.class), any(String[].class)); - verify(sessionState, times(1)).close(eq(false)); // now uses pool after HIVE-7043 + verify(sessionState, times(1)).close(eq(true)); // now uses pool after HIVE-7043 verify(session, times(2)).submitDAG(any(DAG.class)); }