Repository: hive Updated Branches: refs/heads/master 4b04c9c93 -> 721304037
HIVE-14625. Minor qtest fixes. (Siddharth Seth, reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/72130403 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/72130403 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/72130403 Branch: refs/heads/master Commit: 721304037eb54cb1c2810897d4dd33a7292a5dd7 Parents: 4b04c9c Author: Siddharth Seth <[email protected]> Authored: Thu Aug 25 12:39:49 2016 -0700 Committer: Siddharth Seth <[email protected]> Committed: Thu Aug 25 12:39:49 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hive/cli/control/CoreCliDriver.java | 91 ++++++++++++++++---- .../org/apache/hadoop/hive/ql/QTestUtil.java | 14 ++- .../hive/util/ElapsedTimeLoggingWrapper.java | 43 +++++++++ 3 files changed, 129 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/72130403/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index 5435f9f..db58f1d 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -20,16 +20,23 @@ package org.apache.hadoop.hive.cli.control; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Stopwatch; import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; +import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CoreCliDriver extends CliAdapter { + private static final Logger LOG = LoggerFactory.getLogger(CoreCliDriver.class); private static QTestUtil qt; public CoreCliDriver(AbstractCliConfig testCliConfig) { @@ -39,19 +46,41 @@ public class CoreCliDriver extends CliAdapter { @Override @BeforeClass public void beforeClass() { - MiniClusterType miniMR =cliConfig.getClusterType(); - String hiveConfDir = cliConfig.getHiveConfDir(); - String initScript = cliConfig.getInitScript(); - String cleanupScript = cliConfig.getCleanupScript(); - boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase; + String message = "Starting " + CoreCliDriver.class.getName() + " run at " + System.currentTimeMillis(); + LOG.info(message); + System.err.println(message); + final MiniClusterType miniMR =cliConfig.getClusterType(); + final String hiveConfDir = cliConfig.getHiveConfDir(); + final String initScript = cliConfig.getInitScript(); + final String cleanupScript = cliConfig.getCleanupScript(); + final boolean useHBaseMetastore = cliConfig.getMetastoreType() == MetastoreType.hbase; try { - String hadoopVer = cliConfig.getHadoopVersion(); - qt = new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true); + final String hadoopVer = cliConfig.getHadoopVersion(); + + qt = new ElapsedTimeLoggingWrapper<QTestUtil>() { + @Override + public QTestUtil invokeInternal() throws Exception { + return new QTestUtil((cliConfig.getResultsDir()), (cliConfig.getLogDir()), miniMR, + hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true); + } + }.invoke("QtestUtil instance created", LOG, true); // do a one time initialization - qt.cleanUp(); - qt.createSources(); + new ElapsedTimeLoggingWrapper<Void>() { + @Override + public Void invokeInternal() throws Exception { + qt.cleanUp(); + return null; + } + }.invoke("Initialization cleanup done.", LOG, true); + + new ElapsedTimeLoggingWrapper<Void>() { + @Override + public Void invokeInternal() throws Exception { + qt.createSources(); + return null; + } + }.invoke("Initialization createSources done.", LOG, true); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); @@ -65,7 +94,13 @@ public class CoreCliDriver extends CliAdapter { @Before public void setUp() { try { - qt.clearTestSideEffects(); + new ElapsedTimeLoggingWrapper<Void>() { + @Override + public Void invokeInternal() throws Exception { + qt.clearTestSideEffects(); + return null; + } + }.invoke("PerTestSetup done.", LOG, false); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -78,7 +113,13 @@ public class CoreCliDriver extends CliAdapter { @After public void tearDown() { try { - qt.clearPostTestEffects(); + new ElapsedTimeLoggingWrapper<Void>() { + @Override + public Void invokeInternal() throws Exception { + qt.clearPostTestEffects(); + return null; + } + }.invoke("PerTestTearDown done.", LOG, false); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -91,7 +132,13 @@ public class CoreCliDriver extends CliAdapter { @AfterClass public void shutdown() throws Exception { try { - qt.shutdown(); + new ElapsedTimeLoggingWrapper<Void>() { + @Override + public Void invokeInternal() throws Exception { + qt.shutdown(); + return null; + } + }.invoke("Teardown done.", LOG, false); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); @@ -105,33 +152,43 @@ public class CoreCliDriver extends CliAdapter { @Override public void runTest(String tname, String fname, String fpath) throws Exception { - long startTime = System.currentTimeMillis(); + Stopwatch sw = new Stopwatch().start(); + boolean skipped = false; + boolean failed = false; try { + LOG.info("Begin query: " + fname); System.err.println("Begin query: " + fname); qt.addFile(fpath); if (qt.shouldBeSkipped(fname)) { + LOG.info("Test " + fname + " skipped"); System.err.println("Test " + fname + " skipped"); + skipped = true; return; } qt.cliInit(fname, false); int ecode = qt.executeClient(fname); if (ecode != 0) { + failed = true; qt.failed(ecode, fname, debugHint); } ecode = qt.checkCliDriverResults(fname); if (ecode != 0) { + failed = true; qt.failedDiff(ecode, fname, debugHint); } } catch (Throwable e) { + failed = true; qt.failed(e, fname, debugHint); + } finally { + String message = "Done query" + fname + ". succeeded=" + !failed + ", skipped=" + skipped + + ". ElapsedTime(ms)=" + sw.stop().elapsed(TimeUnit.MILLISECONDS); + LOG.info(message); + System.err.println(message); } - - long elapsedTime = System.currentTimeMillis() - startTime; - System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); assertTrue("Test passed", true); } } http://git-wip-us.apache.org/repos/asf/hive/blob/72130403/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 358ba51..4d4a929 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -388,6 +388,10 @@ public class QTestUtil { String confDir, String hadoopVer, String initScript, String cleanupScript, boolean useHBaseMetastore, boolean withLlapIo) throws Exception { + LOG.info("Setting up QtestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + + " hadoopVer={}, initScript={}, cleanupScript={}, useHbaseMetaStore={}, withLlapIo={}" + , outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, + useHBaseMetastore, withLlapIo); this.outDir = outDir; this.logDir = logDir; this.useHBaseMetastore = useHBaseMetastore; @@ -854,7 +858,10 @@ public class QTestUtil { cliDriver = new CliDriver(); } SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true); - cliDriver.processLine(cleanupCommands); + int result = cliDriver.processLine(cleanupCommands); + if (result != 0) { + Assert.fail("Failed during cleanup processLine with code=" + result); + } SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false); } else { LOG.info("No cleanup script detected. Skipping."); @@ -923,7 +930,10 @@ public class QTestUtil { String initCommands = readEntireFileIntoString(scriptFile); LOG.info("Initial setup (" + initScript + "):\n" + initCommands); - cliDriver.processLine(initCommands); + int result = cliDriver.processLine(initCommands); + if (result != 0) { + Assert.fail("Failed during createSurces processLine with code=" + result); + } conf.setBoolean("hive.test.init.phase", false); } http://git-wip-us.apache.org/repos/asf/hive/blob/72130403/itests/util/src/main/java/org/apache/hadoop/hive/util/ElapsedTimeLoggingWrapper.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/util/ElapsedTimeLoggingWrapper.java b/itests/util/src/main/java/org/apache/hadoop/hive/util/ElapsedTimeLoggingWrapper.java new file mode 100644 index 0000000..061a918 --- /dev/null +++ b/itests/util/src/main/java/org/apache/hadoop/hive/util/ElapsedTimeLoggingWrapper.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.util; + +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Stopwatch; +import org.slf4j.Logger; + +public abstract class ElapsedTimeLoggingWrapper<T> { + + public abstract T invokeInternal() throws Exception; + + public T invoke(String message, Logger LOG, boolean toStdErr) throws Exception { + Stopwatch sw = new Stopwatch().start(); + try { + T retVal = invokeInternal(); + return retVal; + } finally { + String logMessage = message + " ElapsedTime(ms)=" + sw.stop().elapsed(TimeUnit.MILLISECONDS); + LOG.info(logMessage); + if (toStdErr) { + System.err.println(logMessage); + } + } + } +}
