Revert "HIVE-13496. Create initial test data once across multiple test runs - TestCliDriver. (Siddharth Seth, reviewed by Ashutosh Chauhan)"
This reverts commit 976e628fc01911936caa19e61ea3342f3a19455a. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3e43c6d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3e43c6d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3e43c6d Branch: refs/heads/llap Commit: e3e43c6dd89b9643d3f7a8f21c4789af24bba6eb Parents: 940fc7d Author: Siddharth Seth <ss...@apache.org> Authored: Thu Apr 14 11:32:19 2016 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Thu Apr 14 11:32:19 2016 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/QTestUtil.java | 247 ++----------------- pom.xml | 2 - ql/src/test/templates/TestCliDriver.vm | 38 +-- 3 files changed, 22 insertions(+), 265 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 79646cd..2f109ab 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -51,6 +51,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.Deque; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -62,7 +63,6 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -84,7 +84,9 @@ import org.apache.hadoop.hive.common.io.SortPrintStream; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; +import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; +import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.Index; @@ -97,7 +99,6 @@ import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton; import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager; import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -138,8 +139,6 @@ public class QTestUtil { // security property names private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri"; private static final String CRLF = System.getProperty("line.separator"); - private static final String TEST_BUILD_DIR = System.getProperty("test.build.dir"); - private static final String CACHED_DATA_DIR_NAME = "cachedData"; private static final Logger LOG = LoggerFactory.getLogger("QTestUtil"); private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES"; @@ -184,16 +183,6 @@ public class QTestUtil { private final String cleanupScript; private boolean useHBaseMetastore = false; - // Parameters which help tracking cached data generation. - private final String driverName; - private Path cachedDataPath; - private String metaStorePathString; - private Path metaStorePath; - private FileSystem localFs; - private boolean attemptingCacheUsage; - - private boolean dbEtcSetup = false; - public interface SuiteAddTestFunctor { public void addTestToSuite(TestSuite suite, Object setup, String tName); } @@ -389,34 +378,11 @@ public class QTestUtil { } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, - String confDir, String hadoopVer, String initScript, String cleanupScript, - boolean useHBaseMetastore, boolean withLlapIo) throws Exception { - // For now, to avoid changing multiple test templates, a null driver name avoids - // data generation optimizations. - this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - useHBaseMetastore, withLlapIo, null); - } - - public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, - boolean useHBaseMetastore, boolean withLlapIo, String driverName) + boolean useHBaseMetastore, boolean withLlapIo) throws Exception { - this.attemptingCacheUsage = (StringUtils.isEmpty(TEST_BUILD_DIR) || - StringUtils.isEmpty(driverName) || useHBaseMetastore) ? false : true; - this.driverName = driverName; this.outDir = outDir; this.logDir = logDir; - LOG.info("Creating QTestUtil with settings: " - + "driverName=" + driverName - + ", attemptingCacheUsage=" + attemptingCacheUsage - + ", test.build.dir=" + System.getProperty("test.build.dir") - + ", useHbaseMetaStore=" + useHBaseMetastore - + ", withLlapIo=" + withLlapIo - + ", confDir=" + confDir - + ", outDir=" + outDir - + ", logDir=" + logDir - + ", initScript=" + initScript - + ", cleanupScript=" + cleanupScript); this.useHBaseMetastore = useHBaseMetastore; if (confDir != null && !confDir.isEmpty()) { @@ -505,7 +471,6 @@ public class QTestUtil { if (scriptsDir == null) { scriptsDir = new File(".").getAbsolutePath() + "/data/scripts"; } - LOG.info("Using DataDir=" + dataDir + ", ScriptsDir=" + scriptsDir); this.initScript = scriptsDir + File.separator + initScript; this.cleanupScript = scriptsDir + File.separator + cleanupScript; @@ -867,17 +832,6 @@ public class QTestUtil { return; } - if (!attemptingCacheUsage) { - cleanupNonCacheUsage(); - } else { - cleanupCacheUsage(); - } - - FunctionRegistry.unregisterTemporaryUDF("test_udaf"); - FunctionRegistry.unregisterTemporaryUDF("test_error"); - } - - private void cleanupNonCacheUsage() throws Exception { clearTablesCreatedDuringTests(); clearKeysCreatedInTests(); @@ -895,42 +849,21 @@ public class QTestUtil { LOG.info("No cleanup script detected. Skipping."); } - cleanupWarehouseDir(); - } - - private void cleanupCacheUsage() throws IOException { - // Remove the Warehouse and metastore directories completely. - // Also close the current db, since files are going to come in to replace it soon. - Preconditions.checkState(attemptingCacheUsage); - Preconditions.checkNotNull(metaStorePath); - Preconditions.checkNotNull(localFs); - Hive.closeCurrent(); - cleanupMetastoreDir(); - cleanupWarehouseDir(); - } - - private void cleanupWarehouseDir() throws IOException { // delete any contents in the warehouse dir Path p = new Path(testWarehouse); FileSystem fs = p.getFileSystem(conf); try { - FileStatus[] ls = fs.listStatus(p); - for (int i = 0; (ls != null) && (i < ls.length); i++) { + FileStatus [] ls = fs.listStatus(p); + for (int i=0; (ls != null) && (i<ls.length); i++) { fs.delete(ls[i].getPath(), true); } } catch (FileNotFoundException e) { // Best effort } - } - private void cleanupMetastoreDir() throws IOException { - try { - LOG.info("Cleaning up metastore Dir: {}", metaStorePath); - localFs.delete(metaStorePath, true); - } catch (FileNotFoundException e) { - // Best effort - } + FunctionRegistry.unregisterTemporaryUDF("test_udaf"); + FunctionRegistry.unregisterTemporaryUDF("test_error"); } protected void runCreateTableCmd(String createTableCmd) throws Exception { @@ -960,10 +893,6 @@ public class QTestUtil { } public void createSources(String tname) throws Exception { - createSources(tname, false); - } - - public void createSources(String tname, boolean forceCreate) throws Exception { boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname); if(!isSessionStateStarted) { startSessionState(canReuseSession); @@ -972,173 +901,34 @@ public class QTestUtil { if(cliDriver == null) { cliDriver = new CliDriver(); } - + cliDriver.processLine("set test.data.dir=" + testFiles + ";"); File scriptFile = new File(this.initScript); if (!scriptFile.isFile()) { LOG.info("No init script detected. Skipping"); - if (attemptingCacheUsage) { - setupDbsEtc(true, true); - } return; } + conf.setBoolean("hive.test.init.phase", true); - if (!attemptingCacheUsage || forceCreate) { - LOG.info("Creating sources without data caching. attemptingCacheUsage={}, forceCreate={}", - attemptingCacheUsage, forceCreate); - cliDriver.processLine("set test.data.dir=" + testFiles + ";"); - conf.setBoolean("hive.test.init.phase", true); - createSourcesNonCached(scriptFile); - } else { - LOG.info("Creating sources with data caching"); - createSourcesCached(scriptFile); - } - - conf.setBoolean("hive.test.init.phase", false); - } - - private void createSourcesNonCached(File scriptFile) throws IOException { String initCommands = readEntireFileIntoString(scriptFile); LOG.info("Initial setup (" + initScript + "):\n" + initCommands); cliDriver.processLine(initCommands); - } - - private void createSourcesCached(File scriptFile) throws IOException, HiveException { - // First check if the cache already exists. If it does just copy it over. - Path cachedWarehousePath = new Path(cachedDataPath, "warehouse"); - Path cachedMetaStorePtah = new Path(cachedDataPath, "metastore"); - if (localFs.exists(cachedDataPath)) { - if (localFs.exists(cachedWarehousePath) && localFs.exists(cachedMetaStorePtah)) { - LOG.info("Cached data found in {}. Attempting to use it", cachedDataPath); - // Data is alredy cached - // Copy the files over to where they should be - Path warehousePath = new Path(testWarehouse); - FileSystem warehouseFs = warehousePath.getFileSystem(conf); - try { - warehouseFs.delete(warehousePath, false); - } catch (FileNotFoundException e) { - // Does not matter if it does not exist. - } - warehouseFs.copyFromLocalFile(false, cachedWarehousePath, warehousePath); - - try { - localFs.delete(metaStorePath, false); - } catch (IOException e) { - // Does not matter if it does not exist. - } - localFs.copyFromLocalFile(false, cachedMetaStorePtah, metaStorePath); - setupDbsEtc(true, false); - cliDriver.processLine("set test.data.dir=" + testFiles + ";"); - conf.setBoolean("hive.test.init.phase", true); - - return; - } else { - // Something is missing. Cleanup. Re-generate and cache - LOG.info("Partial or no cached data found at {}. Cache will be created", cachedDataPath); - localFs.delete(cachedDataPath, true); - } - } else { - LOG.info("No cached data found at {}. Cache will be created", cachedDataPath); - // No caching. Re-generate the data and cache it. - } - - // Generate and cache the data - setupDbsEtc(true, true); - cliDriver.processLine("set test.data.dir=" + testFiles + ";"); - conf.setBoolean("hive.test.init.phase", true); - createSourcesNonCached(scriptFile); - - // Close the DB so that contents can be copied out safely. - Hive.closeCurrent(); - - // Cache the sources - localFs.mkdirs(cachedDataPath); - - Path warehousePath = new Path(testWarehouse); - FileSystem warehouseFs = warehousePath.getFileSystem(conf); - - warehouseFs.copyToLocalFile(false, warehousePath, cachedWarehousePath, true); - localFs.copyToLocalFile(false, metaStorePath, cachedMetaStorePtah, true); - - // Re-open the DB etc. - setupDbsEtc(true, false); - } - - private static final Pattern metaStoreUriPattern = - Pattern.compile("derby.*?databaseName=(.*?)(;|$)"); - - private String getDerbyDbPath(String jdbcConnectString) { - if (StringUtils.isEmpty(jdbcConnectString)) { - return null; - } - Matcher matcher = metaStoreUriPattern.matcher(jdbcConnectString); - if (matcher.find()) { - return matcher.group(1); - } else { - return null; - } + conf.setBoolean("hive.test.init.phase", false); } public void init() throws Exception { - LOG.info("init"); - testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); - LOG.info("TestWarehouseDir set to: [{}]", testWarehouse); - if (attemptingCacheUsage) { - // The derby path comes from METASTORECONNECTURLKEY. Default ends up being target/junit_metastore_db - String metaStoreConnectUrl = conf.getVar(ConfVars.METASTORECONNECTURLKEY); - LOG.info("MetastoreConnectUrl: " + metaStoreConnectUrl); - metaStorePathString = getDerbyDbPath(metaStoreConnectUrl); - - if (metaStorePathString == null) { - LOG.warn( - "Disabling attempted cache usage since metastore path cannot be determined from {}", - metaStoreConnectUrl); - attemptingCacheUsage = false; - } else { - LOG.info("Metastore url path: " + metaStorePathString); - metaStorePath = new Path(metaStorePathString); - if (metaStorePath.isAbsolute() && metaStorePathString.split(File.separator).length >= 3) { - // Turn this on only if the path is absolute, and is at least 3 deep - since we'll be deleting files later. - localFs = FileSystem.getLocal(conf).getRaw(); - assert(TEST_BUILD_DIR != null); - cachedDataPath = new Path(TEST_BUILD_DIR, CACHED_DATA_DIR_NAME); - cachedDataPath = new Path(cachedDataPath, driverName); - LOG.info("Using cachedDataPath: " + cachedDataPath); - } else { - LOG.warn( - "Disableing attempted cache usage since metastore path may not be absolute, or depth is < 3. MetaStorePath={}", - metaStorePathString); - metaStorePath = null; - attemptingCacheUsage = false; - } - } - } + testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE); String execEngine = conf.get("hive.execution.engine"); conf.set("hive.execution.engine", "mr"); SessionState.start(conf); conf.set("hive.execution.engine", execEngine); - - if (!attemptingCacheUsage) { - setupDbsEtc(true, true); - } - } - - private void setupDbsEtc(boolean force, boolean isNewDb) throws HiveException { - if (!dbEtcSetup || force) { - if (isNewDb) { - db = Hive.get(conf); - } else { - db = Hive.getWithFastCheck(conf, false); - } - LOG.info("Obtained db"); - drv = new Driver(conf); - drv.init(); - pd = new ParseDriver(); - sem = new SemanticAnalyzer(conf); - dbEtcSetup = true; - } + db = Hive.get(conf); + drv = new Driver(conf); + drv.init(); + pd = new ParseDriver(); + sem = new SemanticAnalyzer(conf); } public void init(String tname) throws Exception { @@ -1154,9 +944,8 @@ public class QTestUtil { public String cliInit(String tname, boolean recreate) throws Exception { if (recreate) { cleanUp(tname); - createSources(tname, true); + createSources(tname); } - setupDbsEtc(false, true); HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER, "org.apache.hadoop.hive.ql.security.DummyAuthenticator"); http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 08ef998..77cfaeb 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,6 @@ <test.hive.hadoop.classpath>${maven.test.classpath}</test.hive.hadoop.classpath> <test.log4j.scheme>file://</test.log4j.scheme> <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir> - <test.build.dir>${project.build.directory}</test.build.dir> <test.tmp.dir.uri>file://${test.tmp.dir}</test.tmp.dir.uri> <test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir> <test.warehouse.scheme>pfile://</test.warehouse.scheme> @@ -1027,7 +1026,6 @@ <test.data.dir>${basedir}/${hive.path.to.root}/data/files</test.data.dir> <test.tmp.dir>${test.tmp.dir}</test.tmp.dir> <test.tmp.dir.uri>${test.tmp.dir.uri}</test.tmp.dir.uri> - <test.build.dir>${test.build.dir}</test.build.dir> <test.dfs.mkdir>${test.dfs.mkdir}</test.dfs.mkdir> <test.output.overwrite>${test.output.overwrite}</test.output.overwrite> <test.warehouse.dir>${test.warehouse.scheme}${test.warehouse.dir}</test.warehouse.dir> http://git-wip-us.apache.org/repos/asf/hive/blob/e3e43c6d/ql/src/test/templates/TestCliDriver.vm ---------------------------------------------------------------------- diff --git a/ql/src/test/templates/TestCliDriver.vm b/ql/src/test/templates/TestCliDriver.vm index 1961c75..72cfab9 100644 --- a/ql/src/test/templates/TestCliDriver.vm +++ b/ql/src/test/templates/TestCliDriver.vm @@ -17,34 +17,23 @@ */ package org.apache.hadoop.hive.cli; -import com.google.common.base.Stopwatch; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import java.util.concurrent.TimeUnit; - public class $className { - private static final Logger LOG = LoggerFactory.getLogger(${className}.class); - private static final String HIVE_ROOT = QTestUtil.ensurePathEndsInSlash(System.getProperty("hive.root")); private static QTestUtil qt; static { - Stopwatch stopwatch = new Stopwatch().start(); - String message = "Starting TestCliDriver run at " + System.currentTimeMillis(); - LOG.info(message); - System.err.println(message); MiniClusterType miniMR = MiniClusterType.valueForString("$clusterMode"); String hiveConfDir = "$hiveConfDir"; String initScript = "$initScript"; @@ -55,29 +44,15 @@ public class $className { if (!hiveConfDir.isEmpty()) { hiveConfDir = HIVE_ROOT + hiveConfDir; } - // TODO Is ZK startup required for TestCliDriver - // TODO Is LlapIo enabled required for TestCliDriver qt = new QTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, - hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true, "$className"); - message = "QTestUtil instance created. ElapsedTimeSinceStart=" + stopwatch.elapsed( - TimeUnit.MILLISECONDS); - LOG.info(message); - System.err.println(message); + hiveConfDir, hadoopVer, initScript, cleanupScript, useHBaseMetastore, true); // do a one time initialization qt.cleanUp(); - message = "Initialization cleanup done. ElapsedTimeSinceStart=" + stopwatch.elapsed(TimeUnit.MILLISECONDS); - LOG.info(message); - System.err.println(message); - qt.createSources(); - message = "Initialization createSources done. ElapsedTimeSinceStart=" + stopwatch.elapsed(TimeUnit.MILLISECONDS); - LOG.info(message); - System.err.println(message); } catch (Exception e) { - System.err.println("Exception: " + e.getMessage() + ". ElapsedTimeSinceStart=" - + stopwatch.elapsed(TimeUnit.MILLISECONDS)); + System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); System.err.flush(); fail("Unexpected exception in static initialization: "+e.getMessage()); @@ -87,7 +62,6 @@ public class $className { @Before public void setUp() { try { - // TODO This restarts ZK for each test. Is that requried ? qt.clearTestSideEffects(); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); @@ -139,9 +113,7 @@ public class $className { private void runTest(String tname, String fname, String fpath) throws Exception { long startTime = System.currentTimeMillis(); try { - String message = "Begin query: " + fname + ", startTime=" + startTime; - System.err.println(message); - LOG.info(message); + System.err.println("Begin query: " + fname); qt.addFile(fpath); @@ -164,9 +136,7 @@ public class $className { } long elapsedTime = System.currentTimeMillis() - startTime; - String message = "Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"; - System.err.println(message); - LOG.info(message); + System.err.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s"); assertTrue("Test passed", true); } }
