Repository: tajo Updated Branches: refs/heads/master e656ee287 -> fd73074f2
TAJO-1323: Cleanup the unstable test case. (jinho) Closes #368 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd73074f Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd73074f Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd73074f Branch: refs/heads/master Commit: fd73074f2198f92516a3eb8f5e786d31c1c071a5 Parents: e656ee2 Author: jhkim <[email protected]> Authored: Mon Feb 2 14:56:14 2015 +0900 Committer: jhkim <[email protected]> Committed: Mon Feb 2 14:56:14 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../cli/tsql/DefaultTajoCliOutputFormatter.java | 3 +- .../java/org/apache/tajo/querymaster/Stage.java | 23 ++++- .../org/apache/tajo/TajoTestingCluster.java | 96 ++++++++------------ .../org/apache/tajo/cli/tsql/TestTajoCli.java | 42 +++------ .../tajo/engine/query/TestInsertQuery.java | 3 +- .../apache/tajo/querymaster/TestKillQuery.java | 20 +++- .../TestTajoCli/testNonForwardQueryPause.result | 3 +- 8 files changed, 98 insertions(+), 94 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 4a3715b..8466a38 100644 --- a/CHANGES +++ b/CHANGES @@ -327,6 +327,8 @@ Release 0.10.0 - unreleased TASKS + TAJO-1323: Cleanup the unstable test case. (jinho) + TAJO-1295: Remove legacy worker.dataserver package and its unit tests. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java index 17c94b9..5cbe77b 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java @@ -37,6 +37,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { private boolean printPause; private boolean printErrorTrace; private String nullChar; + public static char QUIT_COMMAND = 'q'; @Override public void init(TajoCli.TajoCliContext context) { @@ -123,7 +124,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { } sout.flush(); if (sin != null) { - if (sin.read() == 'q') { + if (sin.read() == QUIT_COMMAND) { endOfTuple = false; sout.println(); break; http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index 208d4a6..5673d5b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -185,7 +185,7 @@ public class Stage implements EventHandler<StageEvent> { // Transitions from KILL_WAIT state .addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT, - StageEventType.SQ_CONTAINER_ALLOCATED, + EnumSet.of(StageEventType.SQ_START, StageEventType.SQ_CONTAINER_ALLOCATED), CONTAINERS_CANCEL_TRANSITION) .addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT, EnumSet.of(StageEventType.SQ_KILL), new KillTasksTransition()) @@ -414,6 +414,18 @@ public class Stage implements EventHandler<StageEvent> { return totalScheduledObjectsCount; } + public int getKilledObjectCount() { + return killedObjectCount; + } + + public int getFailedObjectCount() { + return failedObjectCount; + } + + public int getCompletedTaskCount() { + return completedTaskCount; + } + public ExecutionBlock getBlock() { return block; } @@ -793,7 +805,14 @@ public class Stage implements EventHandler<StageEvent> { stage.taskScheduler.start(); allocateContainers(stage); } else { - stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_KILL)); + /* all tasks are killed before stage are inited */ + if (stage.getTotalScheduledObjectsCount() == stage.getCompletedTaskCount()) { + stage.eventHandler.handle( + new StageEvent(stage.getId(), StageEventType.SQ_STAGE_COMPLETED)); + } else { + stage.eventHandler.handle( + new StageEvent(stage.getId(), StageEventType.SQ_KILL)); + } } } } catch (Throwable e) { http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java index 8714fc4..0d3d660 100644 --- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -77,17 +77,13 @@ public class TajoTestingCluster { private TajoMaster tajoMaster; private List<TajoWorker> tajoWorkers = new ArrayList<TajoWorker>(); private boolean standbyWorkerMode = false; + private boolean isDFSRunning = false; + private boolean isTajoClusterRunning = false; + private boolean isCatalogServerRunning = false; - // If non-null, then already a cluster running. private File clusterTestBuildDir = null; /** - * System property key to get test directory value. - * Name is as it is because mini dfs has hard-codings to put test data here. - */ - public static final String TEST_DIRECTORY_KEY = MiniDFSCluster.PROP_TEST_BUILD_DATA; - - /** * Default parent directory for test output. */ public static final String DEFAULT_TEST_DIRECTORY = "target/test-data"; @@ -111,6 +107,7 @@ public class TajoTestingCluster { this.conf = new TajoConf(); this.conf.setBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE, masterHaEMode); + initTestDir(); setTestingFlagProperties(); initPropertiesAndConfigs(); } @@ -177,22 +174,19 @@ public class TajoTestingCluster { return this.conf; } - public void initTestDir() { - if (System.getProperty(TEST_DIRECTORY_KEY) == null) { - clusterTestBuildDir = setupClusterTestBuildDir(); - System.setProperty(TEST_DIRECTORY_KEY, - clusterTestBuildDir.getAbsolutePath()); - } - } + public void initTestDir() { + if (clusterTestBuildDir == null) { + clusterTestBuildDir = setupClusterTestBuildDir(); + } + } /** * @return Where to write test data on local filesystem; usually * {@link #DEFAULT_TEST_DIRECTORY} * @see #setupClusterTestBuildDir() */ - public static File getTestDir() { - return new File(System.getProperty(TEST_DIRECTORY_KEY, - DEFAULT_TEST_DIRECTORY)); + public File getTestDir() { + return clusterTestBuildDir; } /** @@ -202,10 +196,10 @@ public class TajoTestingCluster { * @see #setupClusterTestBuildDir() */ public static File getTestDir(final String subdirName) { - return new File(getTestDir(), subdirName); + return new File(new File(DEFAULT_TEST_DIRECTORY), subdirName); } - public File setupClusterTestBuildDir() { + public static File setupClusterTestBuildDir() { String randomStr = UUID.randomUUID().toString(); String dirStr = getTestDir(randomStr).toString(); File dir = new File(dirStr).getAbsoluteFile(); @@ -243,9 +237,6 @@ public class TajoTestingCluster { File dir, final String hosts[]) throws IOException { - if (dir == null) { - dir = setupClusterTestBuildDir(); - } conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString()); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); @@ -263,7 +254,7 @@ public class TajoTestingCluster { this.defaultFS = this.dfsCluster.getFileSystem(); this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString()); this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo"); - + isDFSRunning = true; return this.dfsCluster; } @@ -300,22 +291,20 @@ public class TajoTestingCluster { // Catalog Section //////////////////////////////////////////////////////// public MiniCatalogServer startCatalogCluster() throws Exception { - TajoConf c = getConfiguration(); + if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running"); - if(clusterTestBuildDir == null) { - clusterTestBuildDir = setupClusterTestBuildDir(); - } + TajoConf c = getConfiguration(); conf.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore"); conf.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + clusterTestBuildDir.getAbsolutePath() + "/db"); - LOG.info("Apache Derby repository is set to "+conf.get(CatalogConstants.CATALOG_URI)); + LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI)); conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0"); catalogServer = new MiniCatalogServer(conf); CatalogServer catServer = catalogServer.getCatalogServer(); InetSocketAddress sockAddr = catServer.getBindAddress(); c.setVar(ConfVars.CATALOG_ADDRESS, NetUtils.normalizeInetSocketAddress(sockAddr)); - + isCatalogServerRunning = true; return this.catalogServer; } @@ -323,6 +312,7 @@ public class TajoTestingCluster { if (catalogServer != null) { this.catalogServer.shutdown(); } + isCatalogServerRunning = false; } public MiniCatalogServer getMiniCatalogCluster() { @@ -352,10 +342,10 @@ public class TajoTestingCluster { c.setVar(ConfVars.ROOT_DIR, getMiniDFSCluster().getFileSystem().getUri() + "/tajo"); } else { - c.setVar(ConfVars.ROOT_DIR, clusterTestBuildDir.getAbsolutePath() + "/tajo"); + c.setVar(ConfVars.ROOT_DIR, testBuildDir.getAbsolutePath() + "/tajo"); } - setupCatalogForTesting(c, clusterTestBuildDir); + setupCatalogForTesting(c, testBuildDir); tajoMaster = new TajoMaster(); tajoMaster.init(c); @@ -374,6 +364,7 @@ public class TajoTestingCluster { if(standbyWorkerMode) { startTajoWorkers(numSlaves); } + isTajoClusterRunning = true; LOG.info("Mini Tajo cluster is up"); LOG.info("===================================================================================="); LOG.info("= MiniTajoCluster starts up ="); @@ -473,8 +464,8 @@ public class TajoTestingCluster { /** * @throws java.io.IOException If a cluster -- dfs or engine -- already running. */ - void isRunningCluster(String passedBuildPath) throws IOException { - if (this.clusterTestBuildDir == null || passedBuildPath != null) return; + void isRunningCluster() throws IOException { + if (!isTajoClusterRunning && !isCatalogServerRunning && !isDFSRunning) return; throw new IOException("Cluster already running at " + this.clusterTestBuildDir); } @@ -501,19 +492,13 @@ public class TajoTestingCluster { LOG.info("Starting up minicluster with 1 master(s) and " + numSlaves + " worker(s) and " + numDataNodes + " datanode(s)"); - // If we already put up a cluster, fail. - String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null); - isRunningCluster(testBuildPath); - if (testBuildPath != null) { - LOG.info("Using passed path: " + testBuildPath); + // If we already bring up the cluster, fail. + isRunningCluster(); + if (clusterTestBuildDir != null) { + LOG.info("Using passed path: " + clusterTestBuildDir); } - // Make a new random dir to home everything in. Set it as system property. - // minidfs reads home from system property. - this.clusterTestBuildDir = testBuildPath == null? - setupClusterTestBuildDir() : new File(testBuildPath); - - startMiniDFSCluster(numDataNodes, setupClusterTestBuildDir(), dataNodeHosts); + startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts); this.dfsCluster.waitClusterUp(); hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir); @@ -559,20 +544,11 @@ public class TajoTestingCluster { } public void startMiniClusterInLocal(final int numSlaves) throws Exception { - // If we already put up a cluster, fail. - String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null); - isRunningCluster(testBuildPath); - if (testBuildPath != null) { - LOG.info("Using passed path: " + testBuildPath); - } - - // Make a new random dir to home everything in. Set it as system property. - // minidfs reads home from system property. - this.clusterTestBuildDir = testBuildPath == null? - setupClusterTestBuildDir() : new File(testBuildPath); + isRunningCluster(); - System.setProperty(TEST_DIRECTORY_KEY, - this.clusterTestBuildDir.getAbsolutePath()); + if (clusterTestBuildDir != null) { + LOG.info("Using passed path: " + clusterTestBuildDir); + } startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, true); } @@ -592,6 +568,7 @@ public class TajoTestingCluster { if(this.catalogServer != null) { shutdownCatalogCluster(); + isCatalogServerRunning = false; } if(this.yarnCluster != null) { @@ -612,6 +589,7 @@ public class TajoTestingCluster { } catch (IOException e) { System.err.println("error closing file system: " + e); } + isDFSRunning = false; } if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) { @@ -630,6 +608,7 @@ public class TajoTestingCluster { } LOG.info("Minicluster is down"); + isTajoClusterRunning = false; } public static TajoClient newTajoClient() throws Exception { @@ -803,7 +782,8 @@ public class TajoTestingCluster { } catch (InterruptedException e) { } if (++i > 200) { - throw new IOException("Timed out waiting"); + throw new IOException("Timed out waiting. expected: " + expected + + ", actual: " + query != null ? String.valueOf(query.getSynchronizedState()) : String.valueOf(query)); } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index aff1677..e014b52 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -38,9 +38,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintWriter; +import java.io.*; import java.net.URL; import static org.junit.Assert.*; @@ -77,7 +75,8 @@ public class TestTajoCli { } @After - public void tearDown() { + public void tearDown() throws IOException { + out.close(); if (tajoCli != null) { tajoCli.close(); } @@ -350,38 +349,27 @@ public class TestTajoCli { assertOutputResult(new String(out.toByteArray())); } - @Test + @Test(timeout = 3000) public void testNonForwardQueryPause() throws Exception { final String sql = "select * from default.lineitem"; + TajoCli cli = null; try { TableDesc tableDesc = cluster.getMaster().getCatalog().getTableDesc("default", "lineitem"); assertNotNull(tableDesc); assertEquals(0L, tableDesc.getStats().getNumRows().longValue()); - setVar(tajoCli, SessionVars.CLI_PAGE_ROWS, "2"); - setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); - Thread t = new Thread() { - public void run() { - try { - tajoCli.executeScript(sql); - } catch (Exception e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - }; - t.start(); + + InputStream testInput = new ByteArrayInputStream(new byte[]{(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND}); + cli = new TajoCli(cluster.getConfiguration(), new String[]{}, testInput, out); + setVar(cli, SessionVars.CLI_PAGE_ROWS, "2"); + setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName()); + + cli.executeScript(sql); + String consoleResult; - while (true) { - Thread.sleep(3 * 1000); - consoleResult = new String(out.toByteArray()); - if (consoleResult.indexOf("row") >= 0) { - t.interrupt(); - break; - } - } + consoleResult = new String(out.toByteArray()); assertOutputResult(consoleResult); } finally { - setVar(tajoCli, SessionVars.CLI_PAGE_ROWS, "100"); + cli.close(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index cc7dced..0799d22 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -660,12 +660,11 @@ public class TestInsertQuery extends QueryTestCaseBase { @Test public final void testInsertOverwriteTableWithNonFromQuery2() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithEvalQuery"); + String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithEvalQuery2"); ResultSet res = executeString("create table " + tableName +" (col1 int4, col2 float4, col3 text)"); res.close(); CatalogService catalog = testingCluster.getMaster().getCatalog(); assertTrue(catalog.existsTable(getCurrentDatabase(), tableName)); - res = executeString("insert overwrite into " + tableName + " (col1, col3) select 1::INT4, 'test';"); res.close(); http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java index 42ad8da..0574bea 100644 --- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java @@ -127,12 +127,24 @@ public class TestKillQuery { Query q = queryMasterTask.getQuery(); q.handle(new QueryEvent(queryId, QueryEventType.KILL)); - try{ + try { cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 50); - } finally { assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); + } catch (Exception e) { + e.printStackTrace(); + if (stage != null) { + System.err.println(String.format("Stage: [%s] (Total: %d, Complete: %d, Success: %d, Killed: %d, Failed: %d)", + stage.getId().toString(), + stage.getTotalScheduledObjectsCount(), + stage.getCompletedTaskCount(), + stage.getSucceededObjectCount(), + stage.getKilledObjectCount(), + stage.getFailedObjectCount())); + } + throw e; + } finally { + queryMasterTask.stop(); } - queryMasterTask.stop(); } @Test @@ -145,6 +157,8 @@ public class TestKillQuery { QueryMasterTask qmt = cluster.getQueryMasterTask(queryId); Query query = qmt.getQuery(); + // wait for a stage created + cluster.waitForQueryState(query, TajoProtos.QueryState.QUERY_RUNNING, 10); query.handle(new QueryEvent(queryId, QueryEventType.KILL)); try{ http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result b/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result index e9485d0..d4ba604 100644 --- a/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result +++ b/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result @@ -2,4 +2,5 @@ l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice ------------------------------- 1, 1, 7706, 1, 17.0, 21168.23, 0.04, 0.02, N, O, 1996-03-13, 1996-02-12, 1996-03-22, DELIVER IN PERSON, TRUCK, egular courts above the 1, 1, 7311, 2, 36.0, 45983.16, 0.09, 0.06, N, O, 1996-04-12, 1996-02-28, 1996-04-20, TAKE BACK RETURN, MAIL, ly final dependencies: slyly bold -(2 rows, continue... 'q' is quit) \ No newline at end of file +(2 rows, continue... 'q' is quit) +(unknown row number, , 604 B selected) \ No newline at end of file
