Repository: tajo Updated Branches: refs/heads/branch-0.11.0 d214d50c6 -> 458345f6e
TAJO-993: Cleanup the result data in HDFS after query finished. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/458345f6 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/458345f6 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/458345f6 Branch: refs/heads/branch-0.11.0 Commit: 458345f6eee3dff29e1370e81c8abd409de34039 Parents: d214d50 Author: Jinho Kim <[email protected]> Authored: Thu Sep 3 10:14:26 2015 +0900 Committer: Jinho Kim <[email protected]> Committed: Thu Sep 3 10:14:26 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../java/org/apache/tajo/conf/TajoConf.java | 14 ++++ .../org/apache/tajo/master/TestQueryResult.java | 74 ++++++++++++++++++++ .../testTableResultOnClose.result | 3 + .../testTemporalResultOnClose.result | 7 ++ .../exec/NonForwardQueryResultFileScanner.java | 9 +++ 6 files changed, 110 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/458345f6/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 0c5b433..2a586e5 100644 --- a/CHANGES +++ b/CHANGES @@ -32,6 +32,9 @@ Release 0.11.0 - unreleased IMPROVEMENT + TAJO-993: Cleanup the result data in HDFS after query finished. + (jinho) + TAJO-1766: Improve the performance of cross join. (jihoon) TAJO-1792: tajo-cluster-tests is not available when it is used as an http://git-wip-us.apache.org/repos/asf/tajo/blob/458345f6/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 75826e6..b50ce81 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; +import org.apache.tajo.QueryId; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.service.BaseServiceTracker; @@ -773,6 +774,19 @@ public class TajoConf extends Configuration { return new Path(stagingDirString); } + /** + * It returns the temporal query directory + * An example dir is <pre>/{staging-dir}/{queryId}/RESULT</pre>. + * + * @param conf TajoConf + * @param queryId queryId + * @throws IOException + */ + public static Path getTemporalResultDir(TajoConf conf, QueryId queryId) throws IOException { + Path queryDir = new Path(getDefaultRootStagingDir(conf), queryId.toString()); + return new Path(queryDir, TajoConstants.RESULT_DIR_NAME); + } + public static Path getQueryHistoryDir(TajoConf conf) throws IOException { String historyDirString = conf.getVar(ConfVars.HISTORY_QUERY_DIR); if (!hasScheme(historyDirString)) { http://git-wip-us.apache.org/repos/asf/tajo/blob/458345f6/tajo-core-tests/src/test/java/org/apache/tajo/master/TestQueryResult.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestQueryResult.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestQueryResult.java new file mode 100644 index 0000000..6775c84 --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestQueryResult.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.master; + +import org.apache.tajo.QueryId; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.jdbc.FetchResultSet; +import org.apache.tajo.jdbc.TajoMemoryResultSet; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.sql.ResultSet; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestQueryResult extends QueryTestCaseBase { + + public TestQueryResult() { + super(TajoConstants.DEFAULT_DATABASE_NAME); + } + + @Test + public final void testTemporalResultOnClose() throws Exception { + + ResultSet res = executeString("select l_orderkey, l_partkey from lineitem where 1=1;"); + QueryId queryId = getQueryId(res); + Path resultPath = TajoConf.getTemporalResultDir(testingCluster.getConfiguration(), queryId); + assertTrue(testingCluster.getDefaultFileSystem().exists(resultPath)); + + assertResultSet(res); + cleanupQuery(res); + assertFalse(testingCluster.getDefaultFileSystem().exists(resultPath)); + } + + @Test + public final void testTableResultOnClose() throws Exception { + + ResultSet res = executeString("select * from lineitem limit 1"); + QueryId queryId = getQueryId(res); + Path resultPath = TajoConf.getTemporalResultDir(testingCluster.getConfiguration(), queryId); + assertFalse(testingCluster.getDefaultFileSystem().exists(resultPath)); + assertResultSet(res); + cleanupQuery(res); + } + + private QueryId getQueryId(ResultSet resultSet) { + if (resultSet instanceof TajoMemoryResultSet) { + return ((TajoMemoryResultSet) resultSet).getQueryId(); + } else if (resultSet instanceof FetchResultSet) { + return ((FetchResultSet) resultSet).getQueryId(); + } else { + throw new IllegalArgumentException(resultSet.toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/458345f6/tajo-core-tests/src/test/resources/results/TestQueryResult/testTableResultOnClose.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestQueryResult/testTableResultOnClose.result b/tajo-core-tests/src/test/resources/results/TestQueryResult/testTableResultOnClose.result new file mode 100644 index 0000000..dd9349f --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestQueryResult/testTableResultOnClose.result @@ -0,0 +1,3 @@ +l_orderkey,l_partkey,l_suppkey,l_linenumber,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment +------------------------------- +1,1,7706,1,17.0,21168.23,0.04,0.02,N,O,1996-03-13,1996-02-12,1996-03-22,DELIVER IN PERSON,TRUCK,egular courts above the \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/458345f6/tajo-core-tests/src/test/resources/results/TestQueryResult/testTemporalResultOnClose.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestQueryResult/testTemporalResultOnClose.result b/tajo-core-tests/src/test/resources/results/TestQueryResult/testTemporalResultOnClose.result new file mode 100644 index 0000000..1378536 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestQueryResult/testTemporalResultOnClose.result @@ -0,0 +1,7 @@ +l_orderkey,l_partkey +------------------------------- +1,1 +1,1 +2,2 +3,2 +3,3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/458345f6/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index 877e32b..9e132b0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -40,6 +40,7 @@ import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TaskAttemptContext; +import org.apache.hadoop.fs.Path; import java.io.IOException; import java.util.ArrayList; @@ -117,6 +118,14 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc scanExec.close(); scanExec = null; } + + //remove temporal final output + if (!tajoConf.getBoolVar(TajoConf.ConfVars.$DEBUG_ENABLED)) { + Path temporalResultDir = TajoConf.getTemporalResultDir(tajoConf, queryId); + if (tableDesc.getUri().equals(temporalResultDir.toUri())) { + temporalResultDir.getParent().getFileSystem(tajoConf).delete(temporalResultDir, true); + } + } } public List<ByteString> getNextRows(int fetchRowNum) throws IOException {
