Repository: tajo Updated Branches: refs/heads/branch-0.11.1 ebabe6e2d -> 8ead4bf7e
TAJO-1954: Fix memory leak in physical operator. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/8ead4bf7 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/8ead4bf7 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/8ead4bf7 Branch: refs/heads/branch-0.11.1 Commit: 8ead4bf7eaf87dcc634b251c2cdb8fe4480e29aa Parents: ebabe6e Author: Jinho Kim <[email protected]> Authored: Mon Nov 2 18:44:26 2015 +0900 Committer: Jinho Kim <[email protected]> Committed: Mon Nov 2 18:44:26 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 ++ .../java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 7 ++++--- .../physical/DistinctGroupbySortAggregationExec.java | 11 +++++++---- .../planner/physical/SortBasedColPartitionStoreExec.java | 2 ++ .../master/exec/NonForwardQueryResultFileScanner.java | 1 + .../src/main/java/org/apache/tajo/storage/RawFile.java | 7 +++++-- 6 files changed, 21 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/8ead4bf7/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 5822dfe..5f3640a 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,8 @@ Release 0.11.1 - unreleased BUG FIXES + TAJO-1954: Fix memory leak in physical operator. (jinho) + TAJO-1643: SQLParse can't parse parentheses table clause. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/8ead4bf7/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index f0f01bf..279fce7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -31,6 +31,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.junit.Assert; import org.junit.Test; import static junit.framework.Assert.assertTrue; @@ -150,9 +151,9 @@ public class TestHAServiceHDFSImpl { TajoClient client = null; try { client = new TajoClientImpl(tracker); - client.existDatabase("default"); - client.existTable("default.ha_test1"); - client.existTable("default.ha_test2"); + Assert.assertTrue(client.existDatabase("default")); + Assert.assertTrue(client.existTable("default.ha_test1")); + Assert.assertTrue(client.existTable("default.ha_test2")); } finally { IOUtils.cleanup(null, client); } http://git-wip-us.apache.org/repos/asf/tajo/blob/8ead4bf7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java index 58cfca4..cb7478a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java @@ -68,10 +68,6 @@ public class DistinctGroupbySortAggregationExec extends PhysicalExec { for(int i = 0; i < resultColumnIds.length; i++) { resultColumnIdIndexes[resultColumnIds[i]] = i; } - - for (SortAggregateExec eachExec: aggregateExecs) { - eachExec.init(); - } } boolean first = true; @@ -172,6 +168,13 @@ public class DistinctGroupbySortAggregationExec extends PhysicalExec { @Override public void init() throws IOException { + super.init(); + + if (aggregateExecs != null) { + for (SortAggregateExec eachExec: aggregateExecs) { + eachExec.init(); + } + } } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/8ead4bf7/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java index 607dff7..176b6fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java @@ -102,6 +102,8 @@ public class SortBasedColPartitionStoreExec extends ColPartitionStoreExec { StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); context.setResultStats(aggregatedStats); } + + super.close(); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/8ead4bf7/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index ce69270..1706318 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -252,6 +252,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc } return resultSetBuilder.build(); } catch (Throwable t) { + close(); throw new TajoInternalError(t.getCause()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/8ead4bf7/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java index 97e293c..6f1e687 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java @@ -79,6 +79,7 @@ public class RawFile { super(conf, schema, meta, fragment); } + @Override public void init() throws IOException { File file; try { @@ -100,8 +101,10 @@ public class RawFile { + ", fragment length :" + fragment.getLength()); } - buf = BufferPool.directBuffer(conf.getInt(READ_BUFFER_SIZE, DEFAULT_BUFFER_SIZE)); - buffer = buf.nioBuffer(0, buf.capacity()); + if(buf == null) { + buf = BufferPool.directBuffer(conf.getInt(READ_BUFFER_SIZE, DEFAULT_BUFFER_SIZE)); + buffer = buf.nioBuffer(0, buf.capacity()); + } columnTypes = new DataType[schema.size()]; for (int i = 0; i < schema.size(); i++) {
