Repository: tajo Updated Branches: refs/heads/master 3ca900d7a -> f4abd4e99
TAJO-1954: Fix memory leak in physical operator. Closes #838 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f4abd4e9 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f4abd4e9 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f4abd4e9 Branch: refs/heads/master Commit: f4abd4e993f149b8d796da43d2dfdb47d1701dba Parents: 3ca900d Author: Jinho Kim <[email protected]> Authored: Mon Nov 2 18:34:53 2015 +0900 Committer: Jinho Kim <[email protected]> Committed: Mon Nov 2 18:34:53 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 ++ .../java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java | 7 ++++--- .../physical/DistinctGroupbySortAggregationExec.java | 11 +++++++---- .../planner/physical/SortBasedColPartitionStoreExec.java | 2 ++ .../master/exec/NonForwardQueryResultFileScanner.java | 1 + .../src/main/java/org/apache/tajo/storage/RawFile.java | 7 +++++-- 6 files changed, 21 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/f4abd4e9/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 53658b1..d4f9180 100644 --- a/CHANGES +++ b/CHANGES @@ -24,6 +24,8 @@ Release 0.12.0 - unreleased BUG FIXES + TAJO-1954: Fix memory leak in physical operator. (jinho) + TAJO-1643: SQLParse can't parse parentheses table clause. (hyunsik) TAJO-1924: Repair partition need to calculate partition volume. (jaehwa) http://git-wip-us.apache.org/repos/asf/tajo/blob/f4abd4e9/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java index f0f01bf..279fce7 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ha/TestHAServiceHDFSImpl.java @@ -31,6 +31,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.service.ServiceTracker; import org.apache.tajo.service.ServiceTrackerFactory; +import org.junit.Assert; import org.junit.Test; import static junit.framework.Assert.assertTrue; @@ -150,9 +151,9 @@ public class TestHAServiceHDFSImpl { TajoClient client = null; try { client = new TajoClientImpl(tracker); - client.existDatabase("default"); - client.existTable("default.ha_test1"); - client.existTable("default.ha_test2"); + Assert.assertTrue(client.existDatabase("default")); + Assert.assertTrue(client.existTable("default.ha_test1")); + Assert.assertTrue(client.existTable("default.ha_test2")); } finally { IOUtils.cleanup(null, client); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f4abd4e9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java index 58cfca4..cb7478a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java @@ -68,10 +68,6 @@ public class DistinctGroupbySortAggregationExec extends PhysicalExec { for(int i = 0; i < resultColumnIds.length; i++) { resultColumnIdIndexes[resultColumnIds[i]] = i; } - - for (SortAggregateExec eachExec: aggregateExecs) { - eachExec.init(); - } } boolean first = true; @@ -172,6 +168,13 @@ public class DistinctGroupbySortAggregationExec extends PhysicalExec { @Override public void init() throws IOException { + super.init(); + + if (aggregateExecs != null) { + for (SortAggregateExec eachExec: aggregateExecs) { + eachExec.init(); + } + } } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/f4abd4e9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java index 607dff7..176b6fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java @@ -102,6 +102,8 @@ public class SortBasedColPartitionStoreExec extends ColPartitionStoreExec { StatisticsUtil.aggregateTableStat(aggregatedStats, appender.getStats()); context.setResultStats(aggregatedStats); } + + super.close(); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/f4abd4e9/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java index ce69270..1706318 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultFileScanner.java @@ -252,6 +252,7 @@ public class NonForwardQueryResultFileScanner implements NonForwardQueryResultSc } return resultSetBuilder.build(); } catch (Throwable t) { + close(); throw new TajoInternalError(t.getCause()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/f4abd4e9/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java index fba12dd..a7b33fa 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java @@ -78,6 +78,7 @@ public class RawFile { super(conf, schema, meta, fragment); } + @Override public void init() throws IOException { File file; try { @@ -99,8 +100,10 @@ public class RawFile { + ", fragment length :" + fragment.getLength()); } - buf = BufferPool.directBuffer(conf.getInt(READ_BUFFER_SIZE, DEFAULT_BUFFER_SIZE)); - buffer = buf.nioBuffer(0, buf.capacity()); + if(buf == null) { + buf = BufferPool.directBuffer(conf.getInt(READ_BUFFER_SIZE, DEFAULT_BUFFER_SIZE)); + buffer = buf.nioBuffer(0, buf.capacity()); + } columnTypes = new DataType[schema.size()]; for (int i = 0; i < schema.size(); i++) {
