[
https://issues.apache.org/jira/browse/TAJO-1343?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14636616#comment-14636616
]
Hudson commented on TAJO-1343:
------------------------------
ABORTED: Integrated in Tajo-master-CODEGEN-build #402 (See
[https://builds.apache.org/job/Tajo-master-CODEGEN-build/402/])
TAJO-1343: Improve the memory usage of physical executors. (jihoon) (jihoonson:
rev 4820610f4a2a384372aaecf5212bd486c79a65b2)
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RawFile.java
* tajo-core/src/main/java/org/apache/tajo/engine/planner/Projector.java
* tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/TupleMap.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ComparableVector.java
* tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/sequencefile/SequenceFileScanner.java
*
tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/FrameTuple.java
*
tajo-core/src/test/resources/results/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.result
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLJoinExec.java
* tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
* tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/WindowAggExec.java
* CHANGES
*
tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithJson.result
* tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/TupleList.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/AggregationExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java
* tajo-core/src/main/java/org/apache/tajo/engine/planner/KeyProjector.java
* tajo-core/src/test/resources/results/TestGroupByQuery/testPythonUdaf3.result
* tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/KeyTuple.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java
*
tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseScanner.java
*
tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithPythonFunc.result
* tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/index/bst/BSTIndex.java
* tajo-jdbc/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
*
tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithLimit3.sql
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
* tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
* tajo-core/src/main/java/org/apache/tajo/engine/utils/CacheHolder.java
*
tajo-core/src/test/resources/queries/TestSelectNestedRecord/testNestedFieldAsGroupbyKey1.sql
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/rcfile/RCFile.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/RowFile.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
*
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
* tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/avro/AvroScanner.java
*
tajo-core/src/test/resources/results/TestWindowQuery/testWindowWithAggregation6.result
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java
*
tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/HeapTuple.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java
*
tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
*
tajo-core/src/test/resources/results/TestWindowQuery/testWindowWithAggregation4.result
*
tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java
*
tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
* tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BNLJoinExec.java
* tajo-client/src/main/java/org/apache/tajo/storage/RowStoreUtil.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
*
tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/CSVFile.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonJoinExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java
*
tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithPythonFunc2.result
* tajo-common/src/main/java/org/apache/tajo/storage/NullTuple.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftSemiJoinExec.java
* tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlanner.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftAntiJoinExec.java
*
tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/tuple/offheap/UnSafeTuple.java
* tajo-plan/src/main/java/org/apache/tajo/plan/TablePropertyUtil.java
* tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
*
tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/LazyTuple.java
*
tajo-core/src/test/resources/results/TestGroupByQuery/testGroupbyWithLimit3.result
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java
* tajo-core/src/test/resources/queries/TestGroupByQuery/testGroupbyWithJson.json
* tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/TupleSet.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/TupleSorter.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/VectorizedSorter.java
*
tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestTupleSorter.java
*
tajo-core/src/test/resources/results/TestGroupByQuery/testHavingWithAggFunction.result
* tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java
*
tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java
* tajo-core/src/test/java/org/apache/tajo/engine/query/TestGroupByQuery.java
> Improve the memory usage of physical executors
> ----------------------------------------------
>
> Key: TAJO-1343
> URL: https://issues.apache.org/jira/browse/TAJO-1343
> Project: Tajo
> Issue Type: Improvement
> Components: Physical Operator
> Reporter: Jihoon Son
> Assignee: Jihoon Son
> Priority: Critical
> Fix For: 0.11.0
>
> Attachments: 1343-memory.png, master-memory.png
>
>
> *Introduction*
> Basically, the tuple instance is maintained as a singleton in physical
> operators. However, there are some memory-based operator types which need to
> keep multiple tuples in the memory. In these operators, multiple instances
> must be created for each tuple.
> *Problem*
> Currently, there are some temporal routines to avoid unexpected problems due
> to the singleton instance of tuple. However, the methodology is inconsistent
> and complex, which causes unexpected bugs.
> *Solution*
> A consistent methodology is needed to handle this problem. Only the operators
> that keep multiple tuples in memory must maintain those tuples with separate
> instances.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
