[
https://issues.apache.org/jira/browse/TAJO-178?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13766218#comment-13766218
]
Hudson commented on TAJO-178:
-----------------------------
SUCCESS: Integrated in Tajo-trunk-postcommit #430 (See
[https://builds.apache.org/job/Tajo-trunk-postcommit/430/])
TAJO-178: Implements StorageManager for scanning asynchronously. (hyoungjunkim
via hyunsik) (hyunsik:
https://git-wip-us.apache.org/repos/asf?p=incubator-tajo.git&a=commit&h=5ad7fbae98e45307cde51bc58d54e4de95b627ad)
*
tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskMountInfo.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/Fragment.java
* tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
* tajo-core/tajo-core-backend/src/main/resources/webapps/admin/index.jsp
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/BackendTestingUtil.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskDeviceInfo.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/plan/global/TestGlobalQueryPlanner.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java
*
tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
* tajo-core/tajo-core-backend/src/main/resources/webapps/worker/index.jsp
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
*
tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorageManager.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalPlanner.java
* tajo-core/tajo-core-backend/src/main/resources/webapps/admin/query.jsp
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java
* tajo-core/tajo-core-backend/pom.xml
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
* tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/cli/TajoCli.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskUtil.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java
*
tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/FileScanRunner.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/IndexedStoreExec.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TaskRunnerManager.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Query.java
*
tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TaskSchedulerImpl.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
* tajo-core/tajo-core-backend/src/main/resources/webapps/admin/WEB-INF/web.xml
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/webapp/HttpServer.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/DiskInfo.java
* tajo-core/tajo-core-storage/src/main/resources/storage-default.xml
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMaster.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestResultSetImpl.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
* tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/Task.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoWorker.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/GlobalEngine.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
*
tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
* tajo-core/tajo-core-backend/src/main/resources/webapps/worker/querydetail.jsp
* tajo-core/tajo-core-backend/src/main/resources/webapps/worker/style.css
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/PartitionedStoreExec.java
* tajo-core/tajo-core-backend/src/main/resources/webapps/worker/WEB-INF/web.xml
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/global/TestGlobalQueryOptimizer.java
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/client/TestTajoClient.java
* tajo-core/tajo-core-backend/src/main/proto/TajoMasterProtocol.proto
*
tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java
*
tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java
* tajo-core/tajo-core-storage/src/test/resources/storage-default.xml
*
tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/WorkerResource.java
* pom.xml
> Implements StorageManager for scanning asynchronously
> -----------------------------------------------------
>
> Key: TAJO-178
> URL: https://issues.apache.org/jira/browse/TAJO-178
> Project: Tajo
> Issue Type: Improvement
> Components: storage
> Affects Versions: 0.2-incubating
> Reporter: hyoungjunkim
> Assignee: hyoungjunkim
> Fix For: 0.2-incubating
>
> Attachments: TAJO-178_1.path, TAJO-178_3.patch, TAJO-178_4.patch,
> TAJO-178.patch_2, TAJO-178.path, tajo_storage_manager.png
>
>
> The current StorageManager does not provide scan scheduling function. All
> scan operations run concurrently. This is the cause of random disk access and
> disk read performance is not good.
> The proposed StorageManager is based on double buffering. Each disk has a
> scheduler to schedule by order of scanned adjust. Each Scanner has a
> InputStream and a Tuple pool. The next() operation of ScanNode is blocked
> until Tuple pool is filled. Assigned Scanner by the scheduler read data(xMB)
> and fills Tuple Pool and notifies to next() operation. After scanning Scanner
> re-enter DiskScanQueue.
> In this way Scanner can pass column vector to Vectorized Query Engine.
> See the attached file.
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
