[
https://issues.apache.org/jira/browse/PHOENIX-1734?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15298493#comment-15298493
]
Hudson commented on PHOENIX-1734:
---------------------------------
FAILURE: Integrated in Phoenix-master #1234 (See
[https://builds.apache.org/job/Phoenix-master/1234/])
PHOENIX-1734 Local index improvements(Rajeshbabu) (rajeshbabu: rev
10909ae502095bac775d98e6d92288c5cad9b9a6)
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/TxWriteFailureIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryIT.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/SortMergeJoinIT.java
* phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
*
phoenix-core/src/main/java/org/apache/phoenix/mapreduce/AbstractBulkLoadTool.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
*
phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexSplitter.java
* phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryIT.java
* phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
* phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java
*
phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleWriterIndexCommitter.java
*
phoenix-core/src/it/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancerIT.java
*
phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexSplitTransaction.java
* phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
*
phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/RollbackIT.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificViewIndexIT.java
* phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
* phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/balancer/IndexLoadBalancer.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinLocalIndexIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/master/IndexMasterObserver.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
* phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
* phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
*
phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestParalleIndexWriter.java
*
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
* phoenix-core/src/main/java/org/apache/phoenix/iterate/ExplainTable.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriter.java
* phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
* phoenix-core/src/main/java/org/apache/phoenix/compile/ProjectionCompiler.java
*
phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
*
phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
*
phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexMerger.java
*
phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/index/txn/MutableRollbackIT.java
* phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ScanRegionObserver.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/recovery/TrackingParallelWriterIndexCommitter.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
* phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
* phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
*
phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
* phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexCommitter.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/IndexRegionSplitPolicy.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ReadOnlyIndexFailureIT.java
*
phoenix-core/src/it/java/org/apache/phoenix/end2end/SubqueryUsingSortMergeJoinIT.java
*
phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/ParallelWriterIndexCommitter.java
* phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexCodec.java
* phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
> Local index improvements
> ------------------------
>
> Key: PHOENIX-1734
> URL: https://issues.apache.org/jira/browse/PHOENIX-1734
> Project: Phoenix
> Issue Type: Improvement
> Reporter: Rajeshbabu Chintaguntla
> Assignee: Rajeshbabu Chintaguntla
> Fix For: 4.8.0
>
> Attachments: PHOENI-1734-WIP.patch, PHOENIX-1734_v1.patch,
> PHOENIX-1734_v4.patch, PHOENIX-1734_v5.patch, TestAtomicLocalIndex.java
>
>
> Local index design considerations:
> 1. Colocation: We need to co-locate regions of local index regions and data
> regions. The co-location can be a hard guarantee or a soft (best approach)
> guarantee. The co-location is a performance requirement, and also maybe
> needed for consistency(2). Hard co-location means that either both the data
> region and index region are opened atomically, or neither of them open for
> serving.
> 2. Index consistency : Ideally we want the index region and data region to
> have atomic updates. This means that they should either (a)use transactions,
> or they should (b)share the same WALEdit and also MVCC for visibility. (b) is
> only applicable if there is hard colocation guarantee.
> 3. Local index clients : How the local index will be accessed from clients.
> In case of the local index being managed in a table, the HBase client can be
> used for doing scans, etc. If the local index is hidden inside the data
> regions, there has to be a different mechanism to access the data through the
> data region.
> With the above considerations, we imagine three possible implementation for
> the local index solution, each detailed below.
> APPROACH 1: Current approach
> (1) Current approach uses balancer as a soft guarantee. Because of this, in
> some rare cases, colocation might not happen.
> (2) The index and data regions do not share the same WALEdits. Meaning
> consistency cannot be achieved. Also there are two WAL writes per write from
> client.
> (3) Regular Hbase client can be used to access index data since index is just
> another table.
> APPROACH 2: Shadow regions + shared WAL & MVCC
> (1) Introduce a shadow regions concept in HBase. Shadow regions are not
> assigned by AM. Phoenix implements atomic open (and split/merge) of region
> opening for data regions and index regions so that hard co-location is
> guaranteed.
> (2) For consistency requirements, the index regions and data regions will
> share the same WALEdit (and thus recovery) and they will also share the same
> MVCC mechanics so that index update and data update is visible atomically.
> (3) Regular Hbase client can be used to access index data since index is just
> another table.
> APPROACH 3: Storing index data in separate column families in the table.
> (1) Regions will have store files for cfs, which is sorted using the primary
> sort order. Regions may also maintain stores, sorted in secondary sort
> orders. This approach is similar in vein how a RDBMS keeps data (a B-TREE in
> primary sort order and multiple B-TREEs in secondary sort orders with
> pointers to primary key). That means store the index data in separate column
> families in the data region. This way a region is extended to be more similar
> to a RDBMS (but LSM instead of BTree). This is sometimes called shadow cf’s
> as well. This approach guarantees hard co-location.
> (2) Since everything is in a single region, they automatically share the
> same WALEdit and MVCC numbers. Atomicity is easily achieved.
> (3) Current Phoenix implementation need to change in such a way that column
> families selection in read/write path is based data table/index table(logical
> table in phoenix).
> I think that APPROACH 3 is the best one for long term, since it does not
> require to change anything in HBase, mainly we don't need to muck around with
> the split/merge stuff in HBase. It will be win-win.
> However, APPROACH 2 still needs a “shadow regions” concept to be implemented
> in HBase itself, and also a way to share WALEdits and MVCCs from multiple
> regions.
> APPROACH 1 is a good start for local indexes, but I think we are not getting
> the full benefits for the feature. We can support this for the short term,
> and decide on the next steps for a longer term implementation.
> we won't be able to get to implementing it immediately, and want to start a
> brainstorm.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
