[
https://issues.apache.org/jira/browse/PHOENIX-6181?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17214225#comment-17214225
]
ASF GitHub Bot commented on PHOENIX-6181:
-----------------------------------------
kadirozde commented on a change in pull request #915:
URL: https://github.com/apache/phoenix/pull/915#discussion_r504922136
##########
File path:
phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GlobalIndexRegionScanner.java
##########
@@ -240,4 +320,1033 @@ protected boolean isColumnIncluded(Cell cell) {
byte[] qualifier = CellUtil.cloneQualifier(cell);
return set.contains(qualifier);
}
+ @VisibleForTesting
+ public boolean shouldVerify(IndexTool.IndexVerifyType verifyType,
+ byte[] indexRowKey, Scan scan, Region region,
IndexMaintainer indexMaintainer,
+ IndexVerificationResultRepository
verificationResultRepository, boolean shouldVerifyCheckDone) throws IOException
{
+ this.verifyType = verifyType;
+ this.indexRowKeyforReadRepair = indexRowKey;
+ this.scan = scan;
+ this.region = region;
+ this.indexMaintainer = indexMaintainer;
+ this.verificationResultRepository = verificationResultRepository;
+ this.shouldVerifyCheckDone = shouldVerifyCheckDone;
+ return shouldVerify();
+ }
+
+ protected boolean shouldVerify() throws IOException {
+ // In case of read repair, proceed with rebuild
+ // All other types of rebuilds/verification should be incrementally
performed if appropriate param is passed
+ byte[] lastVerifyTimeValue =
scan.getAttribute(UngroupedAggregateRegionObserver.INDEX_RETRY_VERIFY);
+ Long lastVerifyTime = lastVerifyTimeValue == null ? 0 :
Bytes.toLong(lastVerifyTimeValue);
+ if(indexRowKeyforReadRepair != null || lastVerifyTime == 0 ||
shouldVerifyCheckDone) {
+ return true;
+ }
+
+ IndexToolVerificationResult verificationResultTemp =
verificationResultRepository
+ .getVerificationResult(lastVerifyTime, scan, region,
indexMaintainer.getIndexTableName()) ;
+ if(verificationResultTemp != null) {
+ verificationResult = verificationResultTemp;
+ }
+ shouldVerifyCheckDone = true;
+ return verificationResultTemp == null;
+ }
+
+ @Override
+ public HRegionInfo getRegionInfo() {
+ return region.getRegionInfo();
+ }
+
+ @Override
+ public boolean isFilterDone() {
+ return false;
+ }
+
+ private void closeTables() throws IOException {
+ hTableFactory.shutdown();
+ if (indexHTable != null) {
+ indexHTable.close();
+ }
+ if (dataHTable != null) {
+ dataHTable.close();
+ }
+ }
+ @Override
+ public void close() throws IOException {
+ innerScanner.close();
+ if (indexRowKeyforReadRepair != null) {
+ closeTables();
+ return;
+ }
+ if (verify) {
+ try {
+ if (verificationResultRepository != null) {
+
verificationResultRepository.logToIndexToolResultTable(verificationResult,
+ verifyType,
region.getRegionInfo().getRegionName(), skipped);
+ }
+ } finally {
+ this.pool.stop("IndexRegionObserverRegionScanner is closing");
+ closeTables();
+ if (verificationResultRepository != null) {
+ verificationResultRepository.close();
+ }
+ if (verificationOutputRepository != null) {
+ verificationOutputRepository.close();
+ }
+ }
+ }
+ else {
+ this.pool.stop("IndexRegionObserverRegionScanner is closing");
+ closeTables();
+ }
+ }
+
+ @VisibleForTesting
+ public int setIndexTableTTL(int ttl) {
+ indexTableTTL = ttl;
+ return 0;
+ }
+
+ @VisibleForTesting
+ public int setIndexMaintainer(IndexMaintainer indexMaintainer) {
+ this.indexMaintainer = indexMaintainer;
+ return 0;
+ }
+
+ @VisibleForTesting
+ public long setMaxLookBackInMills(long maxLookBackInMills) {
+ this.maxLookBackInMills = maxLookBackInMills;
+ return 0;
+ }
+
+ public void logToIndexToolOutputTable(byte[] dataRowKey, byte[]
indexRowKey, long dataRowTs, long indexRowTs,
+ String errorMsg, boolean
isBeforeRebuild,
+
IndexVerificationOutputRepository.IndexVerificationErrorType errorType) throws
IOException {
+ logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs,
indexRowTs, errorMsg, null,
+ null, isBeforeRebuild, errorType);
+ }
+
+ @VisibleForTesting
+ public void logToIndexToolOutputTable(byte[] dataRowKey, byte[]
indexRowKey, long dataRowTs, long indexRowTs,
+ String errorMsg, byte[]
expectedVaue, byte[] actualValue, boolean isBeforeRebuild,
+
IndexVerificationOutputRepository.IndexVerificationErrorType errorType) throws
IOException {
+ ungroupedAggregateRegionObserver.checkForRegionClosing();
+ verificationOutputRepository.logToIndexToolOutputTable(dataRowKey,
indexRowKey, dataRowTs, indexRowTs,
+ errorMsg, expectedVaue, actualValue,
scan.getTimeRange().getMax(),
+ region.getRegionInfo().getTable().getName(), isBeforeRebuild,
errorType);
+ }
+
+ private static Cell getCell(Mutation m, byte[] family, byte[] qualifier) {
+ List<Cell> cellList = m.getFamilyCellMap().get(family);
+ if (cellList == null) {
+ return null;
+ }
+ for (Cell cell : cellList) {
+ if (CellUtil.matchingQualifier(cell, qualifier)) {
+ return cell;
+ }
+ }
+ return null;
+ }
+
+ private void logMismatch(Mutation expected, Mutation actual, int
iteration, IndexToolVerificationResult.PhaseResult verificationPhaseResult,
boolean isBeforeRebuild) throws IOException {
+ if (getTimestamp(expected) != getTimestamp(actual)) {
+ String errorMsg = "Not matching timestamp";
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new
ImmutableBytesWritable(expected.getRow()), viewConstants);
+ logToIndexToolOutputTable(dataKey, expected.getRow(),
getTimestamp(expected), getTimestamp(actual),
+ errorMsg, null, null, isBeforeRebuild, INVALID_ROW);
+ return;
+ }
+ int expectedCellCount = 0;
+ for (List<Cell> cells : expected.getFamilyCellMap().values()) {
+ if (cells == null) {
+ continue;
+ }
+ for (Cell expectedCell : cells) {
+ expectedCellCount++;
+ byte[] family = CellUtil.cloneFamily(expectedCell);
+ byte[] qualifier = CellUtil.cloneQualifier(expectedCell);
+ Cell actualCell = getCell(actual, family, qualifier);
+ if (actualCell == null ||
+ !CellUtil.matchingType(expectedCell, actualCell)) {
+ byte[] dataKey = indexMaintainer.buildDataRowKey(new
ImmutableBytesWritable(expected.getRow()), viewConstants);
+ String errorMsg = "Missing cell (in iteration " +
iteration + ") " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
+ logToIndexToolOutputTable(dataKey, expected.getRow(),
getTimestamp(expected),
+ getTimestamp(actual), errorMsg, isBeforeRebuild,
INVALID_ROW);
+
verificationPhaseResult.setIndexHasMissingCellsCount(verificationPhaseResult.getIndexHasMissingCellsCount()
+ 1);
+ return;
+ }
+ if (!CellUtil.matchingValue(actualCell, expectedCell)) {
+ String errorMsg = "Not matching value (in iteration " +
iteration + ") for " + Bytes.toString(family) + ":" + Bytes.toString(qualifier);
Review comment:
Did you mean 5923? Single cell format for indexes will be optional.
Also, please note that there will be a separate cell for each family.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> IndexRepairRegionScanner to verify and repair every global index row
> --------------------------------------------------------------------
>
> Key: PHOENIX-6181
> URL: https://issues.apache.org/jira/browse/PHOENIX-6181
> Project: Phoenix
> Issue Type: Improvement
> Affects Versions: 5.0.0, 4.14.3
> Reporter: Kadir OZDEMIR
> Assignee: Kadir OZDEMIR
> Priority: Major
> Attachments: PHOENIX-6181.4.x.001.patch
>
>
> IndexRebuildRegionScanner is the server side engine to rebuild and verify
> every index row pointed by the data table. IndexRebuildRegionScanner runs on
> data table regions and scans every data table rows locally, and then rebuilds
> and verifies index table rows referenced by the data table rows over
> server-to-server RPCs using the HBase client installed on region servers.
> However, IndexRebuildRegionScanner cannot clean up the index rows that are
> not referenced by the data table if there are such index rows. In order to do
> that we need another region scanner that scans index table regions and makes
> sure that every index row is valid. This region scanner will be called
> IndexRepairRegionScanner.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
