[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314456283 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java ## @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +public class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger( + HBCKMetaTableAccessor.class); + + // Constants + private static final byte[] MERGE_QUALIFIER_PREFIX = Bytes.toBytes("merge"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell : cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo + .parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + /** + * Delete the passed RegionInfo from the hbase:meta table. + * + * @param connection connection we're using + * @param regionInfo the regionInfo to delete from the meta table + * @throws IOException + */ + public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo) + throws IOException { +Delete delete = new Delete(regionInfo.getRegionName()); +delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP); +deleteFromMetaTable(connection, delete); +LOG.info("Deleted {}", regionInfo.getRegionNameAsString()); + } + + // Private helper methods + + // COPIED from MetaTableAccessor.isMergeQualifierPrefix() + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the MERGE_QUALIFIER_PREFIX +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && qualifierStartsWith(cell, +MERGE_QUALIFIER_PREFIX); + } + + /** + * Finds if the start of the qualifier part of the Cell matches 'startsWith' + * + * COPIED from PrivateCellUtil.qualifierStartsWith() + * @param left the cell with which we need to match the qualifier + * @param startsWith the serialized keyvalue format byte[] + * @return true if the qualifier have same staring characters, false otherwise + */ + private static boolean qualifierStartsWith(final Cell left, final byte[] startsWith) { +if (startsWith == null || startsWith.length == 0) { + throw new IllegalArgumentException("Cannot pass an empty startsWith"); +} +return Bytes +.equals(left.getQualifierArray(), left.getQualifierOffset(), startsWith.length, startsWith, Review comment: Yeah so left.getQualifierArray takes of that I guess. It does a CellUtil.cloneQualifier(this) for byteBuffer backed cells, which in turn calls copyQualifierTo(), which takes care of bytebuffer copying by ByteBufferUtils.copyFromBufferToArray(). Let me know if it doesn't sound correct Stack. This is an automated message from the Apache Git Service. To respond to the message, please log on to G
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314447358 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java ## @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +public class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger( + HBCKMetaTableAccessor.class); + + // Constants + private static final byte[] MERGE_QUALIFIER_PREFIX = Bytes.toBytes("merge"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell : cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo + .parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + /** + * Delete the passed RegionInfo from the hbase:meta table. + * + * @param connection connection we're using + * @param regionInfo the regionInfo to delete from the meta table + * @throws IOException + */ + public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo) + throws IOException { +Delete delete = new Delete(regionInfo.getRegionName()); +delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP); +deleteFromMetaTable(connection, delete); +LOG.info("Deleted {}", regionInfo.getRegionNameAsString()); + } + + // Private helper methods + + // COPIED from MetaTableAccessor.isMergeQualifierPrefix() + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the MERGE_QUALIFIER_PREFIX +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && qualifierStartsWith(cell, +MERGE_QUALIFIER_PREFIX); + } + + /** + * Finds if the start of the qualifier part of the Cell matches 'startsWith' + * + * COPIED from PrivateCellUtil.qualifierStartsWith() + * @param left the cell with which we need to match the qualifier + * @param startsWith the serialized keyvalue format byte[] + * @return true if the qualifier have same staring characters, false otherwise + */ + private static boolean qualifierStartsWith(final Cell left, final byte[] startsWith) { +if (startsWith == null || startsWith.length == 0) { + throw new IllegalArgumentException("Cannot pass an empty startsWith"); +} +return Bytes +.equals(left.getQualifierArray(), left.getQualifierOffset(), startsWith.length, startsWith, Review comment: Let me double-check this one. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r31064 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java ## @@ -39,12 +39,13 @@ * hbck's local version of the MetaTableAccessor from the hbase repo */ @InterfaceAudience.Private -class HBCKMetaTableAccessor { +public class HBCKMetaTableAccessor { Review comment: Yes, Stack. It's being used by hbck1.HbaseFsck & hbck1.HbaseFsckRepair. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314435700 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell: cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the merge qualifier 'merge' +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && +HBCKPrivateCellUtil.qualifierStartsWith(cell, HBCKConstants.MERGE_QUALIFIER_PREFIX); + } + + /** + * Returns the column family used for meta columns. + * @return HConstants.CATALOG_FAMILY. + */ + public static byte[] getCatalogFamily() { +return HConstants.CATALOG_FAMILY; + } + + /** + * Delete the passed d from the hbase:meta table. + * @param connection connection we're using + * @param d Delete to add to hbase:meta + */ + private static void deleteFromMetaTable(final Connection connection, final Delete d) + throws IOException { +List dels = new ArrayList<>(1); +dels.add(d); +deleteFromMetaTable(connection, dels); + } + + /** + * Callers should call close on the returned {@link Table} instance. + * @param connection connection we're using to access Meta + * @return An {@link Table} for hbase:meta + */ + public static Table getMetaHTable(final Connection connection) + throws IOException { +// We used to pass whole CatalogTracker in here, now we just pass in Connection +if (connection == null) { + throw new NullPointerException("No connection"); +} else if (connection.isClosed()) { + throw new IOException("connection is closed"); +} +return connection.getTable(TableName.META_TABLE_NAME); + } + + private static void debugLogMutations(List mutations) throws IOException { +if (!HBCKMETALOG.isDebugEnabled()) { + return; +} +// Logging each mutation in separate line makes it easier to see diff between them visually +// because of common starting indentation. +for (Mutation mutation : mutations) { + debugLogMutation(mutation); +} + } + + private static void debugLogMutation(Mutation p) throws IOException { Review comment: Removed this logging. Added a normal debug log, we anyways are only doing a single delete mutation. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314435763 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell: cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the merge qualifier 'merge' +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && +HBCKPrivateCellUtil.qualifierStartsWith(cell, HBCKConstants.MERGE_QUALIFIER_PREFIX); + } + + /** + * Returns the column family used for meta columns. + * @return HConstants.CATALOG_FAMILY. + */ + public static byte[] getCatalogFamily() { +return HConstants.CATALOG_FAMILY; + } + + /** + * Delete the passed d from the hbase:meta table. + * @param connection connection we're using + * @param d Delete to add to hbase:meta + */ + private static void deleteFromMetaTable(final Connection connection, final Delete d) + throws IOException { +List dels = new ArrayList<>(1); +dels.add(d); +deleteFromMetaTable(connection, dels); + } + + /** + * Callers should call close on the returned {@link Table} instance. + * @param connection connection we're using to access Meta + * @return An {@link Table} for hbase:meta + */ + public static Table getMetaHTable(final Connection connection) + throws IOException { +// We used to pass whole CatalogTracker in here, now we just pass in Connection +if (connection == null) { + throw new NullPointerException("No connection"); +} else if (connection.isClosed()) { + throw new IOException("connection is closed"); +} +return connection.getTable(TableName.META_TABLE_NAME); + } + + private static void debugLogMutations(List mutations) throws IOException { +if (!HBCKMETALOG.isDebugEnabled()) { + return; +} +// Logging each mutation in separate line makes it easier to see diff between them visually +// because of common starting indentation. +for (Mutation mutation : mutations) { + debugLogMutation(mutation); +} + } + + private static void debugLogMutation(Mutation p) throws IOException { +HBCKMETALOG.debug("{} {}", p.getClass().getSimpleName(), p.toJSON()); + } + + /** + * Delete the passed deletes from the hbase:meta table. + * @param connection connection we're using + * @param deletes Deletes to add to hbase:meta This list should support #remove. + */ + private static void deleteFromMetaTable(final Connection connection, final List deletes) + throws IOException { +try (Table t = getMetaHTable(connection)) { + debugLogMutations(deletes); + t.delete(deletes); +} + } + + /** + * Delete the passed RegionInfo from the hbase:meta table. + * @param connection connection we're using + * @param regionInfo the regionInfo to delete from the meta table + * @throws IOException + */ + public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo) + throws IOException { +Delete delete = new Delete(regionInfo.getRegionName()); +delete.addFamily(getCatalogFamily(), HConstants.LATEST_TIMESTAMP); Review comment: Done.
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314435285 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; Review comment: Moved it out of the hbck1 package. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314435464 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell: cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the merge qualifier 'merge' +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && +HBCKPrivateCellUtil.qualifierStartsWith(cell, HBCKConstants.MERGE_QUALIFIER_PREFIX); + } + + /** + * Returns the column family used for meta columns. + * @return HConstants.CATALOG_FAMILY. + */ + public static byte[] getCatalogFamily() { +return HConstants.CATALOG_FAMILY; + } + + /** + * Delete the passed d from the hbase:meta table. + * @param connection connection we're using + * @param d Delete to add to hbase:meta + */ + private static void deleteFromMetaTable(final Connection connection, final Delete d) + throws IOException { +List dels = new ArrayList<>(1); +dels.add(d); +deleteFromMetaTable(connection, dels); Review comment: Done. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314435324 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell: cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the merge qualifier 'merge' +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && +HBCKPrivateCellUtil.qualifierStartsWith(cell, HBCKConstants.MERGE_QUALIFIER_PREFIX); Review comment: Done. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314435416 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell: cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the merge qualifier 'merge' +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && +HBCKPrivateCellUtil.qualifierStartsWith(cell, HBCKConstants.MERGE_QUALIFIER_PREFIX); + } + + /** + * Returns the column family used for meta columns. + * @return HConstants.CATALOG_FAMILY. + */ + public static byte[] getCatalogFamily() { +return HConstants.CATALOG_FAMILY; + } + + /** + * Delete the passed d from the hbase:meta table. + * @param connection connection we're using + * @param d Delete to add to hbase:meta + */ + private static void deleteFromMetaTable(final Connection connection, final Delete d) + throws IOException { +List dels = new ArrayList<>(1); +dels.add(d); +deleteFromMetaTable(connection, dels); + } + + /** + * Callers should call close on the returned {@link Table} instance. + * @param connection connection we're using to access Meta + * @return An {@link Table} for hbase:meta + */ + public static Table getMetaHTable(final Connection connection) Review comment: Consolidated. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314432931 ## File path: pom.xml ## @@ -123,7 +123,7 @@ 1.8 ${compileSource} 3.3.3 -2.1.2 +2.1.6-SNAPSHOT Review comment: I guess, we would have to eventually take it up to 2.1.6 considering the fact that we want to call hbck.fixMeta() (which was added server side) which was added in 2.1.6? This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314433521 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { Review comment: Umm, after moving the class out of this package, couldn't keep it package private anymore. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314433586 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { +if (cells == null) { + return null; +} +List regionsToMerge = null; +for (Cell cell: cells) { + if (!isMergeQualifierPrefix(cell)) { +continue; + } + // Ok. This cell is that of a info:merge* column. + RegionInfo ri = RegionInfo.parseFromOrNull(cell.getValueArray(), cell.getValueOffset(), + cell.getValueLength()); + if (ri != null) { +if (regionsToMerge == null) { + regionsToMerge = new ArrayList<>(); +} +regionsToMerge.add(ri); + } +} +return regionsToMerge; + } + + private static boolean isMergeQualifierPrefix(Cell cell) { +// Check to see if has family and that qualifier starts with the merge qualifier 'merge' +return CellUtil.matchingFamily(cell, HConstants.CATALOG_FAMILY) && +HBCKPrivateCellUtil.qualifierStartsWith(cell, HBCKConstants.MERGE_QUALIFIER_PREFIX); + } + + /** + * Returns the column family used for meta columns. + * @return HConstants.CATALOG_FAMILY. + */ + public static byte[] getCatalogFamily() { Review comment: Done. :) This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314434338 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBaseFsckRepair.java ## @@ -200,6 +200,6 @@ public static HRegion createHDFSRegionDir(Configuration conf, */ public static void removeParentInMeta(Configuration conf, RegionInfo hri) throws IOException { Connection conn = ConnectionFactory.createConnection(conf); -MetaTableAccessor.deleteRegion(conn, hri); Review comment: HBCKMTA is being used in 2 places as of now. `List mergeParents = HBCKMetaTableAccessor.getMergeRegions(result.rawCells());` in hbck1.HbaseFsck `HBCKMetaTableAccessor.deleteRegionInfo(conn, hri);` in hbck1.HbaseFsckRepair This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314433208 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKPrivateCellUtil.java ## @@ -0,0 +1,32 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.ByteBufferExtendedCell; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.util.ByteBufferUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * hbck's local version of PrivateCellUtil from the hbase repo + */ +@InterfaceAudience.Private +final class HBCKPrivateCellUtil { Review comment: Moved the method inside class & made a private one. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314433272 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKMetaTableAccessor.java ## @@ -0,0 +1,134 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * hbck's local version of the MetaTableAccessor from the hbase repo + */ +@InterfaceAudience.Private +class HBCKMetaTableAccessor { + + private static final Logger LOG = LoggerFactory.getLogger(HBCKMetaTableAccessor.class); + private static final Logger HBCKMETALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); + + public static List getMergeRegions(Cell[] cells) { Review comment: Done. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] [hbase-operator-tools] jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758
jatsakthi commented on a change in pull request #14: HBASE-22843 [HBCK2] Fix HBCK2 after HBASE-22777 & HBASE-22758 URL: https://github.com/apache/hbase-operator-tools/pull/14#discussion_r314432307 ## File path: hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBCKConstants.java ## @@ -0,0 +1,22 @@ +package org.apache.hbase.hbck1; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * HBCKConstants holds a bunch of HBase(HBCK)-related constants. + * hbck's local version of the HConstants from the hbase repo. + */ +@InterfaceAudience.Private +final class HBCKConstants { + /** + * Merge qualifier prefix. + * We used to only allow two regions merge; mergeA and mergeB. + * Now we allow many to merge. Each region to merge will be referenced + * in a column whose qualifier starts with this define. + */ + public static final String MERGE_QUALIFIER_PREFIX_STR = "merge"; + + public static final byte [] MERGE_QUALIFIER_PREFIX = + Bytes.toBytes(MERGE_QUALIFIER_PREFIX_STR); +} Review comment: Removed the class and defined it within the class. This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services