This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b498c9797626b8d8eb9cdb9bbba246973463aafb
Merge: c650408d05 f8cb312032
Author: Christopher L. Shannon (cshannon) <christopher.l.shan...@gmail.com>
AuthorDate: Sat Dec 2 09:52:34 2023 -0500
Merge branch '2.1'
.../accumulo/manager/TabletGroupWatcher.java | 71 ++++++++++++++--------
1 file changed, 45 insertions(+), 26 deletions(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index c4e29eaf69,81744441aa..068bb4930b
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -615,131 -669,10 +613,130 @@@ abstract class TabletGroupWatcher exten
}
}
+ // Remove the merged marker from the last tablet in the merge range
+ private void clearMerged(MergeInfo mergeInfo, BatchWriter bw, HighTablet
highTablet)
+ throws AccumuloException {
+ Manager.log.debug("Clearing MERGED marker for {}", mergeInfo.getExtent());
+ var m = new Mutation(highTablet.getExtent().toMetaRow());
+ MergedColumnFamily.MERGED_COLUMN.putDelete(m);
+ bw.addMutation(m);
+ bw.flush();
+ }
+
+ // This method finds returns the deletion starting row (exclusive) for
tablets that
+ // need to be actually deleted. If the startTablet is null then
+ // the deletion start row will just be null as all tablets are being deleted
+ // up to the end. Otherwise, this returns the endRow of the first tablet
+ // as the first tablet should be kept and will have been previously
+ // fenced if necessary
+ private Text getDeletionStartRow(final KeyExtent startTablet) {
+ if (startTablet == null) {
+ Manager.log.debug("First tablet for delete range is null");
+ return null;
+ }
+
+ final Text deletionStartRow = startTablet.endRow();
+ Manager.log.debug("Start row is {} for deletion", deletionStartRow);
+
+ return deletionStartRow;
+ }
+
+ // This method finds returns the deletion ending row (inclusive) for
tablets that
+ // need to be actually deleted. If the endTablet is null then
+ // the deletion end row will just be null as all tablets are being deleted
+ // after the start row. Otherwise, this returns the prevEndRow of the last
tablet
+ // as the last tablet should be kept and will have been previously
+ // fenced if necessary
+ private Text getDeletionEndRow(final KeyExtent endTablet) {
+ if (endTablet == null) {
+ Manager.log.debug("Last tablet for delete range is null");
+ return null;
+ }
+
+ Text deletionEndRow = endTablet.prevEndRow();
+ Manager.log.debug("Deletion end row is {}", deletionEndRow);
+
+ return deletionEndRow;
+ }
+
+ private static boolean isFirstTabletInTable(KeyExtent tablet) {
+ return tablet != null && tablet.prevEndRow() == null;
+ }
+
+ private static boolean isLastTabletInTable(KeyExtent tablet) {
+ return tablet != null && tablet.endRow() == null;
+ }
+
+ private static boolean areContiguousTablets(KeyExtent firstTablet,
KeyExtent lastTablet) {
+ return firstTablet != null && lastTablet != null
+ && Objects.equals(firstTablet.endRow(), lastTablet.prevEndRow());
+ }
+
+ private boolean hasTabletsToDelete(final KeyExtent firstTabletInRange,
+ final KeyExtent lastTableInRange) {
+ // If the tablets are equal (and not null) then the deletion range is
just part of 1 tablet
+ // which will be fenced so there are no tablets to delete. The null check
is because if both
+ // are null then we are just deleting everything, so we do have tablets
to delete
+ if (Objects.equals(firstTabletInRange, lastTableInRange) &&
firstTabletInRange != null) {
+ Manager.log.trace(
+ "No tablets to delete, firstTablet {} equals lastTablet {} in
deletion range and was fenced.",
+ firstTabletInRange, lastTableInRange);
+ return false;
+ // If the lastTablet of the deletion range is the first tablet of the
table it has been fenced
+ // already so nothing to actually delete before it
+ } else if (isFirstTabletInTable(lastTableInRange)) {
+ Manager.log.trace(
+ "No tablets to delete, lastTablet {} in deletion range is the first
tablet of the table and was fenced.",
+ lastTableInRange);
+ return false;
+ // If the firstTablet of the deletion range is the last tablet of the
table it has been fenced
+ // already so nothing to actually delete after it
+ } else if (isLastTabletInTable(firstTabletInRange)) {
+ Manager.log.trace(
+ "No tablets to delete, firstTablet {} in deletion range is the last
tablet of the table and was fenced.",
+ firstTabletInRange);
+ return false;
+ // If the firstTablet and lastTablet are contiguous tablets then there
is nothing to delete as
+ // each will be fenced and nothing between
+ } else if (areContiguousTablets(firstTabletInRange, lastTableInRange)) {
+ Manager.log.trace(
+ "No tablets to delete, firstTablet {} and lastTablet {} in deletion
range are contiguous and were fenced.",
+ firstTabletInRange, lastTableInRange);
+ return false;
+ }
+
+ return true;
+ }
+
private void deleteTablets(MergeInfo info) throws AccumuloException {
- KeyExtent extent = info.getExtent();
+ // Before updated metadata and get the first and last tablets which
+ // are fenced if necessary
+ final Pair<KeyExtent,KeyExtent> firstAndLastTablets =
updateMetadataRecordsForDelete(info);
+
+ // Find the deletion start row (exclusive) for tablets that need to be
actually deleted
+ // This will be null if deleting everything up until the end row or it
will be
+ // the endRow of the first tablet as the first tablet should be kept and
will have
+ // already been fenced if necessary
+ final Text deletionStartRow =
getDeletionStartRow(firstAndLastTablets.getFirst());
+
+ // Find the deletion end row (inclusive) for tablets that need to be
actually deleted
+ // This will be null if deleting everything after the starting row or it
will be
+ // the prevEndRow of the last tablet as the last tablet should be kept
and will have
+ // already been fenced if necessary
+ Text deletionEndRow = getDeletionEndRow(firstAndLastTablets.getSecond());
+
+ // check if there are any tablets to delete and if not return
+ if (!hasTabletsToDelete(firstAndLastTablets.getFirst(),
firstAndLastTablets.getSecond())) {
+ Manager.log.trace("No tablets to delete for range {}, returning",
info.getExtent());
+ return;
+ }
+
+ // Build an extent for the actual deletion range
+ final KeyExtent extent =
+ new KeyExtent(info.getExtent().tableId(), deletionEndRow,
deletionStartRow);
+ Manager.log.debug("Tablet deletion range is {}", extent);
String targetSystemTable = extent.isMeta() ? RootTable.NAME :
MetadataTable.NAME;
Manager.log.debug("Deleting tablets for {}", extent);
- MetadataTime metadataTime = null;
KeyExtent followingTablet = null;
if (extent.endRow() != null) {
Key nextExtent = new Key(extent.endRow()).followingKey(PartialKey.ROW);
@@@ -804,23 -735,31 +800,30 @@@
bw.close();
}
+ // If there is another tablet after the delete range then update the
prev end row
+ // of that tablet as all tablets will have been deleted in the delete
range.
+ // If the delete range includes the last tablet in the table then we
need
+ // to update the last tablet's previous end row as deleteTablets() will
keep the
+ // last tablet and only delete the files as we require at least 1
tablet to exist
+ final KeyExtent goalTablet;
if (followingTablet != null) {
- Manager.log.debug("Updating prevRow of {} to {}", followingTablet,
extent.prevEndRow());
- bw = client.createBatchWriter(targetSystemTable);
- try {
- Mutation m = new Mutation(followingTablet.toMetaRow());
- TabletColumnFamily.PREV_ROW_COLUMN.put(m,
- TabletColumnFamily.encodePrevEndRow(extent.prevEndRow()));
- bw.addMutation(m);
- bw.flush();
- } finally {
- bw.close();
- }
+ goalTablet = followingTablet;
} else {
- // Recreate the default tablet to hold the end of the table
- MetadataTableUtil.addTablet(new KeyExtent(extent.tableId(), null,
extent.prevEndRow()),
- ServerColumnFamily.DEFAULT_TABLET_DIR_NAME, manager.getContext(),
- metadataTime.getType(), manager.managerLock);
+ goalTablet = lastTabletInRange;
+ Preconditions.checkState(goalTablet != null && goalTablet.endRow() ==
null,
+ "If followingTablet is null then last tablet in delete range
should be the last tablet in the table");
+ }
+
+ Manager.log.debug("Updating prevRow of {} to {}", goalTablet,
extent.prevEndRow());
+ bw = client.createBatchWriter(targetSystemTable);
+ try {
+ Mutation m = new Mutation(goalTablet.toMetaRow());
+ TabletColumnFamily.PREV_ROW_COLUMN.put(m,
+ TabletColumnFamily.encodePrevEndRow(extent.prevEndRow()));
- ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
+ bw.addMutation(m);
+ bw.flush();
+ } finally {
+ bw.close();
}
} catch (RuntimeException | TableNotFoundException ex) {
throw new AccumuloException(ex);
@@@ -1243,29 -884,8 +1246,29 @@@
}
}
+ // Divide each new DFV in half and make sure the sum equals the original
+ @VisibleForTesting
+ protected static Pair<DataFileValue,DataFileValue>
computeNewDfv(DataFileValue value) {
+ final DataFileValue file1Value = new DataFileValue(Math.max(1,
value.getSize() / 2),
+ Math.max(1, value.getNumEntries() / 2), value.getTime());
+
+ final DataFileValue file2Value =
+ new DataFileValue(Math.max(1, value.getSize() - file1Value.getSize()),
+ Math.max(1, value.getNumEntries() - file1Value.getNumEntries()),
value.getTime());
+
+ return new Pair<>(file1Value, file2Value);
+ }
+
+ private Optional<TabletMetadata> loadTabletMetadata(TableId tabletId, final
Text row,
+ ColumnType... columns) {
+ try (TabletsMetadata tabletsMetadata =
manager.getContext().getAmple().readTablets()
+ .forTable(tabletId).overlapping(row, true,
row).fetch(columns).build()) {
+ return tabletsMetadata.stream().findFirst();
+ }
+ }
+
- private void deleteTablets(MergeInfo info, Range scanRange, BatchWriter bw,
AccumuloClient client)
- throws TableNotFoundException, MutationsRejectedException {
+ private KeyExtent deleteTablets(MergeInfo info, Range scanRange,
BatchWriter bw,
+ AccumuloClient client) throws TableNotFoundException, AccumuloException
{
Scanner scanner;
Mutation m;
// Delete everything in the other tablets
