This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push: new 09828fe04a Create getTabletInformation API method (#3645) 09828fe04a is described below commit 09828fe04ad0ceffc5280a2e67bcdd4a56f08383 Author: Mark Owens <jmar...@apache.org> AuthorDate: Thu Aug 10 11:59:59 2023 -0400 Create getTabletInformation API method (#3645) This PR creates a new public API method, getTabletInformation, which returns a stream of TabletInformation objects containing information for each tablet within a provided table. The method can be accessed from the code or JShell via the TableOperations object. A new TabletInformation class is used to hold information for an individual tablet. The 'listtablets' command has been modified to use this new method rather than using non-public API. Additionally, the listtablets command now includes the TabletHostingGoal of each tablet in its output. The ListTabletsCommandTest was updated to reflect the new method. A new IT test has been added to ShellServerIT which tests 'listtablets' and effectively tests the new API since the method is used to obtain the information for 'listtablets'. Closes #3503 Co-authored-by: Keith Turner <ktur...@apache.org> --- .../core/client/admin/TableOperations.java | 10 + .../core/client/admin/TabletInformation.java | 76 +++++++ .../core/clientImpl/TableOperationsImpl.java | 53 ++++- .../core/clientImpl/TabletInformationImpl.java | 102 +++++++++ .../shell/commands/ListTabletsCommand.java | 253 ++++----------------- .../shell/commands/ListTabletsCommandTest.java | 211 ++++++++--------- .../apache/accumulo/test/TableOperationsIT.java | 100 +++----- .../apache/accumulo/test/shell/ShellServerIT.java | 79 +++++++ 8 files changed, 494 insertions(+), 390 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java index 6b05f527e2..b30aace300 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java @@ -1034,4 +1034,14 @@ public interface TableOperations { throw new UnsupportedOperationException(); } + /** + * @return a stream of tablets information for tablets that fall in the specified range. The + * stream may be backed by a scanner, so its best to close the stream. + * @since 4.0.0 + */ + default Stream<TabletInformation> getTabletInformation(final String tableName, final Range range) + throws TableNotFoundException { + throw new UnsupportedOperationException(); + } + } diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TabletInformation.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TabletInformation.java new file mode 100644 index 0000000000..b303dadfcf --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TabletInformation.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.client.admin; + +import java.util.Optional; + +import org.apache.accumulo.core.data.TabletId; + +/** + * @since 4.0.0 + */ +public interface TabletInformation { + + /** + * @return the TabletId for this tablet. + */ + TabletId getTabletId(); + + /** + * @return the number of files in the tablet directory. + */ + int getNumFiles(); + + /** + * @return the number of write-ahead logs associated with the tablet. + */ + int getNumWalLogs(); + + /** + * @return an estimated number of entries in the tablet. + */ + long getEstimatedEntries(); + + /** + * @return an estimated size of the tablet data on disk, which is likely the compressed size of + * the data. + */ + long getEstimatedSize(); + + /** + * @return the tablet hosting state. + */ + String getTabletState(); + + /** + * @return the Location of the tablet as a String. + */ + Optional<String> getLocation(); + + /** + * @return the directory name of the tablet. + */ + String getTabletDir(); + + /** + * @return the tablet hosting goal. + */ + TabletHostingGoal getHostingGoal(); + +} diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java index 4eea077cbe..2fa8433d1f 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java @@ -26,10 +26,16 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toSet; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.DIR; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_GOAL; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_REQUESTED; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME; import static org.apache.accumulo.core.util.LazySingletons.RANDOM; import static org.apache.accumulo.core.util.Validators.EXISTING_TABLE_NAME; import static org.apache.accumulo.core.util.Validators.NEW_TABLE_NAME; @@ -95,6 +101,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.SummaryRetriever; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.admin.TabletHostingGoal; +import org.apache.accumulo.core.client.admin.TabletInformation; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.client.admin.compaction.CompactionConfigurer; import org.apache.accumulo.core.client.admin.compaction.CompactionSelector; @@ -134,6 +141,8 @@ import org.apache.accumulo.core.manager.thrift.FateService; import org.apache.accumulo.core.manager.thrift.ManagerClientService; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.TabletDeletedException; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; @@ -696,7 +705,7 @@ public class TableOperationsImpl extends TableOperationsHelper { throws AccumuloException, AccumuloSecurityException, TableNotFoundException { EXISTING_TABLE_NAME.validate(tableName); - List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableName.getBytes(UTF_8))); + List<ByteBuffer> args = List.of(ByteBuffer.wrap(tableName.getBytes(UTF_8))); Map<String,String> opts = new HashMap<>(); try { doTableFateOperation(tableName, TableNotFoundException.class, FateOperation.TABLE_DELETE, @@ -846,7 +855,7 @@ public class TableOperationsImpl extends TableOperationsHelper { EXISTING_TABLE_NAME.validate(tableName); TableId tableId = context.getTableId(tableName); - List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8))); + List<ByteBuffer> args = List.of(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8))); Map<String,String> opts = new HashMap<>(); try { @@ -1176,7 +1185,7 @@ public class TableOperationsImpl extends TableOperationsHelper { TableId tableId = context.getTableId(tableName); ClientTabletCache tl = ClientTabletCache.getInstance(context, tableId); - // its possible that the cache could contain complete, but old information about a tables + // it's possible that the cache could contain complete, but old information about a tables // tablets... so clear it tl.invalidateCache(); @@ -1420,7 +1429,7 @@ public class TableOperationsImpl extends TableOperationsHelper { return; } - List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8))); + List<ByteBuffer> args = List.of(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8))); Map<String,String> opts = new HashMap<>(); try { @@ -1477,7 +1486,7 @@ public class TableOperationsImpl extends TableOperationsHelper { while (diskUsages == null) { Pair<String,Client> pair = null; try { - // this operation may us a lot of memory... its likely that connections to tabletservers + // this operation may us a lot of memory... it's likely that connections to tabletservers // hosting metadata tablets will be cached, so do not use cached // connections pair = ThriftClientTypes.CLIENT.getTabletServerConnection(context, false); @@ -2086,7 +2095,8 @@ public class TableOperationsImpl extends TableOperationsHelper { public TimeType getTimeType(final String tableName) throws TableNotFoundException { TableId tableId = context.getTableId(tableName); Optional<TabletMetadata> tabletMetadata = context.getAmple().readTablets().forTable(tableId) - .fetch(TabletMetadata.ColumnType.TIME).checkConsistency().build().stream().findFirst(); + .fetch(TIME).checkConsistency().build().stream().findFirst(); + TabletMetadata timeData = tabletMetadata.orElseThrow(() -> new IllegalStateException("Failed to retrieve TimeType")); return timeData.getTime().getType(); @@ -2162,7 +2172,7 @@ public class TableOperationsImpl extends TableOperationsHelper { return tabletsMetadata.stream().peek(tm -> { if (scanRangeStart != null && tm.getEndRow() != null && tm.getEndRow().compareTo(scanRangeStart) < 0) { - log.debug(">>>> tablet {} is before scan start range: {}", tm.getExtent(), scanRangeStart); + log.debug("tablet {} is before scan start range: {}", tm.getExtent(), scanRangeStart); throw new RuntimeException("Bug in ample or this code."); } }).takeWhile(tm -> tm.getPrevEndRow() == null @@ -2170,4 +2180,33 @@ public class TableOperationsImpl extends TableOperationsHelper { .map(tm -> new HostingGoalForTablet(new TabletIdImpl(tm.getExtent()), tm.getHostingGoal())) .onClose(tabletsMetadata::close); } + + @Override + public Stream<TabletInformation> getTabletInformation(final String tableName, final Range range) + throws TableNotFoundException { + EXISTING_TABLE_NAME.validate(tableName); + + final Text scanRangeStart = (range.getStartKey() == null) ? null : range.getStartKey().getRow(); + TableId tableId = context.getTableId(tableName); + + TabletsMetadata tabletsMetadata = + context.getAmple().readTablets().forTable(tableId).overlapping(scanRangeStart, true, null) + .fetch(HOSTING_GOAL, LOCATION, DIR, PREV_ROW, FILES, LAST, LOGS, SUSPEND) + .checkConsistency().build(); + + Set<TServerInstance> liveTserverSet = TabletMetadata.getLiveTServers(context); + + return tabletsMetadata.stream().peek(tm -> { + if (scanRangeStart != null && tm.getEndRow() != null + && tm.getEndRow().compareTo(scanRangeStart) < 0) { + log.debug("tablet {} is before scan start range: {}", tm.getExtent(), scanRangeStart); + throw new RuntimeException("Bug in ample or this code."); + } + }).takeWhile(tm -> tm.getPrevEndRow() == null + || !range.afterEndKey(new Key(tm.getPrevEndRow()).followingKey(PartialKey.ROW))) + .map(tm -> (TabletInformation) new TabletInformationImpl(tm, + TabletState.compute(tm, liveTserverSet).toString())) + .onClose(tabletsMetadata::close); + } + } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletInformationImpl.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletInformationImpl.java new file mode 100644 index 0000000000..064e5e68d6 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletInformationImpl.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.clientImpl; + +import java.util.Optional; + +import org.apache.accumulo.core.client.admin.TabletHostingGoal; +import org.apache.accumulo.core.client.admin.TabletInformation; +import org.apache.accumulo.core.data.TabletId; +import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; + +public class TabletInformationImpl implements TabletInformation { + + private final TabletMetadata tabletMetadata; + private long estimatedSize; + private long estimatedEntries; + private final String tabletState; + + public TabletInformationImpl(TabletMetadata tabletMetadata, String tabletState) { + this.tabletMetadata = tabletMetadata; + estimatedEntries = 0L; + estimatedSize = 0L; + for (DataFileValue dfv : tabletMetadata.getFilesMap().values()) { + estimatedEntries += dfv.getNumEntries(); + estimatedSize += dfv.getSize(); + } + this.tabletState = tabletState; + } + + @Override + public TabletId getTabletId() { + return new TabletIdImpl(tabletMetadata.getExtent()); + } + + @Override + public int getNumFiles() { + return tabletMetadata.getFilesMap().size(); + } + + @Override + public int getNumWalLogs() { + return tabletMetadata.getLogs().size(); + } + + @Override + public long getEstimatedEntries() { + return this.estimatedEntries; + } + + @Override + public long getEstimatedSize() { + return estimatedSize; + } + + @Override + public String getTabletState() { + return tabletState; + } + + @Override + public Optional<String> getLocation() { + Location location = tabletMetadata.getLocation(); + return location == null ? Optional.empty() + : Optional.of(location.getType() + ":" + location.getHostPort()); + } + + @Override + public String getTabletDir() { + return tabletMetadata.getDirName(); + } + + @Override + public TabletHostingGoal getHostingGoal() { + return tabletMetadata.getHostingGoal(); + } + + @Override + public String toString() { + return "TabletInformationImpl{tabletMetadata=" + tabletMetadata + ", estimatedSize=" + + estimatedSize + ", estimatedEntries=" + estimatedEntries + ", tabletState='" + tabletState + + '\'' + '}'; + } +} diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java index 3e0782f723..abc6db9f40 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ListTabletsCommand.java @@ -18,7 +18,6 @@ */ package org.apache.accumulo.shell.commands; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -26,20 +25,15 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.accumulo.core.client.NamespaceNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; -import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.client.admin.TabletInformation; import org.apache.accumulo.core.clientImpl.Namespaces; import org.apache.accumulo.core.data.NamespaceId; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; -import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.core.metadata.TabletState; -import org.apache.accumulo.core.metadata.schema.DataFileValue; -import org.apache.accumulo.core.metadata.schema.TabletMetadata; -import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; -import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.util.NumUtil; import org.apache.accumulo.shell.Shell; import org.apache.accumulo.shell.Shell.Command; @@ -47,7 +41,6 @@ import org.apache.accumulo.shell.ShellOptions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +48,7 @@ import com.google.common.annotations.VisibleForTesting; /** * Utility that generates single line tablet info. The output of this could be fed to sort, awk, - * grep, etc inorder to answer questions like which tablets have the most files. + * grep, etc. in order to answer questions like which tablets have the most files. */ public class ListTabletsCommand extends Command { @@ -67,6 +60,11 @@ public class ListTabletsCommand extends Command { private Option optNamespace; private Option disablePaginationOpt; + static final String header = + String.format("%-4s %-15s %-5s %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s %-10s", "NUM", + "TABLET_DIR", "FILES", "WALS", "ENTRIES", "SIZE", "STATUS", "LOCATION", "ID", + "START (Exclusive)", "END", "GOAL"); + @Override public int execute(String fullCommand, CommandLine cl, Shell shellState) throws Exception { final Set<TableInfo> tableInfoSet = populateTables(cl, shellState); @@ -74,18 +72,30 @@ public class ListTabletsCommand extends Command { log.warn("No tables found that match your criteria"); return 0; } + boolean humanReadable = cl.hasOption(optHumanReadable.getOpt()); List<String> lines = new LinkedList<>(); - lines.add(TabletRowInfo.header); + lines.add(header); for (TableInfo tableInfo : tableInfoSet) { String name = tableInfo.name; lines.add("TABLE: " + name); - List<TabletRowInfo> rows = getTabletRowInfo(shellState, tableInfo); - for (int i = 0; i < rows.size(); i++) { - TabletRowInfo row = rows.get(i); - lines.add(row.format(i + 1, humanReadable)); + List<TabletInformation> tabletsList = shellState.getContext().tableOperations() + .getTabletInformation(name, new Range()).collect(Collectors.toList()); + for (int i = 0; i < tabletsList.size(); i++) { + TabletInformation tabletInfo = tabletsList.get(i); + lines.add(String.format("%-4d %-15s %-5d %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s %-10s", + i + 1, tabletInfo.getTabletDir(), tabletInfo.getNumFiles(), tabletInfo.getNumWalLogs(), + getEstimatedEntries(tabletInfo.getEstimatedEntries(), humanReadable), + getEstimatedSize(tabletInfo.getEstimatedSize(), humanReadable), + tabletInfo.getTabletState(), tabletInfo.getLocation().orElse("None"), + tabletInfo.getTabletId().getTable(), + tabletInfo.getTabletId().getPrevEndRow() == null ? "-INF" + : tabletInfo.getTabletId().getPrevEndRow().toString(), + tabletInfo.getTabletId().getEndRow() == null ? "+INF" + : tabletInfo.getTabletId().getEndRow().toString(), + tabletInfo.getHostingGoal())); } } @@ -97,6 +107,20 @@ public class ListTabletsCommand extends Command { return 0; } + private String getEstimatedSize(long size, boolean humanReadable) { + if (humanReadable) { + return NumUtil.bigNumberForQuantity(size); + } + return String.format("%,d", size); + } + + private String getEstimatedEntries(long numEntries, boolean humanReadable) { + if (humanReadable) { + return NumUtil.bigNumberForQuantity(numEntries); + } + return String.format("%,d", numEntries); + } + @VisibleForTesting protected void printResults(CommandLine cl, Shell shellState, List<String> lines) throws Exception { @@ -180,7 +204,7 @@ public class ListTabletsCommand extends Command { } /** - * Wrapper for tablename and id. Comparisons, equals and hash code use tablename (id is ignored) + * Wrapper for tableName and id. Comparisons, equals and hash code use tableName (id is ignored) */ static class TableInfo implements Comparable<TableInfo> { @@ -215,52 +239,6 @@ public class ListTabletsCommand extends Command { } } - private List<TabletRowInfo> getTabletRowInfo(Shell shellState, TableInfo tableInfo) - throws Exception { - log.trace("scan metadata for tablet info table name: \'{}\', tableId: \'{}\' ", tableInfo.name, - tableInfo.id); - - List<TabletRowInfo> tResults = getMetadataInfo(shellState, tableInfo); - - if (log.isTraceEnabled()) { - for (TabletRowInfo tabletRowInfo : tResults) { - log.trace("Tablet info: {}", tabletRowInfo); - } - } - - return tResults; - } - - protected List<TabletRowInfo> getMetadataInfo(Shell shellState, TableInfo tableInfo) - throws Exception { - List<TabletRowInfo> results = new ArrayList<>(); - final ClientContext context = shellState.getContext(); - Set<TServerInstance> liveTserverSet = TabletMetadata.getLiveTServers(context); - - try (var tabletsMetadata = TabletsMetadata.builder(context).forTable(tableInfo.id).build()) { - for (var md : tabletsMetadata) { - TabletRowInfo.Factory factory = new TabletRowInfo.Factory(tableInfo.name, md.getExtent()); - var fileMap = md.getFilesMap(); - factory.numFiles(fileMap.size()); - long entries = 0L; - long size = 0L; - for (DataFileValue dfv : fileMap.values()) { - entries += dfv.getNumEntries(); - size += dfv.getSize(); - } - factory.numEntries(entries); - factory.size(size); - factory.numWalLogs(md.getLogs().size()); - factory.dir(md.getDirName()); - factory.location(md.getLocation()); - factory.status(TabletState.compute(md, liveTserverSet).toString()); - results.add(factory.build()); - } - } - - return results; - } - @Override public String description() { return "Prints info about every tablet for a table, one tablet per line."; @@ -302,153 +280,4 @@ public class ListTabletsCommand extends Command { return opts; } - static class TabletRowInfo { - - public final String tableName; - public final int numFiles; - public final int numWalLogs; - public final long numEntries; - public final long size; - public final String status; - public final String location; - public final String dir; - public final TableId tableId; - public final KeyExtent tablet; - public final boolean tableExists; - - private TabletRowInfo(String tableName, KeyExtent tablet, int numFiles, int numWalLogs, - long numEntries, long size, String status, String location, String dir, - boolean tableExists) { - this.tableName = tableName; - this.tableId = tablet.tableId(); - this.tablet = tablet; - this.numFiles = numFiles; - this.numWalLogs = numWalLogs; - this.numEntries = numEntries; - this.size = size; - this.status = status; - this.location = location; - this.dir = dir; - this.tableExists = tableExists; - } - - String getNumEntries(final boolean humanReadable) { - if (humanReadable) { - return String.format("%9s", NumUtil.bigNumberForQuantity(numEntries)); - } - // return String.format("%,24d", numEntries); - return Long.toString(numEntries); - } - - String getSize(final boolean humanReadable) { - if (humanReadable) { - return String.format("%9s", NumUtil.bigNumberForSize(size)); - } - // return String.format("%,24d", size); - return Long.toString(size); - } - - public String getEndRow() { - Text t = tablet.endRow(); - if (t == null) { - return "+INF"; - } else { - return t.toString(); - } - } - - public String getStartRow() { - Text t = tablet.prevEndRow(); - if (t == null) { - return "-INF"; - } else { - return t.toString(); - } - } - - public static final String header = String.format( - "%-4s %-15s %-5s %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s", "NUM", "TABLET_DIR", "FILES", - "WALS", "ENTRIES", "SIZE", "STATUS", "LOCATION", "ID", "START (Exclusive)", "END"); - - String format(int number, boolean prettyPrint) { - return String.format("%-4d %-15s %-5d %-5s %-9s %-9s %-10s %-30s %-5s %-20s %-20s", number, - dir, numFiles, numWalLogs, getNumEntries(prettyPrint), getSize(prettyPrint), status, - location, tableId, getStartRow(), getEndRow()); - } - - public String getTablet() { - return getStartRow() + " " + getEndRow(); - } - - public static class Factory { - final String tableName; - final KeyExtent tablet; - final TableId tableId; - int numFiles = 0; - int numWalLogs = 0; - long numEntries = 0; - long size = 0; - String status = ""; - String location = ""; - String dir = ""; - boolean tableExists = false; - - Factory(final String tableName, KeyExtent tablet) { - this.tableName = tableName; - this.tablet = tablet; - this.tableId = tablet.tableId(); - } - - Factory numFiles(int numFiles) { - this.numFiles = numFiles; - return this; - } - - Factory numWalLogs(int numWalLogs) { - this.numWalLogs = numWalLogs; - return this; - } - - public Factory numEntries(long numEntries) { - this.numEntries = numEntries; - return this; - } - - public Factory size(long size) { - this.size = size; - return this; - } - - public Factory status(String status) { - this.status = status; - return this; - } - - public Factory location(Location location) { - if (location == null) { - this.location = "None"; - } else { - String server = location.getHostPort(); - this.location = location.getType() + ":" + server; - } - return this; - } - - public Factory dir(String dirName) { - this.dir = dirName; - return this; - } - - public Factory tableExists(boolean tableExists) { - this.tableExists = tableExists; - return this; - } - - public TabletRowInfo build() { - return new TabletRowInfo(tableName, tablet, numFiles, numWalLogs, numEntries, size, status, - location, dir, tableExists); - } - } - } - } diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java index 583819b96b..e080d29f13 100644 --- a/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ListTabletsCommandTest.java @@ -18,8 +18,9 @@ */ package org.apache.accumulo.shell.commands; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; @@ -27,15 +28,22 @@ import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Stream; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.InstanceOperations; import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.admin.TabletHostingGoal; import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.clientImpl.TabletInformationImpl; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.metadata.TabletState; -import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.shell.Shell; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -44,75 +52,119 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.io.Text; import org.easymock.EasyMock; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import com.google.common.net.HostAndPort; public class ListTabletsCommandTest { - private static final Logger log = LoggerFactory.getLogger(ListTabletsCommandTest.class); - final String tableName = ListTabletsCommandTest.class.getName() + "-aTable"; - private static final TableId tableId = TableId.of("123"); - private static final String rowString = "123;a 123;m 123<"; - private static final List<String> rows = new ArrayList<>(Arrays.asList(rowString.split(" "))); + final static String tableName = ListTabletsCommandTest.class.getName() + "-aTable"; + + private static class TestListTabletsCommand extends ListTabletsCommand { - private class TestListTabletsCommand extends ListTabletsCommand { @Override protected void printResults(CommandLine cl, Shell shellState, List<String> lines) { - log.debug("Command run successfully. Output below..."); - for (String line : lines) { - log.debug(line); - } - assertEquals(TabletRowInfo.header, lines.get(0)); + + // there are three rows of tablet info plus 2 lines of header info + assertEquals(lines.size(), 5, "Incorrect number of rows: " + lines.size()); + assertEquals(header, lines.get(0)); assertTrue(lines.get(1).startsWith("TABLE:")); + assertTrue(lines.get(1).contains(tableName)); + // first table info + List<String> items = new ArrayList<>(Arrays.asList(lines.get(2).split("\\s+"))); assertTrue(lines.get(2).startsWith("1")); - assertTrue(lines.get(2).contains("t-dir1")); - assertTrue(lines.get(2).contains("100")); - assertTrue(lines.get(2).contains("-INF")); - assertTrue(lines.get(2).endsWith("a ")); + assertEquals("1", items.get(0)); // line count + assertEquals("t-dir1", items.get(1)); // dir name + assertEquals("2", items.get(2)); // files + assertEquals("1", items.get(3)); // wals + assertEquals("1,116", items.get(4)); // entries + assertEquals("385,606", items.get(5)); // size + assertEquals("HOSTED", items.get(6)); // status + assertEquals("CURRENT:server1:8555", items.get(7)); // location + assertEquals("123", items.get(8)); // id + assertEquals("-INF", items.get(9)); // start + assertEquals("d", items.get(10)); // end + assertEquals("ONDEMAND", items.get(11)); // goal // second tablet info + items.clear(); + items = new ArrayList<>(Arrays.asList(lines.get(3).split("\\s+"))); assertTrue(lines.get(3).startsWith("2")); - assertTrue(lines.get(3).contains("t-dir2")); - assertTrue(lines.get(3).contains("200")); - assertTrue(lines.get(3).contains("a")); - assertTrue(lines.get(3).endsWith("m ")); + assertEquals("2", items.get(0)); + assertEquals("t-dir2", items.get(1)); + assertEquals("1", items.get(2)); + assertEquals("0", items.get(3)); + assertEquals("142", items.get(4)); + assertEquals("5,323", items.get(5)); + assertEquals("HOSTED", items.get(6)); + assertEquals("CURRENT:server2:2354", items.get(7)); + assertEquals("123", items.get(8)); + assertEquals("e", items.get(9)); + assertEquals("k", items.get(10)); + assertEquals("ALWAYS", items.get(11)); // third tablet info + items.clear(); + items = new ArrayList<>(Arrays.asList(lines.get(4).split("\\s+"))); assertTrue(lines.get(4).startsWith("3")); - assertTrue(lines.get(4).contains("t-dir3")); - assertTrue(lines.get(4).contains("300")); - assertTrue(lines.get(4).contains("m")); - assertTrue(lines.get(4).endsWith("+INF ")); + assertEquals("3", items.get(0)); + assertEquals("t-dir3", items.get(1)); + assertEquals("1", items.get(2)); + assertEquals("2", items.get(3)); + assertEquals("231", items.get(4)); + assertEquals("95,832", items.get(5)); + assertEquals("UNASSIGNED", items.get(6)); + assertEquals("None", items.get(7)); + assertEquals("123", items.get(8)); + assertEquals("l", items.get(9)); + assertEquals("+INF", items.get(10)); + assertEquals("NEVER", items.get(11)); } - @Override - protected List<TabletRowInfo> getMetadataInfo(Shell shellState, - ListTabletsCommand.TableInfo tableInfo) throws Exception { - List<TabletRowInfo> tablets = new ArrayList<>(); - KeyExtent ke1 = new KeyExtent(tableId, new Text("a"), null); - KeyExtent ke2 = new KeyExtent(tableId, new Text("m"), new Text("a")); - KeyExtent ke3 = new KeyExtent(tableId, null, new Text("m")); - Location loc = Location.current("localhost", ""); - ListTabletsCommand.TabletRowInfo.Factory factory = - new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke1).dir("t-dir1").numFiles(1) - .numWalLogs(1).numEntries(1).size(100).status(TabletState.HOSTED.toString()) - .location(loc).tableExists(true); - tablets.add(factory.build()); - factory = new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke2).dir("t-dir2") - .numFiles(2).numWalLogs(2).numEntries(2).size(200).status(TabletState.HOSTED.toString()) - .location(loc).tableExists(true); - tablets.add(factory.build()); - factory = new ListTabletsCommand.TabletRowInfo.Factory(tableName, ke3).dir("t-dir3") - .numFiles(3).numWalLogs(3).numEntries(3).size(300).status(TabletState.HOSTED.toString()) - .location(loc).tableExists(true); - tablets.add(factory.build()); - return tablets; - } } @Test public void mockTest() throws Exception { ListTabletsCommand cmd = new TestListTabletsCommand(); + TableId tableId = TableId.of("123"); + + TServerInstance ser1 = new TServerInstance(HostAndPort.fromParts("server1", 8555), "s001"); + TServerInstance ser2 = new TServerInstance(HostAndPort.fromParts("server2", 2354), "s002"); + + StoredTabletFile sf11 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-dir1/sf11.rf"); + DataFileValue dfv11 = new DataFileValue(5643, 89); + + StoredTabletFile sf12 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-dir1/sf12.rf"); + DataFileValue dfv12 = new DataFileValue(379963, 1027); + + StoredTabletFile sf21 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-dir2/sf21.rf"); + DataFileValue dfv21 = new DataFileValue(5323, 142); + + StoredTabletFile sf31 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-dir3/sf31.rf"); + DataFileValue dfv31 = new DataFileValue(95832L, 231); + + KeyExtent extent = new KeyExtent(tableId, new Text("d"), null); + + LogEntry le1 = new LogEntry(extent, 55, "lf1"); + LogEntry le2 = new LogEntry(extent, 57, "lf2"); + + TabletMetadata tm1 = TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.ONDEMAND) + .putLocation(TabletMetadata.Location.current(ser1)).putFile(sf11, dfv11) + .putFile(sf12, dfv12).putWal(le1).putDirName("t-dir1").build(); + + extent = new KeyExtent(tableId, new Text("k"), new Text("e")); + TabletMetadata tm2 = TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.ALWAYS) + .putLocation(TabletMetadata.Location.current(ser2)).putFile(sf21, dfv21) + .putDirName("t-dir2").build(LOGS); + + extent = new KeyExtent(tableId, null, new Text("l")); + TabletMetadata tm3 = TabletMetadata.builder(extent).putHostingGoal(TabletHostingGoal.NEVER) + .putFile(sf31, dfv31).putWal(le1).putWal(le2).putDirName("t-dir3").build(LOCATION); + + TabletInformationImpl[] tabletInformation = new TabletInformationImpl[3]; + tabletInformation[0] = new TabletInformationImpl(tm1, "HOSTED"); + tabletInformation[1] = new TabletInformationImpl(tm2, "HOSTED"); + tabletInformation[2] = new TabletInformationImpl(tm3, "UNASSIGNED"); + AccumuloClient client = EasyMock.createMock(AccumuloClient.class); ClientContext context = EasyMock.createMock(ClientContext.class); TableOperations tableOps = EasyMock.createMock(TableOperations.class); @@ -120,7 +172,6 @@ public class ListTabletsCommandTest { Shell shellState = EasyMock.createMock(Shell.class); Options opts = cmd.getOptions(); - CommandLineParser parser = new DefaultParser(); String[] args = {"-t", tableName}; CommandLine cli = parser.parse(opts, args); @@ -128,65 +179,17 @@ public class ListTabletsCommandTest { EasyMock.expect(shellState.getAccumuloClient()).andReturn(client).anyTimes(); EasyMock.expect(shellState.getContext()).andReturn(context).anyTimes(); EasyMock.expect(client.tableOperations()).andReturn(tableOps).anyTimes(); + EasyMock.expect(context.tableOperations()).andReturn(tableOps).anyTimes(); + EasyMock.expect(tableOps.getTabletInformation(tableName, new Range())) + .andReturn(Stream.of(tabletInformation)); Map<String,String> idMap = new TreeMap<>(); idMap.put(tableName, tableId.canonical()); EasyMock.expect(tableOps.tableIdMap()).andReturn(idMap); - assertEquals(rows.size(), 3, "Incorrect number of rows: " + rows); - EasyMock.replay(client, context, tableOps, instOps, shellState); - cmd.execute("listTablets -t " + tableName, cli, shellState); + cmd.execute("listtablets -t " + tableName, cli, shellState); EasyMock.verify(client, context, tableOps, instOps, shellState); } - @Test - public void defaultBuilderTest() { - TableId id = TableId.of("123"); - Text startRow = new Text("a"); - Text endRow = new Text("z"); - ListTabletsCommand.TabletRowInfo.Factory factory = - new ListTabletsCommand.TabletRowInfo.Factory("aName", new KeyExtent(id, endRow, startRow)); - - ListTabletsCommand.TabletRowInfo info = factory.build(); - - assertEquals("aName", info.tableName); - assertEquals(id, info.tableId); - assertEquals("a z", info.getTablet()); - assertEquals(0, info.numFiles); - assertEquals(0, info.numWalLogs); - assertEquals(0, info.numEntries); - assertEquals(0, info.size); - assertEquals("", info.status); - assertEquals("", info.location); - assertFalse(info.tableExists); - } - - @Test - public void builderTest() { - TableId id = TableId.of("123"); - Text startRow = new Text("a"); - Text endRow = new Text("z"); - KeyExtent ke = new KeyExtent(id, endRow, startRow); - Location loc = Location.current("localhost", ""); - ListTabletsCommand.TabletRowInfo.Factory factory = - new ListTabletsCommand.TabletRowInfo.Factory("aName", ke).numFiles(1).numWalLogs(2) - .numEntries(3).size(4).status(TabletState.HOSTED.toString()).location(loc) - .tableExists(true); - - ListTabletsCommand.TabletRowInfo info = factory.build(); - - assertEquals("aName", info.tableName); - assertEquals(1, info.numFiles); - assertEquals(2, info.numWalLogs); - assertEquals("3", info.getNumEntries(false)); - assertEquals(3, info.numEntries); - assertEquals("4", info.getSize(false)); - assertEquals(4, info.size); - assertEquals("HOSTED", info.status); - assertEquals("CURRENT:localhost", info.location); - assertEquals(TableId.of("123"), info.tableId); - assertEquals(startRow + " " + endRow, info.getTablet()); - assertTrue(info.tableExists); - } } diff --git a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java index caef8145e7..0d239eadcf 100644 --- a/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java @@ -55,6 +55,7 @@ import org.apache.accumulo.core.client.admin.HostingGoalForTablet; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.admin.TabletHostingGoal; +import org.apache.accumulo.core.client.admin.TabletInformation; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; @@ -427,15 +428,15 @@ public class TableOperationsIT extends AccumuloClusterHarness { List<HostingGoalForTablet> expectedGoals = new ArrayList<>(); setExpectedGoal(expectedGoals, idMap.get(tableOnDemand), null, null, TabletHostingGoal.ONDEMAND); - verifyTabletGoals(tableOnDemand, null, null, expectedGoals); + verifyTabletGoals(tableOnDemand, new Range(), expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, idMap.get(tableAlways), null, null, TabletHostingGoal.ALWAYS); - verifyTabletGoals(tableAlways, null, null, expectedGoals); + verifyTabletGoals(tableAlways, new Range(), expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, idMap.get(tableNever), null, null, TabletHostingGoal.NEVER); - verifyTabletGoals(tableNever, null, null, expectedGoals); + verifyTabletGoals(tableNever, new Range(), expectedGoals); verifyTablesWithSplits(tableOnDemandWithSplits, idMap, splits, TabletHostingGoal.ONDEMAND); verifyTablesWithSplits(tableAlwaysWithSplits, idMap, splits, TabletHostingGoal.ALWAYS); @@ -479,9 +480,6 @@ public class TableOperationsIT extends AccumuloClusterHarness { accumuloClient.tableOperations().setTabletHostingGoal(tableName, range, TabletHostingGoal.NEVER); - List<HostingGoalForTablet> hostingInfo = accumuloClient.tableOperations() - .getTabletHostingGoal(tableName, new Range()).collect(Collectors.toList()); - Map<String,String> idMap = accumuloClient.tableOperations().tableIdMap(); expectedGoals = new ArrayList<>(); String tableId = idMap.get(tableName); @@ -491,8 +489,7 @@ public class TableOperationsIT extends AccumuloClusterHarness { setExpectedGoal(expectedGoals, tableId, "m", "d", TabletHostingGoal.ONDEMAND); setExpectedGoal(expectedGoals, tableId, "s", "m", TabletHostingGoal.ALWAYS); setExpectedGoal(expectedGoals, tableId, null, "s", TabletHostingGoal.NEVER); - assertEquals(4, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, new Range(), expectedGoals); } finally { accumuloClient.tableOperations().delete(tableName); } @@ -508,7 +505,6 @@ public class TableOperationsIT extends AccumuloClusterHarness { SortedSet<Text> splits = Sets.newTreeSet(Arrays.asList(new Text("d"), new Text("m"), new Text("s"))); List<HostingGoalForTablet> expectedGoals = new ArrayList<>(); - List<HostingGoalForTablet> hostingInfo; Map<String,String> idMap; String tableId; @@ -530,26 +526,18 @@ public class TableOperationsIT extends AccumuloClusterHarness { setExpectedGoal(expectedGoals, tableId, "d", null, TabletHostingGoal.ALWAYS); // test using row as range constructor - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, new Range("a")) - .collect(Collectors.toList()); - assertEquals(1, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, new Range("a"), expectedGoals); // test using startRowInclusive set to true Range range = new Range(new Text("c"), true, new Text("c"), true); - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, range) - .collect(Collectors.toList()); - assertEquals(1, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, range, expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, "m", "d", TabletHostingGoal.NEVER); setExpectedGoal(expectedGoals, tableId, "s", "m", TabletHostingGoal.ALWAYS); range = new Range(new Text("m"), new Text("p")); - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, range) - .collect(Collectors.toList()); - assertEquals(expectedGoals, hostingInfo); + verifyTabletGoals(tableName, range, expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, "d", null, TabletHostingGoal.ALWAYS); @@ -558,10 +546,7 @@ public class TableOperationsIT extends AccumuloClusterHarness { setExpectedGoal(expectedGoals, tableId, null, "s", TabletHostingGoal.ONDEMAND); range = new Range("b", false, "t", true); - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, range) - .collect(Collectors.toList()); - assertEquals(4, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, range, expectedGoals); } finally { accumuloClient.tableOperations().delete(tableName); @@ -575,7 +560,6 @@ public class TableOperationsIT extends AccumuloClusterHarness { @Test public void testGetHostingGoals_DelayedSplits() throws AccumuloException, TableExistsException, AccumuloSecurityException, TableNotFoundException { - String tableName = getUniqueNames(1)[0]; try { @@ -589,10 +573,7 @@ public class TableOperationsIT extends AccumuloClusterHarness { List<HostingGoalForTablet> expectedGoals = new ArrayList<>(); String tableId = idMap.get(tableName); setExpectedGoal(expectedGoals, tableId, null, null, TabletHostingGoal.ALWAYS); - List<HostingGoalForTablet> hostingInfo = accumuloClient.tableOperations() - .getTabletHostingGoal(tableName, new Range()).collect(Collectors.toList()); - assertEquals(1, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, new Range(), expectedGoals); // Add splits after the fact SortedSet<Text> splits = @@ -600,18 +581,11 @@ public class TableOperationsIT extends AccumuloClusterHarness { accumuloClient.tableOperations().addSplits(tableName, splits); expectedGoals.clear(); - hostingInfo.clear(); setExpectedGoal(expectedGoals, tableId, "g", null, TabletHostingGoal.ALWAYS); setExpectedGoal(expectedGoals, tableId, "n", "g", TabletHostingGoal.ALWAYS); setExpectedGoal(expectedGoals, tableId, "r", "n", TabletHostingGoal.ALWAYS); setExpectedGoal(expectedGoals, tableId, null, "r", TabletHostingGoal.ALWAYS); - - // Retrieve goals for table - hostingInfo.clear(); - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, new Range()) - .collect(Collectors.toList()); - assertEquals(4, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, new Range(), expectedGoals); } finally { accumuloClient.tableOperations().delete(tableName); } @@ -623,7 +597,7 @@ public class TableOperationsIT extends AccumuloClusterHarness { // which they wre split. Steps are as follows: // - create table // - add two splits; leave first tablet to default ONDEMAND, seconds to NEVER, third to ALWAYS - // - add an additional split within each of the three existing tablets + // - add a split within each of the three existing tablets // - verify the newly created tablets are set with the hostingGoals of the tablet from which they // are split. @Test @@ -651,11 +625,7 @@ public class TableOperationsIT extends AccumuloClusterHarness { setExpectedGoal(expectedGoals, tableId, "h", null, TabletHostingGoal.ONDEMAND); setExpectedGoal(expectedGoals, tableId, "q", "h", TabletHostingGoal.NEVER); setExpectedGoal(expectedGoals, tableId, null, "q", TabletHostingGoal.ALWAYS); - - List<HostingGoalForTablet> hostingInfo = accumuloClient.tableOperations() - .getTabletHostingGoal(tableName, new Range()).collect(Collectors.toList()); - - assertEquals(expectedGoals, hostingInfo); + verifyTabletGoals(tableName, new Range(), expectedGoals); // Add a split within each of the existing tablets. Adding 'd', 'm', and 'v' splits = Sets.newTreeSet(Arrays.asList(new Text("d"), new Text("m"), new Text("v"))); @@ -663,18 +633,13 @@ public class TableOperationsIT extends AccumuloClusterHarness { // verify results expectedGoals.clear(); - hostingInfo.clear(); setExpectedGoal(expectedGoals, tableId, "d", null, TabletHostingGoal.ONDEMAND); setExpectedGoal(expectedGoals, tableId, "h", "d", TabletHostingGoal.ONDEMAND); setExpectedGoal(expectedGoals, tableId, "m", "h", TabletHostingGoal.NEVER); setExpectedGoal(expectedGoals, tableId, "q", "m", TabletHostingGoal.NEVER); setExpectedGoal(expectedGoals, tableId, "v", "q", TabletHostingGoal.ALWAYS); setExpectedGoal(expectedGoals, tableId, null, "v", TabletHostingGoal.ALWAYS); - - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, new Range()) - .collect(Collectors.toList()); - - assertEquals(expectedGoals, hostingInfo); + verifyTabletGoals(tableName, new Range(), expectedGoals); } finally { accumuloClient.tableOperations().delete(tableName); } @@ -684,7 +649,7 @@ public class TableOperationsIT extends AccumuloClusterHarness { SortedSet<Text> splits, TabletHostingGoal goal) throws TableNotFoundException { List<HostingGoalForTablet> expectedGoals = new ArrayList<>(); - List<HostingGoalForTablet> hostingInfo; + List<TabletInformation> tabletInfo; String tableId = idMap.get(tableName); String[] splitPts = splits.stream().map(Text::toString).toArray(String[]::new); @@ -693,43 +658,44 @@ public class TableOperationsIT extends AccumuloClusterHarness { setExpectedGoal(expectedGoals, tableId, splitPts[1], splitPts[0], goal); setExpectedGoal(expectedGoals, tableId, splitPts[2], splitPts[1], goal); setExpectedGoal(expectedGoals, tableId, null, splitPts[2], goal); - // Retrieve goals for table - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, new Range()) - .collect(Collectors.toList()); - assertEquals(4, hostingInfo.size()); - hostingInfo.forEach(p -> assertTrue(expectedGoals.contains(p))); + verifyTabletGoals(tableName, new Range(), expectedGoals); // verify individual tablets can be retrieved expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, splitPts[0], null, goal); - verifyTabletGoals(tableName, new Text(splitPts[0]), null, expectedGoals); + verifyTabletGoals(tableName, new Range(null, new Text(splitPts[0])), expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, splitPts[1], splitPts[0], goal); - verifyTabletGoals(tableName, new Text(splitPts[1]), new Text(splitPts[0]), expectedGoals); + verifyTabletGoals(tableName, + new Range(new Text(splitPts[0]), false, new Text(splitPts[1]), true), expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, splitPts[2], splitPts[1], goal); - verifyTabletGoals(tableName, new Text(splitPts[2]), new Text(splitPts[1]), expectedGoals); + verifyTabletGoals(tableName, + new Range(new Text(splitPts[1]), false, new Text(splitPts[2]), true), expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, null, splitPts[2], goal); - verifyTabletGoals(tableName, null, new Text(splitPts[2]), expectedGoals); + verifyTabletGoals(tableName, new Range(new Text(splitPts[2]), false, null, true), + expectedGoals); expectedGoals.clear(); setExpectedGoal(expectedGoals, tableId, splitPts[1], splitPts[0], goal); setExpectedGoal(expectedGoals, tableId, splitPts[2], splitPts[1], goal); - verifyTabletGoals(tableName, new Text(splitPts[2]), new Text(splitPts[0]), expectedGoals); + verifyTabletGoals(tableName, + new Range(new Text(splitPts[0]), false, new Text(splitPts[2]), true), expectedGoals); } - private void verifyTabletGoals(String tableName, Text endRow, Text prevEndRow, + private void verifyTabletGoals(String tableName, Range range, List<HostingGoalForTablet> expectedGoals) throws TableNotFoundException { - - List<HostingGoalForTablet> hostingInfo; - Range range = new Range(prevEndRow, false, endRow, true); - hostingInfo = accumuloClient.tableOperations().getTabletHostingGoal(tableName, range) - .collect(Collectors.toList()); - assertEquals(expectedGoals, hostingInfo); + List<TabletInformation> tabletInfo = accumuloClient.tableOperations() + .getTabletInformation(tableName, range).collect(Collectors.toList()); + assertEquals(expectedGoals.size(), tabletInfo.size()); + for (var i = 0; i < expectedGoals.size(); i++) { + assertEquals(expectedGoals.get(i).getTabletId(), tabletInfo.get(i).getTabletId()); + assertEquals(expectedGoals.get(i).getHostingGoal(), tabletInfo.get(i).getHostingGoal()); + } } private void setExpectedGoal(List<HostingGoalForTablet> expected, String id, String endRow, diff --git a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java index 3b80698f5d..995cdf97df 100644 --- a/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellServerIT.java @@ -2259,6 +2259,85 @@ public class ShellServerIT extends SharedMiniClusterBase { assertMatches(output, "(?sm).*^.*total[:]2[,]\\s+missing[:]0[,]\\s+extra[:]0.*$.*"); } + // This test serves to verify the listtablets command as well as the getTabletInformation api, + // which is used by listtablets. + @Test + public void testListTablets() throws IOException, InterruptedException { + + final var tables = getUniqueNames(2); + final String table1 = tables[0]; + final String table2 = tables[1]; + + ts.exec("createtable " + table1, true); + ts.exec("addsplits g n u", true); + ts.exec("setgoal -g always -r g", true); + ts.exec("setgoal -g always -r u", true); + insertData(table1, 1000, 3); + ts.exec("compact -w -t " + table1); + ts.exec("scan -t " + table1); + + ts.exec("createtable " + table2, true); + ts.exec("addsplits f m t", true); + ts.exec("setgoal -g always -r n", true); + insertData(table2, 500, 5); + ts.exec("compact -t " + table2); + ts.exec("scan -t " + table1); + ts.exec("setgoal -r g -t " + table2 + " -g NEVER"); + + // give tablet time to become unassigned + for (var i = 0; i < 15; i++) { + Thread.sleep(1000); + String goal = ts.exec("listtablets -t " + table2, true, "m NEVER"); + if (goal.contains("UNASSIGNED None")) { + break; + } + } + + String results = ts.exec("listtablets -np -p ShellServerIT_testListTablets.", true); + assertTrue(results.contains("TABLE: ShellServerIT_testListTablets0")); + assertTrue(results.contains("TABLE: ShellServerIT_testListTablets1")); + assertTrue(results.contains("1 -INF g ALWAYS")); + assertTrue(results.contains("1 g n ONDEMAND")); + assertTrue(results.contains("1 n u ALWAYS")); + assertTrue(results.contains("1 u +INF ONDEMAND")); + assertTrue(results.contains("2 -INF f ONDEMAND")); + assertTrue(results.contains("2 f m NEVER")); + assertTrue(results.contains("2 m t ALWAYS")); + assertTrue(results.contains("2 t +INF ONDEMAND")); + + // verify the sum of the tablets sizes, number of entries, and dir name match the data in a + // metadata scan + String metadata = ts.exec("scan -np -t accumulo.metadata -b 1 -c loc,file"); + for (String line : metadata.split("\n")) { + String[] tokens = line.split("\\s+"); + if (tokens[1].startsWith("loc")) { + String loc = tokens[3]; + assertTrue(results.contains(loc)); + } + if (tokens[1].startsWith("file")) { + String[] parts = tokens[1].split("/"); + String dir = parts[parts.length - 2]; + assertTrue(results.contains(dir)); + String[] sizes = tokens[3].split(","); + String size = String.format("%,d", Integer.parseInt(sizes[0])); + String entries = String.format("%,d", Integer.parseInt(sizes[1])); + assertTrue(results.contains(size)); + assertTrue(results.contains(entries)); + } + } + } + + private void insertData(String table, int numEntries, int rowLen) throws IOException { + for (var i = 0; i < numEntries; i++) { + String alphabet = "abcdefghijklmnopqrstuvwxyz"; + String row = String.valueOf(alphabet.charAt(i % 26)) + i; + var cf = "cf" + i; + var cq = "cq" + i; + var data = "asdfqwerty"; + ts.exec("insert -t " + table + " " + row + " " + cf + " " + cq + " " + data, true); + } + } + private java.nio.file.Path createSplitsFile(final String splitsFile, final SortedSet<Text> splits) throws IOException { String fullSplitsFile = System.getProperty("user.dir") + "/target/" + splitsFile;