Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java?rev=1493756&r1=1493755&r2=1493756&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java Mon Jun 17 13:26:43 2013 @@ -66,7 +66,7 @@ public class RemoveEntriesForMissingFile for (Entry<Key,Value> entry : metadata) { count++; Key key = entry.getKey(); - Path map = new Path(fs.getFullPath(key)); + Path map = fs.getFullPath(key); if (!fs.exists(map)) { missing++; log.info("File " + map + " is missing");
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java?rev=1493756&r1=1493755&r2=1493756&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java Mon Jun 17 13:26:43 2013 @@ -16,24 +16,262 @@ */ package org.apache.accumulo.server.util; +import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; -import org.apache.accumulo.server.cli.ClientOpts; +import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.util.NumUtil; +import org.apache.accumulo.server.ServerConstants; +import org.apache.accumulo.server.cli.ClientOpts; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.log4j.Logger; import com.beust.jcommander.Parameter; public class TableDiskUsage { + + private static final Logger log = Logger.getLogger(Logger.class); + private int nextInternalId = 0; + private Map<String,Integer> internalIds = new HashMap<String,Integer>(); + private Map<Integer,String> externalIds = new HashMap<Integer,String>(); + private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>(); + private Map<String,Long> fileSizes = new HashMap<String,Long>(); + + void addTable(String tableId) { + if (internalIds.containsKey(tableId)) + throw new IllegalArgumentException("Already added table " + tableId); + + int iid = nextInternalId++; + + internalIds.put(tableId, iid); + externalIds.put(iid, tableId); + } + + void linkFileAndTable(String tableId, String file) { + int internalId = internalIds.get(tableId); + + Integer[] tables = tableFiles.get(file); + if (tables == null) { + tables = new Integer[internalIds.size()]; + for (int i = 0; i < tables.length; i++) + tables[i] = 0; + tableFiles.put(file, tables); + } + + tables[internalId] = 1; + } + + void addFileSize(String file, long size) { + fileSizes.put(file, size); + } + + Map<List<String>,Long> calculateUsage() { + + Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>(); + + for (Entry<String,Integer[]> entry : tableFiles.entrySet()) { + log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey())); + List<Integer> key = Arrays.asList(entry.getValue()); + Long size = fileSizes.get(entry.getKey()); + + Long tablesUsage = usage.get(key); + if (tablesUsage == null) + tablesUsage = 0l; + + tablesUsage += size; + + usage.put(key, tablesUsage); + + } + + Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>(); + + for (Entry<List<Integer>,Long> entry : usage.entrySet()) { + List<String> externalKey = new ArrayList<String>(); + List<Integer> key = entry.getKey(); + for (int i = 0; i < key.size(); i++) + if (key.get(i) != 0) + externalKey.add(externalIds.get(i)); + + externalUsage.put(externalKey, entry.getValue()); + } + + return externalUsage; + } + + public interface Printer { + void print(String line); + } + + public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, FileSystem fs, Connector conn, boolean humanReadable) + throws TableNotFoundException, + IOException { + printDiskUsage(acuConf, tables, fs, conn, new Printer() { + @Override + public void print(String line) { + System.out.println(line); + } + }, humanReadable); + } + + public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, FileSystem fs, Connector conn) + throws IOException { + TableDiskUsage tdu = new TableDiskUsage(); + + for (String tableId : tableIds) + tdu.addTable(tableId); + + HashSet<String> tablesReferenced = new HashSet<String>(tableIds); + HashSet<String> emptyTableIds = new HashSet<String>(); + + for (String tableId : tableIds) { + Scanner mdScanner = null; + try { + mdScanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); + } catch (TableNotFoundException e) { + throw new RuntimeException(e); + } + mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY); + mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange()); + + if (!mdScanner.iterator().hasNext()) { + emptyTableIds.add(tableId); + } + + for (Entry<Key,Value> entry : mdScanner) { + String file = entry.getKey().getColumnQualifier().toString(); + String parts[] = file.split("/"); + String uniqueName = parts[parts.length - 1]; + if (file.contains(":") || file.startsWith("../")) { + String ref = parts[parts.length - 3]; + if (!ref.equals(tableId)) { + tablesReferenced.add(ref); + } + } + + tdu.linkFileAndTable(tableId, uniqueName); + } + } + + for (String tableId : tablesReferenced) { + for (String tableDir : ServerConstants.getTablesDirs()) { + FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*")); + if (files != null) { + for (FileStatus fileStatus : files) { + // Assumes that all filenames are unique + String name = fileStatus.getPath().getName(); + tdu.addFileSize(name, fileStatus.getLen()); + } + } + } + } + + HashMap<String,String> reverseTableIdMap = new HashMap<String,String>(); + for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet()) + reverseTableIdMap.put(entry.getValue(), entry.getKey()); + + TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() { + + @Override + public int compare(TreeSet<String> o1, TreeSet<String> o2) { + int len1 = o1.size(); + int len2 = o2.size(); + + int min = Math.min(len1, len2); + + Iterator<String> iter1 = o1.iterator(); + Iterator<String> iter2 = o2.iterator(); + + int count = 0; + + while (count < min) { + String s1 = iter1.next(); + String s2 = iter2.next(); + + int cmp = s1.compareTo(s2); + + if (cmp != 0) + return cmp; + + count++; + } + + return len1 - len2; + } + }); + + for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) { + TreeSet<String> tableNames = new TreeSet<String>(); + for (String tableId : entry.getKey()) + tableNames.add(reverseTableIdMap.get(tableId)); + + usage.put(tableNames, entry.getValue()); + } + + if (!emptyTableIds.isEmpty()) { + TreeSet<String> emptyTables = new TreeSet<String>(); + for (String tableId : emptyTableIds) { + emptyTables.add(reverseTableIdMap.get(tableId)); + } + usage.put(emptyTables, 0L); + } + + return usage; + } + + public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, FileSystem fs, Connector conn, Printer printer, boolean humanReadable) + throws TableNotFoundException, IOException { + + HashSet<String> tableIds = new HashSet<String>(); + + for (String tableName : tables) { + String tableId = conn.tableOperations().tableIdMap().get(tableName); + if (tableId == null) + throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found"); + + tableIds.add(tableId); + } + + Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn); + + String valueFormat = humanReadable ? "%9s" : "%,24d"; + for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) { + Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue(); + printer.print(String.format(valueFormat + " %s", value, entry.getKey())); + } + } + + static class Opts extends ClientOpts { @Parameter(description=" <table> { <table> ... } ") List<String> tables = new ArrayList<String>(); } - + /** * @param args */ @@ -42,7 +280,7 @@ public class TableDiskUsage { Opts opts = new Opts(); opts.parseArgs(TableDiskUsage.class.getName(), args); Connector conn = opts.getConnector(); - org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false); + org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false); } } Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java Mon Jun 17 13:26:43 2013 @@ -95,7 +95,7 @@ public class MultiReaderTest { @Test public void testMultiReader() throws IOException { - String manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps").toString(); + Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps"); MultiReader reader = new MultiReader(fs, manyMaps); IntWritable key = new IntWritable(); BytesWritable value = new BytesWritable(); Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java (original) +++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java Mon Jun 17 13:26:43 2013 @@ -118,7 +118,7 @@ public class SortedLogRecoveryTest { final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir"; FileSystem fs = FileSystemImpl.getLocal(); fs.deleteRecursively(new Path(workdir)); - ArrayList<String> dirs = new ArrayList<String>(); + ArrayList<Path> dirs = new ArrayList<Path>(); try { for (Entry<String,KeyValue[]> entry : logs.entrySet()) { String path = workdir + "/" + entry.getKey(); @@ -129,7 +129,7 @@ public class SortedLogRecoveryTest { } map.close(); ns.create(new Path(path, "finished")).close(); - dirs.add(path); + dirs.add(new Path(path)); } // Recover SortedLogRecovery recovery = new SortedLogRecovery(fs); Modified: accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff ============================================================================== --- accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java (original) +++ accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java Mon Jun 17 13:26:43 2013 @@ -43,8 +43,11 @@ import org.apache.accumulo.fate.zookeepe import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason; import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher; import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy; +import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.client.HdfsZooInstance; import org.apache.accumulo.server.fs.FileRef; +import org.apache.accumulo.server.fs.FileSystem; +import org.apache.accumulo.server.fs.FileSystemImpl; import org.apache.accumulo.server.master.state.Assignment; import org.apache.accumulo.server.master.state.TServerInstance; import org.apache.accumulo.server.security.SecurityConstants; @@ -134,7 +137,7 @@ public class SplitRecoveryTest extends F for (int i = 0; i < extents.length; i++) { KeyExtent extent = extents[i]; - String tdir = "/dir_" + i; + String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString() + "/dir_" + i; MetadataTable.addTablet(extent, tdir, SecurityConstants.getSystemCredentials(), TabletTime.LOGICAL_TIME_ID, zl); SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>(); mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i));
