Author: ecn Date: Fri Apr 12 15:34:12 2013 New Revision: 1467312 URL: http://svn.apache.org/r1467312 Log: ACCUMULO-1264 put delete markers for the METADATA table into the root tablet
Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1467312&r1=1467311&r2=1467312&view=diff ============================================================================== --- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java (original) +++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/Constants.java Fri Apr 12 15:34:12 2013 @@ -97,7 +97,9 @@ public class Constants { public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new Text(new byte[] {'~' + 1})); public static final Range METADATA_RESERVED_KEYSPACE = new Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, METADATA_RESERVED_KEYSPACE_STOP_KEY, false); public static final String METADATA_DELETE_FLAG_PREFIX = "~del"; + public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + METADATA_DELETE_FLAG_PREFIX; public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false); + public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new Key(new Text("!!~dem")), false); public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = bulk load in progress public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false); Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1467312&r1=1467311&r2=1467312&view=diff ============================================================================== --- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java (original) +++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java Fri Apr 12 15:34:12 2013 @@ -101,9 +101,11 @@ public class MetadataConstraints impleme byte[] row = mutation.getRow(); - // always allow rows that fall within reserved area + // always allow rows that fall within reserved areas if (row.length > 0 && row[0] == '~') return null; + if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~') + return null; for (byte b : row) { if (b == ';') { Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1467312&r1=1467311&r2=1467312&view=diff ============================================================================== --- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java (original) +++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java Fri Apr 12 15:34:12 2013 @@ -20,8 +20,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; @@ -438,23 +442,37 @@ public class SimpleGarbageCollector impl } return candidates; } + + checkForBulkProcessingFiles = false; + Range range = Constants.METADATA_DELETES_FOR_METADATA_KEYSPACE; + candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX, range)); + if (candidateMemExceeded) + return candidates; - Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); + range = Constants.METADATA_DELETES_KEYSPACE; + candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_PREFIX, range)); + return candidates; + } + private Collection<String> getBatch(String prefix, Range range) throws Exception { + // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them, + // then will never inspect deletes after N if (continueKey != null) { - // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them, then will never inspect deletes after N - scanner.setRange(new Range(continueKey, true, Constants.METADATA_DELETES_KEYSPACE.getEndKey(), Constants.METADATA_DELETES_KEYSPACE.isEndKeyInclusive())); + if (!range.contains(continueKey)) { + // continue key is for some other range + return Collections.emptyList(); + } + range = new Range(continueKey, true, range.getEndKey(), range.isEndKeyInclusive()); continueKey = null; - } else { - // scan the reserved keyspace for deletes - scanner.setRange(Constants.METADATA_DELETES_KEYSPACE); } + Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS); + scanner.setRange(range); + List<String> result = new ArrayList<String>(); // find candidates for deletion; chop off the prefix - checkForBulkProcessingFiles = false; for (Entry<Key,Value> entry : scanner) { - String cand = entry.getKey().getRow().toString().substring(Constants.METADATA_DELETE_FLAG_PREFIX.length()); - candidates.add(cand); + String cand = entry.getKey().getRow().toString().substring(prefix.length()); + result.add(cand); checkForBulkProcessingFiles |= cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX); if (almostOutOfMemory()) { candidateMemExceeded = true; @@ -464,7 +482,7 @@ public class SimpleGarbageCollector impl } } - return candidates; + return result; } static public boolean almostOutOfMemory() { @@ -567,7 +585,18 @@ public class SimpleGarbageCollector impl } } } - + + final static String METADATA_TABLE_DIR = "/" + Constants.METADATA_TABLE_ID; + private static Mutation getMarkerDeleteMutation(final String delete) { + String prefix = Constants.METADATA_DELETE_FLAG_PREFIX; + if (delete.startsWith(METADATA_TABLE_DIR)) + prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX; + + Mutation m = new Mutation(new Text(prefix + delete)); + m.putDelete(EMPTY_TEXT, EMPTY_TEXT); + return m; + } + /** * This method attempts to do its best to remove files from the filesystem that have been confirmed for deletion. */ @@ -584,7 +613,6 @@ public class SimpleGarbageCollector impl log.error("Unable to create writer to remove file from the !METADATA table", e); } } - // when deleting a dir and all files in that dir, only need to delete the dir // the dir will sort right before the files... so remove the files in this case // to minimize namenode ops @@ -597,10 +625,8 @@ public class SimpleGarbageCollector impl } else if (lastDir != null) { if (delete.startsWith(lastDir)) { log.debug("Ignoring " + delete + " because " + lastDir + " exist"); - Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete)); - m.putDelete(EMPTY_TEXT, EMPTY_TEXT); try { - writer.addMutation(m); + writer.addMutation(getMarkerDeleteMutation(delete)); } catch (MutationsRejectedException e) { throw new RuntimeException(e); } @@ -623,10 +649,11 @@ public class SimpleGarbageCollector impl public void run() { boolean removeFlag; - log.debug("Deleting " + ServerConstants.getTablesDir() + delete); + String fullPath = ServerConstants.getTablesDir() + delete; + log.debug("Deleting " + fullPath); try { - Path p = new Path(ServerConstants.getTablesDir() + delete); + Path p = new Path(fullPath); if (moveToTrash(p) || fs.delete(p, true)) { // delete succeeded, still want to delete @@ -668,15 +695,14 @@ public class SimpleGarbageCollector impl // proceed to clearing out the flags for successful deletes and // non-existent files if (removeFlag && finalWriter != null) { - Mutation m = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete)); - m.putDelete(EMPTY_TEXT, EMPTY_TEXT); - finalWriter.addMutation(m); + finalWriter.addMutation(getMarkerDeleteMutation(delete)); } } catch (Exception e) { log.error(e, e); } } + }; deleteThreadPool.execute(deleteTask); Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1467312&r1=1467311&r2=1467312&view=diff ============================================================================== --- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original) +++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Fri Apr 12 15:34:12 2013 @@ -479,11 +479,15 @@ public class MetadataTable extends org.a public static Mutation createDeleteMutation(String tableId, String pathToRemove) { Mutation delFlag; + String prefix = Constants.METADATA_DELETE_FLAG_PREFIX; + if (tableId.equals(Constants.METADATA_TABLE_ID)) + prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX; + if (pathToRemove.startsWith("../")) - delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + pathToRemove.substring(2))); + delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2))); else - delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + "/" + tableId + pathToRemove)); - + delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove)); + delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {})); return delFlag; } Modified: accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java?rev=1467312&r1=1467311&r2=1467312&view=diff ============================================================================== --- accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java (original) +++ accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java Fri Apr 12 15:34:12 2013 @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.TreeMap; -import junit.framework.Assert; +import org.junit.Assert; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.AccumuloConfiguration;