Author: ecn
Date: Fri Apr 12 16:11:25 2013
New Revision: 1467331

URL: http://svn.apache.org/r1467331
Log:
ACCUMULO-1264 put delete markers for the METADATA table into the root tablet

Modified:
    accumulo/trunk/   (props changed)
    accumulo/trunk/assemble/   (props changed)
    accumulo/trunk/core/   (props changed)
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
    accumulo/trunk/examples/   (props changed)
    accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java   
(props changed)
    
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
   (props changed)
    accumulo/trunk/server/   (props changed)
    
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
    
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
    
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
    
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
    accumulo/trunk/src/   (props changed)

Propchange: accumulo/trunk/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5:r1467312

Propchange: accumulo/trunk/assemble/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/assemble:r1467312

Propchange: accumulo/trunk/core/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/core:r1467312

Modified: 
accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java
URL: 
http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java 
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/Constants.java 
Fri Apr 12 16:11:25 2013
@@ -97,7 +97,9 @@ public class Constants {
   public static final Key METADATA_RESERVED_KEYSPACE_STOP_KEY = new Key(new 
Text(new byte[] {'~' + 1}));
   public static final Range METADATA_RESERVED_KEYSPACE = new 
Range(METADATA_RESERVED_KEYSPACE_START_KEY, true, 
METADATA_RESERVED_KEYSPACE_STOP_KEY, false);
   public static final String METADATA_DELETE_FLAG_PREFIX = "~del";
+  public static final String METADATA_DELETE_FLAG_FOR_METADATA_PREFIX = "!!" + 
METADATA_DELETE_FLAG_PREFIX;
   public static final Range METADATA_DELETES_KEYSPACE = new Range(new Key(new 
Text(METADATA_DELETE_FLAG_PREFIX)), true, new Key(new Text("~dem")), false);
+  public static final Range METADATA_DELETES_FOR_METADATA_KEYSPACE = new 
Range(new Key(new Text(METADATA_DELETE_FLAG_FOR_METADATA_PREFIX)), true, new 
Key(new Text("!!~dem")), false);
   public static final String METADATA_BLIP_FLAG_PREFIX = "~blip"; // BLIP = 
bulk load in progress
   public static final Range METADATA_BLIP_KEYSPACE = new Range(new Key(new 
Text(METADATA_BLIP_FLAG_PREFIX)), true, new Key(new Text("~bliq")), false);
   

Propchange: accumulo/trunk/examples/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/examples:r1467312

Propchange: 
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java
------------------------------------------------------------------------------
  Merged 
/accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/ZooStore.java:r1467312

Propchange: 
accumulo/trunk/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java
------------------------------------------------------------------------------
  Merged 
/accumulo/branches/1.5/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooSession.java:r1467312

Propchange: accumulo/trunk/server/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/server:r1467312

Modified: 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
URL: 
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
 (original)
+++ 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
 Fri Apr 12 16:11:25 2013
@@ -101,9 +101,11 @@ public class MetadataConstraints impleme
     
     byte[] row = mutation.getRow();
     
-    // always allow rows that fall within reserved area
+    // always allow rows that fall within reserved areas
     if (row.length > 0 && row[0] == '~')
       return null;
+    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
+      return null;
     
     for (byte b : row) {
       if (b == ';') {

Modified: 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL: 
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
 (original)
+++ 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
 Fri Apr 12 16:11:25 2013
@@ -20,8 +20,12 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -438,23 +442,37 @@ public class SimpleGarbageCollector impl
       }
       return candidates;
     }
+
+    checkForBulkProcessingFiles = false;
+    Range range = Constants.METADATA_DELETES_FOR_METADATA_KEYSPACE;
+    
candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX, 
range));
+    if (candidateMemExceeded)
+      return candidates;
     
-    Scanner scanner = instance.getConnector(credentials.getPrincipal(), 
CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME,
 Constants.NO_AUTHS);
+    range = Constants.METADATA_DELETES_KEYSPACE;
+    candidates.addAll(getBatch(Constants.METADATA_DELETE_FLAG_PREFIX, range));
+    return candidates;
+  }
 
+  private Collection<String> getBatch(String prefix, Range range) throws 
Exception {
+    // want to ensure GC makes progress... if the 1st N deletes are stable and 
we keep processing them, 
+    // then will never inspect deletes after N
     if (continueKey != null) {
-      // want to ensure GC makes progress... if the 1st N deletes are stable 
and we keep processing them, then will never inspect deletes after N
-      scanner.setRange(new Range(continueKey, true, 
Constants.METADATA_DELETES_KEYSPACE.getEndKey(), 
Constants.METADATA_DELETES_KEYSPACE.isEndKeyInclusive()));
+      if (!range.contains(continueKey)) {
+        // continue key is for some other range
+        return Collections.emptyList();
+      }
+      range = new Range(continueKey, true, range.getEndKey(), 
range.isEndKeyInclusive());
       continueKey = null;
-    } else {
-      // scan the reserved keyspace for deletes
-      scanner.setRange(Constants.METADATA_DELETES_KEYSPACE);
     }
     
+    Scanner scanner = instance.getConnector(credentials.getPrincipal(), 
CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME,
 Constants.NO_AUTHS);
+    scanner.setRange(range);
+    List<String> result = new ArrayList<String>();
     // find candidates for deletion; chop off the prefix
-    checkForBulkProcessingFiles = false;
     for (Entry<Key,Value> entry : scanner) {
-      String cand = 
entry.getKey().getRow().toString().substring(Constants.METADATA_DELETE_FLAG_PREFIX.length());
-      candidates.add(cand);
+      String cand = 
entry.getKey().getRow().toString().substring(prefix.length());
+      result.add(cand);
       checkForBulkProcessingFiles |= 
cand.toLowerCase(Locale.ENGLISH).contains(Constants.BULK_PREFIX);
       if (almostOutOfMemory()) {
         candidateMemExceeded = true;
@@ -464,7 +482,7 @@ public class SimpleGarbageCollector impl
       }
     }
     
-    return candidates;
+    return result;
   }
   
   static public boolean almostOutOfMemory() {
@@ -567,7 +585,18 @@ public class SimpleGarbageCollector impl
       }
     }
   }
-  
+
+  final static String METADATA_TABLE_DIR = "/" + Constants.METADATA_TABLE_ID;
+  private static Mutation getMarkerDeleteMutation(final String delete) {
+    String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+    if (delete.startsWith(METADATA_TABLE_DIR))
+      prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
+    Mutation m = new Mutation(new Text(prefix + delete));
+    m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+    return m;
+  }
+
   /**
    * This method attempts to do its best to remove files from the filesystem 
that have been confirmed for deletion.
    */
@@ -584,7 +613,6 @@ public class SimpleGarbageCollector impl
         log.error("Unable to create writer to remove file from the !METADATA 
table", e);
       }
     }
-    
     // when deleting a dir and all files in that dir, only need to delete the 
dir
     // the dir will sort right before the files... so remove the files in this 
case
     // to minimize namenode ops
@@ -597,10 +625,8 @@ public class SimpleGarbageCollector impl
       } else if (lastDir != null) {
         if (delete.startsWith(lastDir)) {
           log.debug("Ignoring " + delete + " because " + lastDir + " exist");
-          Mutation m = new Mutation(new 
Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete));
-          m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
           try {
-            writer.addMutation(m);
+            writer.addMutation(getMarkerDeleteMutation(delete));
           } catch (MutationsRejectedException e) {
             throw new RuntimeException(e);
           }
@@ -623,10 +649,11 @@ public class SimpleGarbageCollector impl
         public void run() {
           boolean removeFlag;
           
-          log.debug("Deleting " + ServerConstants.getTablesDir() + delete);
+          String fullPath = ServerConstants.getTablesDir() + delete;
+          log.debug("Deleting " + fullPath);
           try {
             
-            Path p = new Path(ServerConstants.getTablesDir() + delete);
+            Path p = new Path(fullPath);
             
             if (moveToTrash(p) || fs.delete(p, true)) {
               // delete succeeded, still want to delete
@@ -668,15 +695,14 @@ public class SimpleGarbageCollector impl
             // proceed to clearing out the flags for successful deletes and
             // non-existent files
             if (removeFlag && finalWriter != null) {
-              Mutation m = new Mutation(new 
Text(Constants.METADATA_DELETE_FLAG_PREFIX + delete));
-              m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
-              finalWriter.addMutation(m);
+              finalWriter.addMutation(getMarkerDeleteMutation(delete));
             }
           } catch (Exception e) {
             log.error(e, e);
           }
           
         }
+
       };
       
       deleteThreadPool.execute(deleteTask);

Modified: 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: 
http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
 (original)
+++ 
accumulo/trunk/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
 Fri Apr 12 16:11:25 2013
@@ -479,11 +479,15 @@ public class MetadataTable extends org.a
   
   public static Mutation createDeleteMutation(String tableId, String 
pathToRemove) {
     Mutation delFlag;
+    String prefix = Constants.METADATA_DELETE_FLAG_PREFIX;
+    if (tableId.equals(Constants.METADATA_TABLE_ID))
+      prefix = Constants.METADATA_DELETE_FLAG_FOR_METADATA_PREFIX;
+
     if (pathToRemove.startsWith("../"))
-      delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + 
pathToRemove.substring(2)));
+      delFlag = new Mutation(new Text(prefix + pathToRemove.substring(2)));
     else
-      delFlag = new Mutation(new Text(Constants.METADATA_DELETE_FLAG_PREFIX + 
"/" + tableId + pathToRemove));
-    
+      delFlag = new Mutation(new Text(prefix + "/" + tableId + pathToRemove));
+
     delFlag.put(EMPTY_TEXT, EMPTY_TEXT, new Value(new byte[] {}));
     return delFlag;
   }

Modified: 
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
URL: 
http://svn.apache.org/viewvc/accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java?rev=1467331&r1=1467330&r2=1467331&view=diff
==============================================================================
--- 
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
 (original)
+++ 
accumulo/trunk/server/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
 Fri Apr 12 16:11:25 2013
@@ -21,7 +21,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.TreeMap;
 
-import junit.framework.Assert;
+import org.junit.Assert;
 
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;

Propchange: accumulo/trunk/src/
------------------------------------------------------------------------------
  Merged /accumulo/branches/1.5/src:r1467312


Reply via email to