Repository: accumulo Updated Branches: refs/heads/master 52ba24e32 -> 39f14472b
ACCUMULO-4021 explicitly return a value that marks the end of GC Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/deef5352 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/deef5352 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/deef5352 Branch: refs/heads/master Commit: deef5352db0bec4b1578568c3da84220c43038a0 Parents: 5ca779a Author: Eric C. Newton <eric.new...@gmail.com> Authored: Tue Oct 20 16:14:35 2015 -0400 Committer: Eric C. Newton <eric.new...@gmail.com> Committed: Tue Oct 20 16:14:35 2015 -0400 ---------------------------------------------------------------------- .../accumulo/gc/GarbageCollectionAlgorithm.java | 16 +++++++++------- .../accumulo/gc/GarbageCollectionEnvironment.java | 6 ++++-- .../apache/accumulo/gc/SimpleGarbageCollector.java | 10 ++++------ .../apache/accumulo/gc/GarbageCollectionTest.java | 5 ++--- 4 files changed, 19 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java index 0e016d5..a8cde31 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java @@ -113,7 +113,7 @@ public class GarbageCollectionAlgorithm { try { relPath = makeRelative(candidate, 0); } catch (IllegalArgumentException iae) { - log.warn("Ingoring invalid deletion candidate " + candidate); + log.warn("Ignoring invalid deletion candidate " + candidate); continue; } ret.put(relPath, candidate); @@ -230,16 +230,15 @@ public class GarbageCollectionAlgorithm { } - private List<String> getCandidates(GarbageCollectionEnvironment gce, String lastCandidate) throws TableNotFoundException, AccumuloException, + private boolean getCandidates(GarbageCollectionEnvironment gce, String lastCandidate, List<String> candidates) + throws TableNotFoundException, AccumuloException, AccumuloSecurityException { Span candidatesSpan = Trace.start("getCandidates"); - List<String> candidates; try { - candidates = gce.getCandidates(lastCandidate); + return gce.getCandidates(lastCandidate, candidates); } finally { candidatesSpan.stop(); } - return candidates; } private void confirmDeletesTrace(GarbageCollectionEnvironment gce, SortedMap<String,String> candidateMap) throws TableNotFoundException, AccumuloException, @@ -268,8 +267,11 @@ public class GarbageCollectionAlgorithm { String lastCandidate = ""; - while (true) { - List<String> candidates = getCandidates(gce, lastCandidate); + boolean outOfMemory = true; + while (outOfMemory) { + List<String> candidates = new ArrayList<String>(); + + outOfMemory = getCandidates(gce, lastCandidate, candidates); if (candidates.size() == 0) break; http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java index cca2411..4eab25a 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionEnvironment.java @@ -45,9 +45,11 @@ public interface GarbageCollectionEnvironment { * @param continuePoint * A row to resume from if a previous invocation was stopped due to finding an extremely large number of candidates to remove which would have * exceeded memory limitations - * @return A collection of candidates files for deletion, may not be the complete collection of files for deletion at this point in time + * @param candidates + * A collection of candidates files for deletion, may not be the complete collection of files for deletion at this point in time + * @return true if the results are short due to insufficient memory, otherwise false */ - List<String> getCandidates(String continuePoint) throws TableNotFoundException, AccumuloException, AccumuloSecurityException; + boolean getCandidates(String continuePoint, List<String> candidates) throws TableNotFoundException, AccumuloException, AccumuloSecurityException; /** * Fetch a list of paths for all bulk loads in progress (blip) from a given table, {@link RootTable#NAME} or {@link MetadataTable#NAME} http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 8761480..3ab95a7 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -19,7 +19,6 @@ package org.apache.accumulo.gc; import java.io.FileNotFoundException; import java.io.IOException; import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -269,7 +268,7 @@ public class SimpleGarbageCollector implements Iface { } @Override - public List<String> getCandidates(String continuePoint) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + public boolean getCandidates(String continuePoint, List<String> result) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { // want to ensure GC makes progress... if the 1st N deletes are stable and we keep processing them, // then will never inspect deletes after N Range range = MetadataSchema.DeletesSection.getRange(); @@ -280,19 +279,18 @@ public class SimpleGarbageCollector implements Iface { Scanner scanner = instance.getConnector(credentials.getPrincipal(), credentials.getToken()).createScanner(tableName, Authorizations.EMPTY); scanner.setRange(range); - List<String> result = new ArrayList<String>(); + result.clear(); // find candidates for deletion; chop off the prefix for (Entry<Key,Value> entry : scanner) { String cand = entry.getKey().getRow().toString().substring(MetadataSchema.DeletesSection.getRowPrefix().length()); result.add(cand); if (almostOutOfMemory(Runtime.getRuntime())) { log.info("List of delete candidates has exceeded the memory threshold. Attempting to delete what has been gathered so far."); - break; + return true; } } - return result; - + return false; } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/deef5352/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java ---------------------------------------------------------------------- diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java index 50c3957..2953d07 100644 --- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java @@ -51,14 +51,13 @@ public class GarbageCollectionTest { ArrayList<String> tablesDirsToDelete = new ArrayList<String>(); @Override - public List<String> getCandidates(String continuePoint) { + public boolean getCandidates(String continuePoint, List<String> ret) { Iterator<String> iter = candidates.tailSet(continuePoint, false).iterator(); - ArrayList<String> ret = new ArrayList<String>(); while (iter.hasNext() && ret.size() < 3) { ret.add(iter.next()); } - return ret; + return ret.size() == 3; } @Override