This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new 073873d Fix #1345 - Allow gc initiated compaction as option. (#1352)
073873d is described below
commit 073873db494fc5467a0b8ededba53d277d41015a
Author: EdColeman <d...@etcoleman.com>
AuthorDate: Thu Sep 12 15:06:06 2019 -0400
Fix #1345 - Allow gc initiated compaction as option. (#1352)
* Fix #1345 - Allow gc initiated compaction as option.
Added a property - gc.use.full.compaction that defaults to current
behaviour. When false,
performs a blocking flush instead.
* fix type in property description.
* wip: use enum, compact, flush or none
* fix test failure by adding PropertyTest
* added timing log messsgae
* update durations to use nanoseconds
---
.../org/apache/accumulo/core/conf/Property.java | 5 ++
.../apache/accumulo/core/conf/PropertyType.java | 3 ++
.../accumulo/core/conf/PropertyTypeTest.java | 6 +++
.../apache/accumulo/gc/SimpleGarbageCollector.java | 60 ++++++++++++++++------
4 files changed, 59 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 494f6b2..5e0dd91 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -556,6 +556,11 @@ public enum Property {
"Archive any files/directories instead of moving to the HDFS trash or
deleting."),
GC_TRACE_PERCENT("gc.trace.percent", "0.01", PropertyType.FRACTION,
"Percent of gc cycles to trace"),
+ GC_USE_FULL_COMPACTION("gc.post.metadata.action", "compact",
PropertyType.GC_POST_ACTION,
+ "When the gc runs it can make a lot of changes to the metadata, on
completion, "
+ + " to force the changes to be written to disk, the metadata and
root tables can be flushed"
+ + " and possibly compacted. Legal values are: compact - which both
flushes and compacts the"
+ + " metadata; flush - which flushes only (compactions may be
triggered if required); or none"),
// properties that are specific to the monitor server behavior
MONITOR_PREFIX("monitor.", null, PropertyType.PREFIX,
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
index a3f0eba..13d403f 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
@@ -112,6 +112,9 @@ public enum PropertyType {
DURABILITY("durability", in(true, null, "none", "log", "flush", "sync"),
"One of 'none', 'log', 'flush' or 'sync'."),
+ GC_POST_ACTION("gc_post_action", in(true, null, "none", "flush", "compact"),
+ "One of 'none', 'flush', or 'compact'."),
+
STRING("string", Predicates.<String>alwaysTrue(),
"An arbitrary string of characters whose format is unspecified and"
+ " interpreted based on the context of the property to which it
applies."),
diff --git
a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
index 3dcf0fd..8505afe 100644
--- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
@@ -160,6 +160,12 @@ public class PropertyTypeTest {
}
@Test
+ public void testTypeGC_POST_ACTION() {
+ valid(null, "none", "flush", "compact");
+ invalid("", "other");
+ }
+
+ @Test
public void testTypeFRACTION() {
valid(null, "1", "0", "1.0", "25%", "2.5%", "10.2E-3", "10.2E-3%", ".3");
invalid("", "other", "20%%", "-0.3", "3.6a", "%25", "3%a");
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 24ce61c..59e0eed 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -44,6 +44,7 @@ import
org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
@@ -183,14 +184,19 @@ public class SimpleGarbageCollector extends
AccumuloServerContext implements Ifa
this.opts = opts;
this.fs = fs;
- long gcDelay = getConfiguration().getTimeInMillis(Property.GC_CYCLE_DELAY);
- log.info("start delay: " + getStartDelay() + " milliseconds");
- log.info("time delay: " + gcDelay + " milliseconds");
- log.info("safemode: " + opts.safeMode);
- log.info("verbose: " + opts.verbose);
- log.info("memory threshold: " + CANDIDATE_MEMORY_PERCENTAGE + " of "
- + Runtime.getRuntime().maxMemory() + " bytes");
- log.info("delete threads: " + getNumDeleteThreads());
+ final AccumuloConfiguration conf = getConfiguration();
+
+ final long gcDelay = conf.getTimeInMillis(Property.GC_CYCLE_DELAY);
+ final String useFullCompaction = conf.get(Property.GC_USE_FULL_COMPACTION);
+
+ log.info("start delay: {} milliseconds", getStartDelay());
+ log.info("time delay: {} milliseconds", gcDelay);
+ log.info("safemode: {}", opts.safeMode);
+ log.info("verbose: {}", opts.verbose);
+ log.info("memory threshold: {} of {} bytes", CANDIDATE_MEMORY_PERCENTAGE,
+ Runtime.getRuntime().maxMemory());
+ log.info("delete threads: {}", getNumDeleteThreads());
+ log.info("gc post metadata action: {}", useFullCompaction);
}
/**
@@ -540,7 +546,6 @@ public class SimpleGarbageCollector extends
AccumuloServerContext implements Ifa
}
private void run() {
- long tStart, tStop;
// Sleep for an initial period, giving the master time to start up and
// old data files to be unused
@@ -586,7 +591,7 @@ public class SimpleGarbageCollector extends
AccumuloServerContext implements Ifa
Trace.on("gc", sampler);
Span gcSpan = Trace.start("loop");
- tStart = System.currentTimeMillis();
+ final long tStart = System.nanoTime();
try {
System.gc(); // make room
@@ -608,8 +613,9 @@ public class SimpleGarbageCollector extends
AccumuloServerContext implements Ifa
log.error("{}", e.getMessage(), e);
}
- tStop = System.currentTimeMillis();
- log.info(String.format("Collect cycle took %.2f seconds", ((tStop -
tStart) / 1000.0)));
+ final long tStop = System.nanoTime();
+ log.info(String.format("Collect cycle took %.2f seconds",
+ (TimeUnit.NANOSECONDS.toMillis(tStop - tStart) / 1000.0)));
// We want to prune references to fully-replicated WALs from the
replication table which are
// no longer referenced in the metadata table
@@ -638,11 +644,35 @@ public class SimpleGarbageCollector extends
AccumuloServerContext implements Ifa
}
gcSpan.stop();
- // we just made a lot of metadata changes: flush them out
+ // We just made a lot of metadata changes. Flush them out.
+ // Either with flush and full compaction, or flush only and allow
automatic triggering
+ // of any necessary compactions.
try {
Connector connector = getConnector();
- connector.tableOperations().compact(MetadataTable.NAME, null, null,
true, true);
- connector.tableOperations().compact(RootTable.NAME, null, null, true,
true);
+
+ final long actionStart = System.nanoTime();
+
+ String action =
getConfiguration().get(Property.GC_USE_FULL_COMPACTION);
+ log.debug("gc post action {} started", action);
+
+ switch (action) {
+ case "compact":
+ connector.tableOperations().compact(MetadataTable.NAME, null,
null, true, true);
+ connector.tableOperations().compact(RootTable.NAME, null, null,
true, true);
+ break;
+ case "flush":
+ connector.tableOperations().flush(MetadataTable.NAME, null, null,
true);
+ connector.tableOperations().flush(RootTable.NAME, null, null,
true);
+ break;
+ default:
+ log.trace("\'none - no action\' or invalid value provided: {}",
action);
+ }
+
+ final long actionComplete = System.nanoTime();
+
+ log.info("gc post action {} completed in {} seconds", action,
String.format("%.2f",
+ (TimeUnit.NANOSECONDS.toMillis(actionComplete - actionStart) /
1000.0)));
+
} catch (Exception e) {
log.warn("{}", e.getMessage(), e);
}
