Repository: crunch Updated Branches: refs/heads/master b34cf6dfb -> ba7a3477c
CRUNCH-508: Cache Enumeration.toString values in Scrunch Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ba7a3477 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ba7a3477 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ba7a3477 Branch: refs/heads/master Commit: ba7a3477cd74777717c19cc0a85916a2a2208e8b Parents: b34cf6d Author: Josh Wills <[email protected]> Authored: Tue Apr 7 11:48:02 2015 -0700 Committer: Josh Wills <[email protected]> Committed: Thu Apr 23 08:49:09 2015 -0500 ---------------------------------------------------------------------- .../apache/crunch/scrunch/IncrementTest.scala | 9 +++++++++ .../org/apache/crunch/scrunch/Increment.scala | 19 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/ba7a3477/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala index 338051c..44aa9a8 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/IncrementTest.scala @@ -22,8 +22,15 @@ import org.apache.crunch.io.{From => from, To => to} import _root_.org.junit.Test import _root_.org.junit.Assert.assertEquals + class IncrementTest extends CrunchSuite { + object Inc extends Enumeration { + type Inc = Value + val A, B, C, D = Value + } + import Inc._ + @Test def testIncrement { val pipeline = Pipeline.mapReduce[IncrementTest](tempDir.getDefaultConfiguration) val input = tempDir.copyResourceFileName("shakes.txt") @@ -34,6 +41,7 @@ class IncrementTest extends CrunchSuite { .filter(!_.isEmpty()) .increment("TOP", "NONEMPTY") .incrementIf(_ startsWith "a")("TOP", "AWORDS_2x", 2) + .increment(Inc, Inc.A) .write(to.avroFile(tempDir.getFileName("somewords"))) val res = pipeline.done() @@ -41,5 +49,6 @@ class IncrementTest extends CrunchSuite { assertEquals(21836, sr0.getCounterValue("TOP", "ALLWORDS")) assertEquals(20366, sr0.getCounterValue("TOP", "NONEMPTY")) assertEquals(3604, sr0.getCounterValue("TOP", "AWORDS_2x")) + assertEquals(20366, sr0.getCounterValue("Inc", "A")) } } http://git-wip-us.apache.org/repos/asf/crunch/blob/ba7a3477/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala index f6b6ffa..046c777 100644 --- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala +++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/Increment.scala @@ -29,6 +29,9 @@ import org.apache.crunch.{FilterFn, Pair => CPair} * {@code Incrementable} trait. */ trait Incrementable[T] { + + private val enumCache = new java.util.HashMap[Enumeration, String]() + def increment(counter: Enum[_]): T = { increment(counter, 1) } @@ -42,7 +45,12 @@ trait Incrementable[T] { } def increment(groupEnum: Enumeration, value: Enumeration#Value, count: Long): T = { - increment(groupEnum.toString, value.toString, count) + var groupName = enumCache.get(groupEnum) + if (groupName == null) { + groupName = groupEnum.toString + enumCache.put(groupEnum, groupName) + } + increment(groupName, value.toString, count) } def increment(groupName: String, counterName: String): T = { @@ -61,6 +69,8 @@ trait Incrementable[T] { */ trait Increment[T] { + private val enumCache = new java.util.HashMap[Enumeration, String]() + def apply(counter: Enum[_]): T = { apply(counter, 1) } @@ -74,7 +84,12 @@ trait Increment[T] { } def apply(groupEnum: Enumeration, value: Enumeration#Value, count: Long): T = { - apply(groupEnum.toString, value.toString, count) + var groupName = enumCache.get(groupEnum) + if (groupName == null) { + groupName = groupEnum.toString + enumCache.put(groupEnum, groupName) + } + apply(groupName, value.toString, count) } def apply(groupName: String, counterName: String): T = {
