Repository: crunch Updated Branches: refs/heads/apache-crunch-0.8 8fe96d6cc -> da3ccd9db
CRUNCH-403: Use ___ instead of _ as substitute for $ in Avro reflected classes in Scrunch Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/da3ccd9d Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/da3ccd9d Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/da3ccd9d Branch: refs/heads/apache-crunch-0.8 Commit: da3ccd9dbcd4bfd8fd834b93283b0051b66dd617 Parents: 8fe96d6 Author: Josh Wills <[email protected]> Authored: Tue May 27 12:34:11 2014 -0700 Committer: Josh Wills <[email protected]> Committed: Wed May 28 20:12:06 2014 -0700 ---------------------------------------------------------------------- .../org/apache/crunch/scrunch/PageRankClassTest.scala | 12 +++++------- .../org/apache/crunch/scrunch/ScalaSafeReflectData.java | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/da3ccd9d/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala index 4cd9e84..3d3cb9f 100644 --- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala +++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala @@ -27,14 +27,14 @@ import scala.collection.mutable.HashMap import _root_.org.junit.Assert._ import _root_.org.junit.Test -case class PageRankData(pr: Float, oldpr: Float, urls: Array[String]) { +case class PageRankData(page_rank: Float, oldpr: Float, urls: Array[String]) { def this() = this(0f, 0f, null) - def scaledPageRank = pr / urls.length + def scaledPageRank = page_rank / urls.length - def next(newPageRank: Float) = new PageRankData(newPageRank, pr, urls) + def next(newPageRank: Float) = new PageRankData(newPageRank, page_rank, urls) - def delta = math.abs(pr - oldpr) + def delta = math.abs(page_rank - oldpr) } class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Float]] { @@ -45,7 +45,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String, Flo override def process(input: P[String, PageRankData], emitFn: Emitter[P[String, Float]]) { val prd = input.second() if (prd.urls.length > 0) { - val newpr = prd.pr / prd.urls.length + val newpr = prd.page_rank / prd.urls.length prd.urls.foreach(url => cache.put(url, cache(url) + newpr)) if (cache.size > 5000) { cleanup(emitFn) @@ -92,7 +92,6 @@ class PageRankClassTest extends CrunchSuite { } @Test def testPageRank { - pipeline.getConfiguration.set("crunch.debug", "true") var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { @@ -104,7 +103,6 @@ class PageRankClassTest extends CrunchSuite { } def testFastPageRank { - pipeline.getConfiguration.set("crunch.debug", "true") var prev = initialInput(tempDir.copyResourceFileName("urls.txt")) var delta = 1.0f while (delta > 0.01f) { http://git-wip-us.apache.org/repos/asf/crunch/blob/da3ccd9d/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java ---------------------------------------------------------------------- diff --git a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java index 7fbf52f..6118834 100644 --- a/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java +++ b/crunch-scrunch/src/main/java/org/apache/crunch/scrunch/ScalaSafeReflectData.java @@ -236,11 +236,11 @@ public class ScalaSafeReflectData extends ReflectData.AllowNull { } private static String clean(String dirty) { - return dirty.replace('$', '_'); + return dirty.replace("$", "___"); } private static String dirty(String clean) { - return clean.replace('_', '$'); + return clean.replace("___", "$"); } // Return of this class and its superclasses to serialize.
