Repository: mahout Updated Branches: refs/heads/master 638ac8c7f -> 8b2bec7f5
MAHOUT-1541, MAHOUT-1568, MAHOUT-1569 fixed a build test problem, drivers have an option new to not search for MAHOUT_HOME and SPARK_HOME Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/32badb1d Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/32badb1d Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/32badb1d Branch: refs/heads/master Commit: 32badb1d360ddf514e6b253f2dea9ae7e5df078a Parents: 2b65475 Author: pferrel <[email protected]> Authored: Fri Jul 4 11:23:20 2014 -0700 Committer: pferrel <[email protected]> Committed: Fri Jul 4 11:23:20 2014 -0700 ---------------------------------------------------------------------- .../apache/mahout/drivers/ItemSimilarityDriver.scala | 12 +++++++++--- .../org/apache/mahout/drivers/MahoutDriver.scala | 4 ++-- .../mahout/drivers/ItemSimilarityDriverSuite.scala | 15 ++++++++++----- 3 files changed, 21 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/32badb1d/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala ---------------------------------------------------------------------- diff --git a/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala b/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala index 77005f1..f78c590 100644 --- a/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala +++ b/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala @@ -167,6 +167,11 @@ object ItemSimilarityDriver extends MahoutDriver { note("\nDefault delimiters will produce output of the form: \"itemID1<tab>itemID2:value2,itemID10:value10...\"") + //Jar inclusion, this option can be set when executing the driver from compiled code + opt[Unit]("dontAddMahoutJars") hidden() action { (_, options) => + options.copy(dontAddMahoutJars = true) //set the value MahoutDriver so the context will be created correctly + }//Hidden option, used when executing tests or calling from other code where classes are all loaded explicitly + //Driver notes--driver specific note("\nNote: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure.\n") @@ -196,14 +201,14 @@ object ItemSimilarityDriver extends MahoutDriver { } override def start(masterUrl: String = options.master, - appName: String = options.appName): + appName: String = options.appName, dontAddMahoutJars: Boolean = options.dontAddMahoutJars): Unit = { sparkConf.set("spark.kryo.referenceTracking", "false") .set("spark.kryoserializer.buffer.mb", "200") .set("spark.executor.memory", options.sparkExecutorMem) - super.start(masterUrl, appName) + super.start(masterUrl, appName, dontAddMahoutJars) val readSchema1 = new Schema("delim" -> options.inDelim, "filter" -> options.filter1, "rowIDPosition" -> options.rowIDPosition, @@ -309,6 +314,7 @@ object ItemSimilarityDriver extends MahoutDriver { inDelim: String = "[,\t ]", rowKeyDelim: String = "\t", columnIdStrengthDelim: String = ":", - tupleDelim: String = ",") + tupleDelim: String = ",", + dontAddMahoutJars: Boolean = false) } http://git-wip-us.apache.org/repos/asf/mahout/blob/32badb1d/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala ---------------------------------------------------------------------- diff --git a/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala b/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala index afc7c1e..0c579d4 100644 --- a/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala +++ b/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala @@ -65,9 +65,9 @@ abstract class MahoutDriver { mc = mahoutSparkContext(masterUrl, appName, customJars, sparkConf) } - protected def start(masterUrl: String, appName: String) : Unit = { + protected def start(masterUrl: String, appName: String, dontAddMahoutJars: Boolean = false) : Unit = { val customJars = Traversable.empty[String] - mc = mahoutSparkContext(masterUrl, appName, customJars, sparkConf) + mc = mahoutSparkContext(masterUrl, appName, customJars, sparkConf, !dontAddMahoutJars) } /** Override (optionally) for special cleanup */ http://git-wip-us.apache.org/repos/asf/mahout/blob/32badb1d/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala ---------------------------------------------------------------------- diff --git a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala index f649d7b..9f928e2 100644 --- a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala +++ b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala @@ -131,7 +131,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc "--inDelim", ",", "--itemIDPosition", "2", "--rowIDPosition", "0", - "--filterPosition", "1")) + "--filterPosition", "1", + "--dontAddMahoutJars")) beforeEach // restart the test context to read the output of the driver val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String] @@ -185,7 +186,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc "--inDelim", "[,\t]", "--itemIDPosition", "2", "--rowIDPosition", "0", - "--filterPosition", "1")) + "--filterPosition", "1", + "--dontAddMahoutJars")) beforeEach // restart the test context to read the output of the driver val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String] @@ -238,7 +240,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc "--inDelim", "\t", "--itemIDPosition", "4", "--rowIDPosition", "1", - "--filterPosition", "2")) + "--filterPosition", "2", + "--dontAddMahoutJars")) beforeEach // restart the test context to read the output of the driver val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String] @@ -286,7 +289,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc ItemSimilarityDriver.main(Array( "--input", InPath, "--output", OutPath, - "--master", masterUrl)) + "--master", masterUrl, + "--dontAddMahoutJars")) beforeEach // restart the test context to read the output of the driver val indicatorLines = mahoutCtx.textFile(OutPath+"/indicator-matrix/").collect.toSet[String] @@ -362,7 +366,8 @@ class ItemSimilarityDriverSuite extends FunSuite with MahoutSuite with MahoutLoc "--rowIDPosition", "0", "--filterPosition", "1", "--filenamePattern", "m..tsv", - "--recursive")) + "--recursive", + "--dontAddMahoutJars")) beforeEach()// restart the test context to read the output of the driver val indicatorLines = mahoutCtx.textFile(OutPath + "/indicator-matrix/").collect.toSet[String]
