Repository: incubator-predictionio Updated Branches: refs/heads/develop 9bbd1f51a -> cc4e2e0a2
[PIO-56] No-setup unit tests for core and data Closes #355 Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/cc4e2e0a Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/cc4e2e0a Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/cc4e2e0a Branch: refs/heads/develop Commit: cc4e2e0a2303962f2b2112ec7612f39579786439 Parents: 9bbd1f5 Author: lucasbm88 <[email protected]> Authored: Thu Jul 27 11:40:19 2017 -0700 Committer: Donald Szeto <[email protected]> Committed: Thu Jul 27 11:40:19 2017 -0700 ---------------------------------------------------------------------- core/build.sbt | 5 +- .../controller/EvaluationTest.scala | 6 +- .../controller/MetricEvaluatorTest.scala | 6 +- .../apache/predictionio/workflow/BaseTest.scala | 59 ++++++++++++++++-- .../workflow/EvaluationWorkflowTest.scala | 5 +- data/build.sbt | 5 +- .../predictionio/data/storage/Storage.scala | 41 +++++++++++-- .../data/api/EventServiceSpec.scala | 39 ++++++------ .../data/api/SegmentIOAuthSpec.scala | 36 ++++++----- .../data/storage/StorageMockContext.scala | 64 ++++++++++++++++++++ 10 files changed, 211 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/build.sbt ---------------------------------------------------------------------- diff --git a/core/build.sbt b/core/build.sbt index b95a957..14b3449 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -30,7 +30,10 @@ libraryDependencies ++= Seq( "org.scalaj" %% "scalaj-http" % "1.1.6", "org.slf4j" % "slf4j-log4j12" % "1.7.18", "org.scalatest" %% "scalatest" % "2.1.7" % "test", - "org.specs2" %% "specs2" % "2.3.13" % "test") + "org.specs2" %% "specs2" % "2.3.13" % "test", + "org.scalamock" %% "scalamock-scalatest-support" % "3.5.0" % "test", + "com.h2database" % "h2" % "1.4.196" % "test" +) parallelExecution in Test := false http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala b/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala index a64826c..b60d358 100644 --- a/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala +++ b/core/src/test/scala/org/apache/predictionio/controller/EvaluationTest.scala @@ -17,12 +17,10 @@ package org.apache.predictionio.controller -import org.apache.predictionio.workflow.SharedSparkContext - +import org.apache.predictionio.workflow.{SharedSparkContext, SharedStorageContext} import org.scalatest.FunSuite import org.scalatest.Inside import org.scalatest.Matchers._ - import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD @@ -42,7 +40,7 @@ object EvaluationSuite { class EvaluationSuite -extends FunSuite with Inside with SharedSparkContext { +extends FunSuite with Inside with SharedSparkContext with SharedStorageContext { import org.apache.predictionio.controller.EvaluationSuite._ test("Evaluation makes MetricEvaluator") { http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala b/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala index 9aec6d5..165fc63 100644 --- a/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala +++ b/core/src/test/scala/org/apache/predictionio/controller/MetricEvaluatorTest.scala @@ -18,8 +18,7 @@ package org.apache.predictionio.controller -import org.apache.predictionio.workflow.SharedSparkContext -import org.apache.predictionio.workflow.WorkflowParams +import org.apache.predictionio.workflow.{SharedSparkContext, SharedStorageContext, WorkflowParams} import org.scalatest.FunSuite object MetricEvaluatorSuite { @@ -30,7 +29,8 @@ object MetricEvaluatorSuite { object Evaluation0 extends Evaluation {} } -class MetricEvaluatorDevSuite extends FunSuite with SharedSparkContext { +class MetricEvaluatorDevSuite extends FunSuite with SharedSparkContext +with SharedStorageContext { import org.apache.predictionio.controller.MetricEvaluatorSuite._ test("a") { http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala b/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala index 004dbb1..59d23f1 100644 --- a/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala +++ b/core/src/test/scala/org/apache/predictionio/workflow/BaseTest.scala @@ -15,20 +15,22 @@ * limitations under the License. */ -//package org.apache.spark +// package org.apache.spark package org.apache.predictionio.workflow -import _root_.io.netty.util.internal.logging.{Slf4JLoggerFactory, InternalLoggerFactory} +import _root_.io.netty.util.internal.logging.{InternalLoggerFactory, Slf4JLoggerFactory} +import org.apache.predictionio.data.storage.{EnvironmentFactory, EnvironmentService} import org.scalatest.BeforeAndAfterAll import org.scalatest.BeforeAndAfterEach import org.scalatest.Suite import org.apache.spark.SparkContext import org.apache.spark.SparkConf +import org.scalamock.scalatest.MockFactory /** Manages a local `sc` {@link SparkContext} variable, correctly stopping it * after each test. */ -trait LocalSparkContext +trait LocalSparkContext extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite => @transient var sc: SparkContext = _ @@ -43,7 +45,7 @@ extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite => super.afterEach() } - def resetSparkContext() = { + def resetSparkContext() : Unit = { LocalSparkContext.stop(sc) sc = null } @@ -60,7 +62,7 @@ object LocalSparkContext { } /** Runs `f` by passing in `sc` and ensures that `sc` is stopped. */ - def withSpark[T](sc: SparkContext)(f: SparkContext => T) = { + def withSpark[T](sc: SparkContext)(f: SparkContext => T) : Unit = { try { f(sc) } finally { @@ -90,3 +92,50 @@ trait SharedSparkContext extends BeforeAndAfterAll { self: Suite => } } +trait SharedStorageContext extends BeforeAndAfterAll { self: Suite => + + override def beforeAll(): Unit ={ + ConfigurationMockUtil.createJDBCMockedConfig + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + } + +} + +object ConfigurationMockUtil extends MockFactory { + + def createJDBCMockedConfig: Unit = { + val mockedEnvService = mock[EnvironmentService] + (mockedEnvService.envKeys _) + .expects + .returning(List("PIO_STORAGE_REPOSITORIES_METADATA_NAME", + "PIO_STORAGE_SOURCES_MYSQL_TYPE")) + .twice + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_REPOSITORIES_METADATA_NAME") + .returning("test_metadata") + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_REPOSITORIES_METADATA_SOURCE") + .returning("MYSQL") + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_SOURCES_MYSQL_TYPE") + .returning("jdbc") + + (mockedEnvService.filter _) + .expects(*) + .returning(Map( + "URL" -> "jdbc:h2:~/test;MODE=MySQL;AUTO_SERVER=TRUE", + "USERNAME" -> "sa", + "PASSWORD" -> "") + ) + + EnvironmentFactory.environmentService = new Some(mockedEnvService) + } +} + http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala b/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala index 26fc936..70534e4 100644 --- a/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala +++ b/core/src/test/scala/org/apache/predictionio/workflow/EvaluationWorkflowTest.scala @@ -18,11 +18,12 @@ package org.apache.predictionio.workflow import org.apache.predictionio.controller._ - +import org.scalamock.scalatest.MockFactory import org.scalatest.FunSuite import org.scalatest.Matchers._ -class EvaluationWorkflowSuite extends FunSuite with SharedSparkContext { +class EvaluationWorkflowSuite extends FunSuite with SharedStorageContext + with SharedSparkContext with MockFactory { test("Evaluation return best engine params, simple result type: Double") { val engine = new Engine1() http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/build.sbt ---------------------------------------------------------------------- diff --git a/data/build.sbt b/data/build.sbt index b6b2410..abfa74f 100644 --- a/data/build.sbt +++ b/data/build.sbt @@ -28,7 +28,10 @@ libraryDependencies ++= Seq( "org.clapper" %% "grizzled-slf4j" % "1.0.2", "org.json4s" %% "json4s-native" % json4sVersion.value, "org.scalatest" %% "scalatest" % "2.1.7" % "test", - "org.specs2" %% "specs2" % "2.3.13" % "test") + "org.specs2" %% "specs2" % "3.3.1" % "test" + exclude("org.scalaz.stream", s"scalaz-stream_${scalaBinaryVersion.value}"), + "org.scalamock" %% "scalamock-specs2-support" % "3.5.0" % "test", + "com.h2database" % "h2" % "1.4.196" % "test") parallelExecution in Test := false http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala ---------------------------------------------------------------------- diff --git a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala index 744a5e6..fd05767 100644 --- a/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala +++ b/data/src/main/scala/org/apache/predictionio/data/storage/Storage.scala @@ -112,6 +112,33 @@ class StorageException(message: String, cause: Throwable) def this(message: String) = this(message, null) } +class EnvironmentService{ + + def envKeys(): Iterable[String] = { + sys.env.keys + } + + def getByKey(key: String): String = { + sys.env(key) + } + + def filter(filterExpression: ((String, String)) => Boolean): Map[String, String] = { + sys.env.filter(filterExpression) + } +} + +object EnvironmentFactory{ + + var environmentService: Option[EnvironmentService] = None + + def create(): EnvironmentService = { + if(environmentService.isEmpty){ + environmentService = new Some[EnvironmentService](new EnvironmentService) + } + environmentService.get + } +} + /** Backend-agnostic data storage layer with lazy initialization. Use this * object when you need to interface with Event Store in your engine. * @@ -123,6 +150,8 @@ object Storage extends Logging { client: BaseStorageClient, config: StorageClientConfig) + var environmentService: EnvironmentService = EnvironmentFactory.create + private case class DataObjectMeta(sourceName: String, namespace: String) private var errors = 0 @@ -131,7 +160,7 @@ object Storage extends Logging { private val sourceTypesRegex = """PIO_STORAGE_SOURCES_([^_]+)_TYPE""".r - private val sourceKeys: Seq[String] = sys.env.keys.toSeq.flatMap { k => + private val sourceKeys: Seq[String] = environmentService.envKeys.toSeq.flatMap { k => sourceTypesRegex findFirstIn k match { case Some(sourceTypesRegex(sourceType)) => Seq(sourceType) case None => Nil @@ -152,7 +181,7 @@ object Storage extends Logging { private val repositoryNamesRegex = """PIO_STORAGE_REPOSITORIES_([^_]+)_NAME""".r - private val repositoryKeys: Seq[String] = sys.env.keys.toSeq.flatMap { k => + private val repositoryKeys: Seq[String] = environmentService.envKeys.toSeq.flatMap { k => repositoryNamesRegex findFirstIn k match { case Some(repositoryNamesRegex(repositoryName)) => Seq(repositoryName) case None => Nil @@ -175,8 +204,8 @@ object Storage extends Logging { repositoryKeys.map(r => try { val keyedPath = repositoriesPrefixPath(r) - val name = sys.env(prefixPath(keyedPath, "NAME")) - val sourceName = sys.env(prefixPath(keyedPath, "SOURCE")) + val name = environmentService.getByKey(prefixPath(keyedPath, "NAME")) + val sourceName = environmentService.getByKey(prefixPath(keyedPath, "SOURCE")) if (sourceKeys.contains(sourceName)) { r -> DataObjectMeta( sourceName = sourceName, @@ -244,8 +273,8 @@ object Storage extends Logging { Option[ClientMeta] = { try { val keyedPath = sourcesPrefixPath(k) - val sourceType = sys.env(prefixPath(keyedPath, "TYPE")) - val props = sys.env.filter(t => t._1.startsWith(keyedPath)).map( + val sourceType = environmentService.getByKey(prefixPath(keyedPath, "TYPE")) + val props = environmentService.filter(t => t._1.startsWith(keyedPath)).map( t => t._1.replace(s"${keyedPath}_", "") -> t._2) val clientConfig = StorageClientConfig( properties = props, http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala ---------------------------------------------------------------------- diff --git a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala b/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala index 988146e..7a45ca1 100644 --- a/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala +++ b/data/src/test/scala/org/apache/predictionio/data/api/EventServiceSpec.scala @@ -18,40 +18,41 @@ package org.apache.predictionio.data.api -import org.apache.predictionio.data.storage.Storage - +import org.apache.predictionio.data.storage.{Storage, StorageMockContext} import akka.testkit.TestProbe -import akka.actor.ActorSystem -import akka.actor.Props - +import akka.actor.{ActorRef, ActorSystem, Props} import spray.http.HttpEntity import spray.http.HttpResponse import spray.http.ContentTypes import spray.httpx.RequestBuilding.Get - import org.specs2.mutable.Specification class EventServiceSpec extends Specification { val system = ActorSystem("EventServiceSpecSystem") - val eventClient = Storage.getLEvents() - val accessKeysClient = Storage.getMetaDataAccessKeys() - val channelsClient = Storage.getMetaDataChannels() - - val eventServiceActor = system.actorOf( - Props( - new EventServiceActor( - eventClient, - accessKeysClient, - channelsClient, - EventServerConfig() + def createEventServiceActor: ActorRef = { + val eventClient = Storage.getLEvents() + val accessKeysClient = Storage.getMetaDataAccessKeys() + val channelsClient = Storage.getMetaDataChannels() + + system.actorOf( + Props( + new EventServiceActor( + eventClient, + accessKeysClient, + channelsClient, + EventServerConfig() + ) ) ) - ) + } + "GET / request" should { - "properly produce OK HttpResponses" in { + "properly produce OK HttpResponses" in new StorageMockContext { + Thread.sleep(2000) + val eventServiceActor = createEventServiceActor val probe = TestProbe()(system) probe.send(eventServiceActor, Get("/")) probe.expectMsg( http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala ---------------------------------------------------------------------- diff --git a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala b/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala index a276c59..5927824 100644 --- a/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala +++ b/data/src/test/scala/org/apache/predictionio/data/api/SegmentIOAuthSpec.scala @@ -17,17 +17,18 @@ package org.apache.predictionio.data.api -import akka.actor.{ActorSystem, Props} +import akka.actor.{ActorRef, ActorSystem, Props} import akka.testkit.TestProbe import org.apache.predictionio.data.storage._ import org.joda.time.DateTime +import org.scalamock.specs2.MockContext import org.specs2.mutable.Specification import spray.http.HttpHeaders.RawHeader import spray.http.{ContentTypes, HttpEntity, HttpResponse} import spray.httpx.RequestBuilding._ import sun.misc.BASE64Encoder -import scala.concurrent.{Future, ExecutionContext} +import scala.concurrent.{ExecutionContext, Future} class SegmentIOAuthSpec extends Specification { @@ -78,23 +79,26 @@ class SegmentIOAuthSpec extends Specification { } } - val channelsClient = Storage.getMetaDataChannels() - val eventServiceActor = system.actorOf( - Props( - new EventServiceActor( - eventClient, - accessKeysClient, - channelsClient, - EventServerConfig() + val base64Encoder = new BASE64Encoder + + def createEventServiceActor(): ActorRef = { + val channelsClient = Storage.getMetaDataChannels() + system.actorOf( + Props( + new EventServiceActor( + eventClient, + accessKeysClient, + channelsClient, + EventServerConfig() + ) ) ) - ) - - val base64Encoder = new BASE64Encoder + } "Event Service" should { - "reject with CredentialsRejected with invalid credentials" in { + "reject with CredentialsRejected with invalid credentials" in new StorageMockContext { + val eventServiceActor = createEventServiceActor val accessKey = "abc123:" val probe = TestProbe()(system) probe.send( @@ -119,6 +123,7 @@ class SegmentIOAuthSpec extends Specification { } "reject with CredentialsMissed without credentials" in { + val eventServiceActor = createEventServiceActor val probe = TestProbe()(system) probe.send( eventServiceActor, @@ -137,6 +142,7 @@ class SegmentIOAuthSpec extends Specification { } "process SegmentIO identity request properly" in { + val eventServiceActor = createEventServiceActor val jsonReq = """ |{ @@ -190,3 +196,5 @@ class SegmentIOAuthSpec extends Specification { step(system.shutdown()) } + + http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/cc4e2e0a/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala ---------------------------------------------------------------------- diff --git a/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala b/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala new file mode 100644 index 0000000..8476c91 --- /dev/null +++ b/data/src/test/scala/org/apache/predictionio/data/storage/StorageMockContext.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.predictionio.data.storage + +import org.scalamock.specs2.MockContext + +trait StorageMockContext extends MockContext { + + if(!EnvironmentFactory.environmentService.isDefined){ + val mockedEnvService = mock[EnvironmentService] + (mockedEnvService.envKeys _) + .expects + .returning(List("PIO_STORAGE_REPOSITORIES_METADATA_NAME", + "PIO_STORAGE_SOURCES_MYSQL_TYPE", + "PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME", + "PIO_STORAGE_SOURCES_EVENTDATA_TYPE")) + .twice + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_REPOSITORIES_METADATA_NAME") + .returning("test_metadata") + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_REPOSITORIES_METADATA_SOURCE") + .returning("MYSQL") + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_REPOSITORIES_EVENTDATA_NAME") + .returning("test_eventdata") + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_REPOSITORIES_EVENTDATA_SOURCE") + .returning("MYSQL") + + (mockedEnvService.getByKey _) + .expects("PIO_STORAGE_SOURCES_MYSQL_TYPE") + .returning("jdbc") + + (mockedEnvService.filter _) + .expects(*) + .returning(Map( + "URL" -> "jdbc:h2:~/test;MODE=MySQL;AUTO_SERVER=TRUE", + "USERNAME" -> "sa", + "PASSWORD" -> "") + ) + + EnvironmentFactory.environmentService = new Some(mockedEnvService) + } +}
