This is an automated email from the ASF dual-hosted git repository.
rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 13209cb CouchDB support in OpenWhisk Standalone mode (#4594)
13209cb is described below
commit 13209cbc643f8e061b064692befe142694eb7a86
Author: Chetan Mehrotra <[email protected]>
AuthorDate: Thu Aug 29 11:11:01 2019 -0700
CouchDB support in OpenWhisk Standalone mode (#4594)
This commit allows a standalone CouchDB instance to back the OpenWhisk API
when running using the Standalone JAR, providing persistence across restarts.
At startup, the CouchDB docker image is started and torn down on shutdown. An
explicit clean flag wipes the persisted state.
---
core/standalone/README.md | 7 +-
core/standalone/build.gradle | 4 +
core/standalone/src/main/resources/standalone.conf | 40 +++-
.../openwhisk/standalone/ApiGwLauncher.scala | 26 +-
.../openwhisk/standalone/CouchDBLauncher.scala | 264 +++++++++++++++++++++
.../standalone/StandaloneDockerSupport.scala | 29 ++-
.../openwhisk/standalone/StandaloneOpenWhisk.scala | 58 ++++-
.../standalone/StandaloneCouchTests.scala | 47 ++++
.../standalone/StandaloneServerFixture.scala | 9 +-
9 files changed, 436 insertions(+), 48 deletions(-)
diff --git a/core/standalone/README.md b/core/standalone/README.md
index 6b9aef9..3fc212b 100644
--- a/core/standalone/README.md
+++ b/core/standalone/README.md
@@ -77,8 +77,10 @@ $ java -jar openwhisk-standalone.jar -h
--api-gw Enable API Gateway support
--api-gw-port <arg> Api Gateway Port
+ --clean Clean any existing state like database
-c, --config-file <arg> application.conf which overrides the default
standalone.conf
+ --couchdb Enable CouchDB support
-d, --data-dir <arg> Directory used for storage
--disable-color-logging Disables colored logging
-m, --manifest <arg> Manifest json defining the supported runtimes
@@ -169,7 +171,10 @@ You can then see the runtime config reflect in
`http://localhost:3233`
#### Using CouchDB
-If you need to connect to CouchDB or any other supported artifact store then
you can pass the required config
+If you need to use CouchDB then you can launch the standalone server with
`--couchdb` option. This would launch
+a CouchDB server which would configured to store files in user home directory
under `.openwhisk/standalone` folder.
+
+If you need to connect to external CouchDB or any other supported artifact
store then you can pass the required config
```hocon
include classpath("standalone.conf")
diff --git a/core/standalone/build.gradle b/core/standalone/build.gradle
index 9225032..e571bc5 100644
--- a/core/standalone/build.gradle
+++ b/core/standalone/build.gradle
@@ -87,6 +87,10 @@ processResources {
from(new File(project.rootProject.projectDir,
"ansible/files/runtimes.json")) {
into(".")
}
+ from(new File(project.rootProject.projectDir, "ansible/files")) {
+ include "*.json"
+ into("couch")
+ }
//Implement the logic present in controller Docker file
from(project.swaggerUiDir) {
include "index.html"
diff --git a/core/standalone/src/main/resources/standalone.conf
b/core/standalone/src/main/resources/standalone.conf
index cf2a091..50d44ee 100644
--- a/core/standalone/src/main/resources/standalone.conf
+++ b/core/standalone/src/main/resources/standalone.conf
@@ -52,10 +52,10 @@ whisk {
}
controller {
- protocol: http
+ protocol = http
# Bound only to localhost by default for better security
- interface: localhost
+ interface = localhost
}
# Default set of users which are bootstrapped upon start
@@ -69,26 +69,50 @@ whisk {
# executable =
standalone.container-factory {
#If enabled then pull would also be attempted for standard OpenWhisk
images under`openwhisk` prefix
- pull-standard-images: true
+ pull-standard-images = true
}
container-factory {
# Disable runc by default to keep things stable
- use-runc: false
+ use-runc = false
}
}
swagger-ui {
- file-system : false
- dir-path : "BOOT-INF/classes/swagger-ui"
+ file-system = false
+ dir-path = "BOOT-INF/classes/swagger-ui"
}
standalone {
redis {
- image: "redis:4.0"
+ image = "redis:4.0"
}
api-gateway {
- image: "openwhisk/apigateway:nightly"
+ image = "openwhisk/apigateway:nightly"
+ }
+
+ couchdb {
+ image = "apache/couchdb:2.3"
+ user = "whisk_admin"
+ password = "some_passw0rd"
+ prefix = "whisk_local_"
+ volumes-enabled = true
+ subject-views = [
+ "auth_index.json",
+ "filter_design_document.json",
+ "namespace_throttlings_design_document_for_subjects_db.json"
+ ]
+ whisk-views = [
+ "whisks_design_document_for_entities_db_v2.1.0.json",
+ "filter_design_document.json"
+ ]
+ activation-views = [
+ "whisks_design_document_for_activations_db_v2.1.0.json",
+ "whisks_design_document_for_activations_db_filters_v2.1.0.json",
+ "filter_design_document.json",
+ "activations_design_document_for_activations_db.json",
+ "logCleanup_design_document_for_activations_db.json"
+ ]
}
}
}
diff --git
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/ApiGwLauncher.scala
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/ApiGwLauncher.scala
index 31d7562..2f44249 100644
---
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/ApiGwLauncher.scala
+++
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/ApiGwLauncher.scala
@@ -21,7 +21,6 @@ import akka.actor.{ActorSystem, Scheduler}
import akka.http.scaladsl.model.Uri
import akka.pattern.RetrySupport
import org.apache.openwhisk.common.{Logging, TransactionId}
-import org.apache.openwhisk.core.containerpool.docker.BrokenDockerContainer
import org.apache.openwhisk.standalone.StandaloneDockerSupport.{containerName,
createRunCmd}
import pureconfig.loadConfigOrThrow
@@ -57,7 +56,7 @@ class ApiGwLauncher(docker: StandaloneDockerClient,
apiGwApiPort: Int, apiGwMgmt
val params = Map("-p" -> Set(s"$redisPort:6379"))
val name = containerName("redis")
val args = createRunCmd(name, dockerRunParameters = params)
- val f = runDetached(redisConfig.image, args, pull = true)
+ val f = docker.runDetached(redisConfig.image, args, shouldPull = true)
val sc = ServiceContainer(redisPort, "Redis", name)
f.map(c => (c, Seq(sc)))
}
@@ -101,7 +100,7 @@ class ApiGwLauncher(docker: StandaloneDockerClient,
apiGwApiPort: Int, apiGwMgmt
//TODO ExecManifest is scoped to core. Ideally we would like to do
// ExecManifest.ImageName(apiGwConfig.image).prefix.contains("openwhisk")
val pull = apiGwConfig.image.startsWith("openwhisk")
- val f = runDetached(apiGwConfig.image, args, pull)
+ val f = docker.runDetached(apiGwConfig.image, args, pull)
val sc = Seq(
ServiceContainer(apiGwApiPort, "Api Gateway - Api Service", name),
ServiceContainer(apiGwMgmtPort, "Api Gateway - Management Service",
name))
@@ -115,25 +114,4 @@ class ApiGwLauncher(docker: StandaloneDockerClient,
apiGwApiPort: Int, apiGwMgmt
.waitForServerToStart()
Future.successful(())
}
-
- private def runDetached(image: String, args: Seq[String], pull: Boolean):
Future[StandaloneDockerContainer] = {
- for {
- _ <- if (pull) docker.pull(image) else Future.successful(())
- id <- docker.run(image, args).recoverWith {
- case t @ BrokenDockerContainer(brokenId, _) =>
- // Remove the broken container - but don't wait or check for the
result.
- // If the removal fails, there is nothing we could do to recover
from the recovery.
- docker.rm(brokenId)
- Future.failed(t)
- case t => Future.failed(t)
- }
- ip <- docker.inspectIPAddress(id,
StandaloneDockerSupport.network).recoverWith {
- // remove the container immediately if inspect failed as
- // we cannot recover that case automatically
- case e =>
- docker.rm(id)
- Future.failed(e)
- }
- } yield StandaloneDockerContainer(id, ip)
- }
}
diff --git
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/CouchDBLauncher.scala
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/CouchDBLauncher.scala
new file mode 100644
index 0000000..c34027d
--- /dev/null
+++
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/CouchDBLauncher.scala
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.standalone
+
+import java.io.File
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets.UTF_8
+
+import akka.Done
+import akka.actor.ActorSystem
+import akka.http.scaladsl.model.headers.{Accept, Authorization,
BasicHttpCredentials}
+import akka.http.scaladsl.model.{HttpHeader, HttpMethods, HttpRequest,
MediaTypes, StatusCode, StatusCodes, Uri}
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
+import com.typesafe.config.ConfigFactory
+import org.apache.commons.io.IOUtils
+import org.apache.openwhisk.common.{Logging, TransactionId}
+import org.apache.openwhisk.core.database.CouchDbRestClient
+import org.apache.openwhisk.http.PoolingRestClient
+import org.apache.openwhisk.http.PoolingRestClient._
+import org.apache.openwhisk.standalone.StandaloneDockerSupport.{containerName,
createRunCmd}
+import pureconfig.loadConfigOrThrow
+import spray.json.DefaultJsonProtocol._
+import spray.json._
+
+import scala.concurrent.{ExecutionContext, Future}
+
+class CouchDBLauncher(docker: StandaloneDockerClient, port: Int, dataDir:
File)(implicit logging: Logging,
+
ec: ExecutionContext,
+
actorSystem: ActorSystem,
+
materializer: ActorMaterializer,
+
tid: TransactionId) {
+ case class CouchDBConfig(image: String,
+ user: String,
+ password: String,
+ prefix: String,
+ volumesEnabled: Boolean,
+ subjectViews: List[String],
+ whiskViews: List[String],
+ activationViews: List[String])
+ private val dbConfig =
loadConfigOrThrow[CouchDBConfig](StandaloneConfigKeys.couchDBConfigKey)
+ private val couchClient = new PoolingRestClient("http",
StandaloneDockerSupport.getLocalHostName(), port, 100)
+ private val baseHeaders: List[HttpHeader] =
+ List(Authorization(BasicHttpCredentials(dbConfig.user,
dbConfig.password)), Accept(MediaTypes.`application/json`))
+ private val subjectDb = dbConfig.prefix + "subjects"
+ private val activationsDb = dbConfig.prefix + "activations"
+ private val whisksDb = dbConfig.prefix + "whisks"
+ private val resourcePrefix = "couch"
+
+ def run(): Future[ServiceContainer] = {
+ if (dataDir.list().nonEmpty) {
+ logging.info(this, s"Using pre-existing database from
${dataDir.getAbsolutePath}")
+ } else {
+ logging.info(this, s"Creating new database at
${dataDir.getAbsolutePath}")
+ }
+ for {
+ (_, dbSvcs) <- runCouch()
+ _ <- waitForCouchDB()
+ _ <- createDbIfNotExist("_users")
+ _ <- createDbWithViews(subjectDb, dbConfig.subjectViews)
+ _ <- createDbWithViews(whisksDb, dbConfig.whiskViews)
+ _ <- createDbWithViews(activationsDb, dbConfig.activationViews)
+ _ <- {
+ updateConfig()
+ logging.info(
+ this,
+ s"CouchDB started successfully at http://${StandaloneDockerSupport
+ .getLocalHostName()}:$port/_utils . Username: [${dbConfig.user}],
Password: [${dbConfig.password}]")
+ Future.successful(Done)
+ }
+ } yield dbSvcs
+ }
+
+ def runCouch(): Future[(StandaloneDockerContainer, ServiceContainer)] = {
+ logging.info(this, s"Starting CouchDB at $port")
+ val baseParams = Map("-p" -> Set(s"$port:5984"))
+ val params =
+ if (dbConfig.volumesEnabled) baseParams + ("-v" ->
Set(s"${dataDir.getAbsolutePath}:/opt/couchdb/data"))
+ else baseParams
+ val env = Map("COUCHDB_USER" -> dbConfig.user, "COUCHDB_PASSWORD" ->
dbConfig.password)
+ val name = containerName("couch")
+ val args = createRunCmd(name, env, params)
+ val f = docker.runDetached(dbConfig.image, args, shouldPull = true)
+ val sc = ServiceContainer(port, "CouchDB", name)
+ f.map(c => (c, sc))
+ }
+
+ def waitForCouchDB(): Future[Done] = {
+ new
ServerStartupCheck(Uri(s"http://${StandaloneDockerSupport.getLocalHostName()}:$port/_utils/"),
"CouchDB")
+ .waitForServerToStart()
+ Future.successful(Done)
+ }
+
+ private def createDbWithViews(dbName: String, views: List[String]):
Future[Done] = {
+ for {
+ _ <- createDbIfNotExist(dbName)
+ _ <- createDocs(views, dbName)
+ } yield Done
+ }
+
+ private def createDbIfNotExist(dbName: String): Future[Done] = {
+ for {
+ userDbExist <- doesDbExist(dbName)
+ _ <- if (userDbExist) Future.successful(Done) else createDb(dbName)
+ } yield Done
+ }
+
+ private def doesDbExist(dbName: String): Future[Boolean] = {
+ requestString(mkRequest(HttpMethods.HEAD, uri(dbName), headers =
baseHeaders))
+ .map {
+ case Right(_) => true
+ case Left(StatusCodes.NotFound) => false
+ case Left(s) => throw new
IllegalStateException("Unknown status code while checking user db" + s)
+ }
+ }
+
+ private def createDb(dbName: String): Future[Done] = {
+ requestString(mkRequest(HttpMethods.PUT, uri(dbName), headers =
baseHeaders))
+ .map {
+ case Right(_) => Done
+ case Left(s) => throw new IllegalStateException(("Unknown status code
while creating user db" + s))
+ }
+ }
+
+ private def createDocs(jsonFiles: List[String], dbName: String):
Future[Done] = {
+ val client = createDbClient(dbName)
+ val f = jsonFiles
+ .map { p =>
+ val s = IOUtils.resourceToString(s"/$resourcePrefix/$p", UTF_8)
+ val js = s.parseJson.asJsObject
+ logging.info(this, s"Creating view doc from file $p for db $dbName")
+ createDocIfRequired(js, dbName, client)
+ }
+
+ Future.sequence(f).map(_ => client.shutdown()).map(_ => Done)
+ }
+
+ private def createDocIfRequired(doc: JsObject, dbName: String, client:
CouchDbRestClient): Future[Done] = {
+ val id = doc.fields("_id").convertTo[String]
+ for {
+ jsOpt <- getDoc(id, client)
+ _ <- jsOpt match {
+ case Some(js) => {
+ val rev = js.fields("_rev").convertTo[String]
+ val docWithRev = JsObject(doc.fields + ("_rev" -> JsString(rev)))
+ if (docWithRev != js) {
+ logging.info(this, s"Updating doc $id for db ${dbName}")
+ createDoc(id, Some(rev), doc, client)
+ } else Future.successful(Done)
+ }
+ case None => {
+ logging.info(this, s"Creating doc $id for db ${dbName}")
+ createDoc(id, None, doc, client)
+ }
+ }
+ } yield Done
+ }
+
+ private def getDoc(id: String, client: CouchDbRestClient):
Future[Option[JsObject]] = {
+ client.getDoc(id).map {
+ case Right(js) => Some(js)
+ case Left(StatusCodes.NotFound) => None
+ case Left(s) => throw new
IllegalStateException(s"Unknown status code while fetching doc $id" + s)
+ }
+ }
+
+ private def createDoc(id: String, rev: Option[String], doc: JsObject,
client: CouchDbRestClient): Future[Done] = {
+ val f = rev match {
+ case Some(r) => client.putDoc(id, r, doc)
+ case None => client.putDoc(id, doc)
+ }
+ f.map {
+ case Right(_) => Done
+ case Left(s) => throw new IllegalStateException(s"Unknown status code
while creating doc $id" + s)
+ }
+ }
+
+ // Properly encodes the potential slashes in each segment.
+ private def uri(segments: Any*): Uri = {
+ val encodedSegments = segments.map(s => URLEncoder.encode(s.toString,
UTF_8.name))
+ Uri(s"/${encodedSegments.mkString("/")}")
+ }
+
+ private def createDbClient(dbName: String) =
+ new NonEscapingClient(
+ "http",
+ StandaloneDockerSupport.getLocalHostName(),
+ port,
+ dbConfig.user,
+ dbConfig.password,
+ dbName)(actorSystem, logging)
+
+ private def updateConfig(): Unit = {
+ //The config needs to pushed via system property and then the Typesafe
ConfigFactory cache
+ //should be purged such that config gets read again and hence read these
system properties
+ setp("host", StandaloneDockerSupport.getLocalHostName())
+ setp("port", port.toString)
+ setp("password", dbConfig.password)
+ setp("username", dbConfig.user)
+ setp("protocol", "http")
+ setp("provider", "CouchDB")
+ setp("databases.WhiskActivation", activationsDb)
+ setp("databases.WhiskAuth", subjectDb)
+ setp("databases.WhiskEntity", whisksDb)
+
+ System.setProperty(s"whisk.spi.ArtifactStoreProvider",
"org.apache.openwhisk.core.database.CouchDbStoreProvider")
+ ConfigFactory.invalidateCaches()
+ logging.info(this, "Invalidated config cached")
+ }
+
+ private def setp(key: String, value: String): Unit = {
+ System.setProperty(s"whisk.couchdb.$key", value)
+ }
+
+ /**
+ * This is similar to PoolingRestClient#requestJson just that here we
materialize to String. As some of the db
+ * related operation return with empty body
+ */
+ private def requestString(futureRequest: Future[HttpRequest]):
Future[Either[StatusCode, String]] = {
+ couchClient.request(futureRequest).flatMap { response =>
+ if (response.status.isSuccess) {
+ Unmarshal(response.entity.withoutSizeLimit).to[String].map(Right.apply)
+ } else {
+ Unmarshal(response.entity).to[String].flatMap { body =>
+ val statusCode = response.status
+ val reason =
+ if (body.nonEmpty) s"${statusCode.reason} (details: $body)" else
statusCode.reason
+ val customStatusCode = StatusCodes
+ .custom(intValue = statusCode.intValue, reason = reason,
defaultMessage = statusCode.defaultMessage)
+ // This is important, as it drains the entity stream.
+ // Otherwise the connection stays open and the pool dries up.
+ response.discardEntityBytes().future.map(_ => Left(customStatusCode))
+ }
+ }
+ }
+ }
+}
+
+private class NonEscapingClient(protocol: String,
+ host: String,
+ port: Int,
+ username: String,
+ password: String,
+ db: String)(implicit system: ActorSystem,
logging: Logging)
+ extends CouchDbRestClient(protocol, host, port, username, password, db) {
+
+ //Do not escape the design doc id like _design/subjects etc
+ override protected def uri(segments: Any*): Uri =
Uri(s"/${segments.mkString("/")}")
+}
diff --git
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneDockerSupport.scala
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneDockerSupport.scala
index cf95c27..05e7a1e 100644
---
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneDockerSupport.scala
+++
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneDockerSupport.scala
@@ -26,7 +26,12 @@ import akka.actor.{ActorSystem, CoordinatedShutdown}
import org.apache.commons.lang3.SystemUtils
import org.apache.openwhisk.common.{Logging, TransactionId}
import org.apache.openwhisk.core.ConfigKeys
-import org.apache.openwhisk.core.containerpool.docker.{DockerClient,
DockerClientConfig, WindowsDockerClient}
+import org.apache.openwhisk.core.containerpool.docker.{
+ BrokenDockerContainer,
+ DockerClient,
+ DockerClientConfig,
+ WindowsDockerClient
+}
import org.apache.openwhisk.core.containerpool.{ContainerAddress, ContainerId}
import pureconfig.{loadConfig, loadConfigOrThrow}
@@ -171,6 +176,28 @@ class StandaloneDockerClient(implicit log: Logging, as:
ActorSystem, ec: Executi
super.runCmd(args, timeout)
val clientConfig: DockerClientConfig =
loadConfigOrThrow[DockerClientConfig](ConfigKeys.dockerClient)
+
+ def runDetached(image: String, args: Seq[String], shouldPull: Boolean)(
+ implicit tid: TransactionId): Future[StandaloneDockerContainer] = {
+ for {
+ _ <- if (shouldPull) pull(image) else Future.successful(())
+ id <- run(image, args).recoverWith {
+ case t @ BrokenDockerContainer(brokenId, _) =>
+ // Remove the broken container - but don't wait or check for the
result.
+ // If the removal fails, there is nothing we could do to recover
from the recovery.
+ rm(brokenId)
+ Future.failed(t)
+ case t => Future.failed(t)
+ }
+ ip <- inspectIPAddress(id, StandaloneDockerSupport.network).recoverWith {
+ // remove the container immediately if inspect failed as
+ // we cannot recover that case automatically
+ case e =>
+ rm(id)
+ Future.failed(e)
+ }
+ } yield StandaloneDockerContainer(id, ip)
+ }
}
case class StandaloneDockerContainer(id: ContainerId, addr: ContainerAddress)
diff --git
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala
index 1181a15..e76bc4d 100644
---
a/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala
+++
b/core/standalone/src/main/scala/org/apache/openwhisk/standalone/StandaloneOpenWhisk.scala
@@ -55,6 +55,8 @@ class Conf(arguments: Seq[String]) extends
ScallopConf(arguments) {
val verbose = tally()
val disableColorLogging = opt[Boolean](descr = "Disables colored logging",
noshort = true)
val apiGw = opt[Boolean](descr = "Enable API Gateway support", noshort =
true)
+ val couchdb = opt[Boolean](descr = "Enable CouchDB support", noshort = true)
+ val clean = opt[Boolean](descr = "Clean any existing state like database",
noshort = true)
val apiGwPort = opt[Int](descr = "Api Gateway Port", default = Some(3234),
noshort = true)
val dataDir = opt[File](descr = "Directory used for storage", default =
Some(StandaloneOpenWhisk.defaultWorkDir))
@@ -71,6 +73,7 @@ object StandaloneConfigKeys {
val usersConfigKey = "whisk.users"
val redisConfigKey = "whisk.standalone.redis"
val apiGwConfigKey = "whisk.standalone.api-gateway"
+ val couchDBConfigKey = "whisk.standalone.couchdb"
}
object StandaloneOpenWhisk extends SLF4JLogging {
@@ -134,10 +137,14 @@ object StandaloneOpenWhisk extends SLF4JLogging {
implicit val ec: ExecutionContext = actorSystem.dispatcher
val (dataDir, workDir) = initializeDirs(conf)
- val (apiGwApiPort, svcs) = if (conf.apiGw()) {
- startApiGateway(conf)
+ val (dockerClient, dockerSupport) = prepareDocker()
+
+ val (apiGwApiPort, apiGwSvcs) = if (conf.apiGw()) {
+ startApiGateway(conf, dockerClient, dockerSupport)
} else (-1, Seq.empty)
+ val couchSvcs = if (conf.couchdb()) Some(startCouchDb(dataDir,
dockerClient)) else None
+ val svcs = Seq(apiGwSvcs, couchSvcs.toList).flatten
if (svcs.nonEmpty) {
new ServiceInfoLogger(conf, svcs, dataDir).run()
}
@@ -288,8 +295,21 @@ object StandaloneOpenWhisk extends SLF4JLogging {
new ColoredAkkaLogging(adapter)
}
- private def startApiGateway(
- conf: Conf)(implicit logging: Logging, as: ActorSystem, ec:
ExecutionContext): (Int, Seq[ServiceContainer]) = {
+ private def prepareDocker()(implicit logging: Logging,
+ as: ActorSystem,
+ ec: ExecutionContext): (StandaloneDockerClient,
StandaloneDockerSupport) = {
+ val dockerClient = new StandaloneDockerClient()
+ val dockerSupport = new StandaloneDockerSupport(dockerClient)
+
+ //Remove any existing launched containers
+ dockerSupport.cleanup()
+ (dockerClient, dockerSupport)
+ }
+
+ private def startApiGateway(conf: Conf, dockerClient:
StandaloneDockerClient, dockerSupport: StandaloneDockerSupport)(
+ implicit logging: Logging,
+ as: ActorSystem,
+ ec: ExecutionContext): (Int, Seq[ServiceContainer]) = {
implicit val tid: TransactionId = TransactionId(systemPrefix + "apiMgmt")
// api port is the port used by rout management actions to configure the
api gw upon wsk api commands
@@ -297,11 +317,6 @@ object StandaloneOpenWhisk extends SLF4JLogging {
val apiGwApiPort = StandaloneDockerSupport.checkOrAllocatePort(9000)
val apiGwMgmtPort = conf.apiGwPort()
- val dockerClient = new StandaloneDockerClient()
- val dockerSupport = new StandaloneDockerSupport(dockerClient)
-
- //Remove any existing launched containers
- dockerSupport.cleanup()
val gw = new ApiGwLauncher(dockerClient, apiGwApiPort, apiGwMgmtPort,
conf.port())
val f = gw.run()
val g = f.andThen {
@@ -326,10 +341,14 @@ object StandaloneOpenWhisk extends SLF4JLogging {
installer.run()
}
- private def initializeDirs(conf: Conf): (File, File) = {
+ private def initializeDirs(conf: Conf)(implicit logging: Logging): (File,
File) = {
val baseDir = conf.dataDir()
val thisServerDir = s"server-${conf.port()}"
val dataDir = new File(baseDir, thisServerDir)
+ if (conf.clean() && dataDir.exists()) {
+ FileUtils.deleteDirectory(dataDir)
+ logging.info(this, s"Cleaned existing directory
${dataDir.getAbsolutePath}")
+ }
FileUtils.forceMkdir(dataDir)
log.info(s"Using [${dataDir.getAbsolutePath}] as data directory")
@@ -343,4 +362,23 @@ object StandaloneOpenWhisk extends SLF4JLogging {
val m = loadConfigOrThrow[Map[String,
String]](StandaloneConfigKeys.usersConfigKey)
m.map { case (name, key) => (name.replace('-', '.'), key) }
}
+
+ private def startCouchDb(dataDir: File, dockerClient:
StandaloneDockerClient)(
+ implicit logging: Logging,
+ as: ActorSystem,
+ ec: ExecutionContext,
+ materializer: ActorMaterializer): ServiceContainer = {
+ implicit val tid: TransactionId = TransactionId(systemPrefix + "couchDB")
+ val port = StandaloneDockerSupport.checkOrAllocatePort(5984)
+ val dbDataDir = new File(dataDir, "couchdb")
+ FileUtils.forceMkdir(dbDataDir)
+ val db = new CouchDBLauncher(dockerClient, port, dbDataDir)
+ val f = db.run()
+ val g = f.andThen {
+ case Success(_) =>
+ case Failure(t) =>
+ logging.error(this, "Error starting CouchDB" + t)
+ }
+ Await.result(g, 5.minutes)
+ }
}
diff --git
a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneCouchTests.scala
b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneCouchTests.scala
new file mode 100644
index 0000000..d2214f7
--- /dev/null
+++
b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneCouchTests.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.standalone
+
+import common.WskProps
+import org.apache.commons.io.{FileUtils, FilenameUtils}
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.scalatest.{Canceled, Outcome}
+import system.basic.WskRestBasicTests
+
+@RunWith(classOf[JUnitRunner])
+class StandaloneCouchTests extends WskRestBasicTests with
StandaloneServerFixture {
+ override implicit val wskprops = WskProps().copy(apihost = serverUrl)
+
+ override protected def extraArgs: Seq[String] =
+ Seq("--couchdb", "--data-dir",
FilenameUtils.concat(FileUtils.getTempDirectoryPath, "standalone"))
+
+ override protected def extraVMArgs: Seq[String] =
Seq("-Dwhisk.standalone.couchdb.volumes-enabled=false")
+
+ //This is more of a sanity test. So just run one of the test which trigger
interaction with couchdb
+ //and skip running all other tests
+ private val supportedTests = Set("Wsk Action REST should create, update, get
and list an action")
+
+ override def withFixture(test: NoArgTest): Outcome = {
+ if (supportedTests.contains(test.name)) {
+ super.withFixture(test)
+ } else {
+ Canceled()
+ }
+ }
+}
diff --git
a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala
b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala
index ebfa109..fa0efe8 100644
---
a/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala
+++
b/tests/src/test/scala/org/apache/openwhisk/standalone/StandaloneServerFixture.scala
@@ -48,6 +48,7 @@ trait StandaloneServerFixture extends TestSuite with
BeforeAndAfterAll with Stre
private val whiskServerPreDefined = System.getProperty(WHISK_SERVER) != null
protected def extraArgs: Seq[String] = Seq.empty
+ protected def extraVMArgs: Seq[String] = Seq.empty
protected def waitForOtherThings(): Unit = {}
@@ -68,10 +69,10 @@ trait StandaloneServerFixture extends TestSuite with
BeforeAndAfterAll with Stre
//For tests let it bound on all ip to make it work on travis which
uses linux
"-Dwhisk.controller.interface=0.0.0.0",
s"-Dwhisk.standalone.wsk=${Wsk.defaultCliPath}",
- s"-D$disablePullConfig=false",
- "-jar",
- standaloneServerJar.getAbsolutePath,
- "--disable-color-logging") ++ extraArgs,
+ s"-D$disablePullConfig=false")
+ ++ extraVMArgs
+ ++ Seq("-jar", standaloneServerJar.getAbsolutePath,
"--disable-color-logging")
+ ++ extraArgs,
Seq("-p", serverPort.toString),
manifestFile.map(f => Seq("-m",
f.getAbsolutePath)).getOrElse(Seq.empty)).flatten
