markusthoemmes commented on a change in pull request #2661: Add tests to check ha of controller URL: https://github.com/apache/incubator-openwhisk/pull/2661#discussion_r137987972
########## File path: tests/src/test/scala/ha/ShootComponentsTests.scala ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ha + +import java.io.File +import java.time.Instant + +import scala.concurrent.Future +import scala.concurrent.duration.DurationInt +import scala.util.Try + +import org.junit.runner.RunWith +import org.scalatest.FlatSpec +import org.scalatest.Matchers +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.junit.JUnitRunner + +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.HttpRequest +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer +import common.TestUtils +import common.WaitFor +import common.WhiskProperties +import common.Wsk +import common.WskActorSystem +import common.WskProps +import common.WskTestHelpers +import whisk.core.WhiskConfig +import whisk.utils.retry + +@RunWith(classOf[JUnitRunner]) +class ShootComponentsTests + extends FlatSpec + with Matchers + with WaitFor + with WskTestHelpers + with ScalaFutures + with WskActorSystem { + + implicit val wskprops = WskProps() + val wsk = new Wsk + val defaultAction = Some(TestUtils.getTestActionFilename("hello.js")) + + implicit val materializer = ActorMaterializer() + implicit val testConfig = PatienceConfig(1.minute) + + def restartComponent(host: String, component: String) = { + def file(path: String) = Try(new File(path)).filter(_.exists).map(_.getAbsolutePath).toOption + val docker = (file("/usr/bin/docker") orElse file("/usr/local/bin/docker")).getOrElse("docker") + + val cmd = Seq(docker, "--host", host, "restart", component) + println(s"Running command: ${cmd.mkString(" ")}") + + TestUtils.runCmd(0, new File("."), cmd: _*) + } + + def ping(host: String, port: Int) = { + val response = Try { Http().singleRequest(HttpRequest(uri = s"http://$host:$port/ping";)).futureValue }.toOption + + response.map { res => + (res.status, Unmarshal(res).to[String].futureValue) + } + } + + def isControllerAlive(instance: Int): Boolean = { + require(instance >= 0 && instance < 2, "Controller instance not known.") + + val host = WhiskProperties.getProperty("controller.hosts").split(",")(instance) + val port = WhiskProperties.getControllerBasePort + instance + + val res = ping(host, port) + res.isDefined && res.get._1 == StatusCodes.OK && res.get._2 == "pong" + } + + def doRequests(amount: Int, actionName: String): Seq[(Int, Int)] = { + (0 until amount).map { i => + val start = Instant.now + + // Do POSTs and GETs + val invokeExit = Future { wsk.action.invoke(actionName, expectedExitCode = TestUtils.DONTCARE_EXIT).exitCode } + val getExit = Future { wsk.action.get(actionName, expectedExitCode = TestUtils.DONTCARE_EXIT).exitCode } + + println(s"Done rerquests with responses: invoke: ${invokeExit.futureValue} and get: ${getExit.futureValue}") + + // Do at most one action invocation per second to avoid getting 429s. (60 req/min - limit) + val wait = 1000 - (Instant.now.toEpochMilli - start.toEpochMilli) + Thread.sleep(if (wait < 0) 0L else if (wait > 1000) 1000L else wait) + (invokeExit.futureValue, getExit.futureValue) + } + } + + behavior of "Controllers hot standby" + + it should "use controller1 if controller0 goes down" in withAssetCleaner(wskprops) { (wp, assetHelper) => + if (WhiskProperties.getProperty(WhiskConfig.controllerInstances).toInt >= 2) { + val actionName = "shootcontroller" + + assetHelper.withCleaner(wsk.action, actionName) { (action, _) => + action.create(actionName, defaultAction) + } + + // Produce some load on the system for 90 seconds. Kill the controller after 4 requests + val requestsBeforeRestart = doRequests(4, actionName) + + // Kill the controller + val dockerHost = WhiskProperties.getBaseControllerHost() + ":" + WhiskProperties.getProperty( + WhiskConfig.dockerPort) + restartComponent(dockerHost, "controller0") + // Wait until down + retry({ + isControllerAlive(0) shouldBe false + }, 100, Some(100.milliseconds)) + // Check that second controller is still up + isControllerAlive(1) shouldBe true + + val requestsAfterRestart = doRequests(96, actionName) Review comment: Would it make sense to formulate this as `100 - 4` (define a `val totalRequests = 100` above?) ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
