jeremiaswerner closed pull request #2870: Remove broken containers if docker 
run fails
URL: https://github.com/apache/incubator-openwhisk/pull/2870
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git 
a/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerClient.scala
 
b/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerClient.scala
index d714a9fbaa..bf943252aa 100644
--- 
a/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerClient.scala
+++ 
b/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerClient.scala
@@ -35,6 +35,18 @@ import scala.collection.concurrent.TrieMap
 import whisk.core.containerpool.ContainerId
 import whisk.core.containerpool.ContainerAddress
 
+object DockerContainerId {
+
+  val containerIdRegex = """^([0-9a-f]{64})$""".r
+
+  def parse(id: String): Try[ContainerId] = {
+    id match {
+      case containerIdRegex(_) => Success(ContainerId(id))
+      case _                   => Failure(new IllegalArgumentException(s"Does 
not comply with Docker container ID format: ${id}"))
+    }
+  }
+}
+
 /**
  * Serves as interface to the docker CLI tool.
  *
@@ -63,8 +75,26 @@ class DockerClient(dockerHost: Option[String] = 
None)(executionContext: Executio
     Seq(dockerBin) ++ host
   }
 
-  def run(image: String, args: Seq[String] = Seq.empty[String])(implicit 
transid: TransactionId): Future[ContainerId] =
-    runCmd((Seq("run", "-d") ++ args ++ Seq(image)): _*).map(ContainerId.apply)
+  def run(image: String, args: Seq[String] = Seq.empty[String])(
+    implicit transid: TransactionId): Future[ContainerId] = {
+    runCmd((Seq("run", "-d") ++ args ++ Seq(image)): _*)
+      .map {
+        ContainerId(_)
+      }
+      .recoverWith {
+        // https://docs.docker.com/v1.12/engine/reference/run/#/exit-status
+        // Exit code 125 means an error reported by the Docker daemon.
+        // Examples:
+        // - Unrecognized option specified
+        // - Not enough disk space
+        case pre: ProcessRunningException if pre.exitCode == 125 =>
+          Future.failed(
+            DockerContainerId
+              .parse(pre.stdout)
+              .map(BrokenDockerContainer(_, s"Broken container: 
${pre.getMessage}"))
+              .getOrElse(pre))
+      }
+  }
 
   def inspectIPAddress(id: ContainerId, network: String)(implicit transid: 
TransactionId): Future[ContainerAddress] =
     runCmd("inspect", "--format", 
s"{{.NetworkSettings.Networks.${network}.IPAddress}}", id.asString).flatMap {
@@ -190,3 +220,6 @@ trait DockerApi {
    */
   def isOomKilled(id: ContainerId)(implicit transid: TransactionId): 
Future[Boolean]
 }
+
+/** Indicates any error while starting a container that leaves a broken 
container behind that needs to be removed */
+case class BrokenDockerContainer(id: ContainerId, msg: String) extends 
Exception(msg)
diff --git 
a/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainer.scala
 
b/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainer.scala
index a7b17851d5..4827eb5843 100644
--- 
a/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainer.scala
+++ 
b/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainer.scala
@@ -98,7 +98,14 @@ object DockerContainer {
     for {
       _ <- pulled
       id <- docker.run(image, args).recoverWith {
-        case _ => Future.failed(WhiskContainerStartupError(s"Failed to run 
container with image '${image}'."))
+        case BrokenDockerContainer(brokenId, message) =>
+          // Remove the broken container - but don't wait or check for the 
result.
+          // If the removal fails, there is nothing we could do to recover 
from the recovery.
+          docker.rm(brokenId)
+          Future.failed(
+            WhiskContainerStartupError(s"Failed to run container with image 
'${image}'. Removing broken container."))
+        case _ =>
+          Future.failed(WhiskContainerStartupError(s"Failed to run container 
with image '${image}'."))
       }
       ip <- docker.inspectIPAddress(id, network).recoverWith {
         // remove the container immediately if inspect failed as
diff --git 
a/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
 
b/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
index 1235b9220c..8f447cfaa6 100644
--- 
a/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
+++ 
b/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerClientTests.scala
@@ -23,6 +23,8 @@ import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 import scala.concurrent.duration.DurationInt
 import scala.concurrent.duration.FiniteDuration
+import scala.concurrent.Promise
+import scala.util.Success
 import org.junit.runner.RunWith
 import org.scalatest.BeforeAndAfterEach
 import org.scalatest.FlatSpec
@@ -33,10 +35,12 @@ import whisk.common.LogMarker
 import whisk.common.LoggingMarkers.INVOKER_DOCKER_CMD
 import whisk.common.TransactionId
 import whisk.core.containerpool.docker.DockerClient
-import scala.concurrent.Promise
 import whisk.core.containerpool.ContainerId
 import whisk.core.containerpool.ContainerAddress
 import whisk.utils.retry
+import whisk.core.containerpool.docker.ProcessRunningException
+import whisk.core.containerpool.docker.DockerContainerId
+import whisk.core.containerpool.docker.BrokenDockerContainer
 
 @RunWith(classOf[JUnitRunner])
 class DockerClientTests extends FlatSpec with Matchers with StreamLogging with 
BeforeAndAfterEach {
@@ -44,7 +48,7 @@ class DockerClientTests extends FlatSpec with Matchers with 
StreamLogging with B
   override def beforeEach = stream.reset()
 
   implicit val transid = TransactionId.testing
-  val id = ContainerId("Id")
+  val id = 
ContainerId("55db56ee082239428b27d3728b4dd324c09068458aad9825727d5bfc1bba6d52")
 
   val commandTimeout = 500.milliseconds
   def await[A](f: Future[A], timeout: FiniteDuration = commandTimeout) = 
Await.result(f, timeout)
@@ -57,6 +61,31 @@ class DockerClientTests extends FlatSpec with Matchers with 
StreamLogging with B
     override def executeProcess(args: String*)(implicit ec: ExecutionContext) 
= execResult
   }
 
+  behavior of "DockerContainerId"
+
+  it should "convert a proper container ID" in {
+    DockerContainerId.parse(id.asString) shouldBe Success(id)
+  }
+
+  it should "reject improper container IDs with IllegalArgumentException" in {
+    def verifyFailure(improperId: String) = {
+      val iae = the[IllegalArgumentException] thrownBy 
DockerContainerId.parse(improperId).get
+      iae.getMessage should include(improperId)
+    }
+
+    Seq[(String, String)](
+      ("", "String empty (too short)"),
+      ("1" * 63, "String too short"),
+      ("1" * 65, "String too long"),
+      (("1" * 63) + "x", "Improper characters"),
+      ("abcxdef", "Improper characters and too short")).foreach {
+      case (improperId, clue) =>
+        withClue(s"${clue} - length('${improperId}') = ${improperId.length}: 
") {
+          verifyFailure(improperId)
+        }
+    }
+  }
+
   behavior of "DockerClient"
 
   it should "return a list of containers and pass down the correct arguments 
when using 'ps'" in {
@@ -201,4 +230,33 @@ class DockerClientTests extends FlatSpec with Matchers 
with StreamLogging with B
     runAndVerify(dc.run("image"), "run")
     runAndVerify(dc.pull("image"), "pull")
   }
+
+  it should "fail with BrokenDockerContainer when run returns with exit code 
125 and a container ID" in {
+    val dc = dockerClient {
+      Future.failed(
+        ProcessRunningException(
+          exitCode = 125,
+          stdout = id.asString,
+          stderr =
+            """/usr/bin/docker: Error response from daemon: mkdir 
/var/run/docker.1.1/libcontainerd.1.1/55db56ee082239428b27d3728b4dd324c09068458aad9825727d5bfc1bba6d52:
 no space left on device."""))
+    }
+    val bdc = the[BrokenDockerContainer] thrownBy await(dc.run("image", Seq()))
+    bdc.id shouldBe id
+  }
+
+  it should "fail with ProcessRunningException when run returns with exit code 
!=125 or no container ID" in {
+    def runAndVerify(pre: ProcessRunningException, clue: String) = {
+      val dc = dockerClient { Future.failed(pre) }
+      withClue(s"${clue} - exitCode = ${pre.exitCode}, stdout = 
'${pre.stdout}', stderr = '${pre.stderr}': ") {
+        the[ProcessRunningException] thrownBy await(dc.run("image", Seq())) 
shouldBe pre
+      }
+    }
+
+    Seq[(ProcessRunningException, String)](
+      (ProcessRunningException(126, id.asString, "Unknown command"), "Exit 
code not 125"),
+      (ProcessRunningException(125, "", "Unknown flag: --foo"), "No container 
ID"),
+      (ProcessRunningException(1, "", ""), "Exit code not 125 and no container 
ID")).foreach {
+      case (pre, clue) => runAndVerify(pre, clue)
+    }
+  }
 }
diff --git 
a/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerContainerTests.scala
 
b/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerContainerTests.scala
index af16af303c..8ded76e0d3 100644
--- 
a/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerContainerTests.scala
+++ 
b/tests/src/test/scala/whisk/core/containerpool/docker/test/DockerContainerTests.scala
@@ -189,7 +189,7 @@ class DockerContainerTests
       override def run(image: String,
                        args: Seq[String] = Seq.empty[String])(implicit 
transid: TransactionId): Future[ContainerId] = {
         runs += ((image, args))
-        Future.failed(new RuntimeException())
+        Future.failed(ProcessRunningException(1, "", ""))
       }
     }
     implicit val runc = stub[RuncApi]
@@ -207,6 +207,29 @@ class DockerContainerTests
     docker.rms should have size 0
   }
 
+  it should "remove the container if run fails with a broken container" in {
+    implicit val docker = new TestDockerClient {
+      override def run(image: String,
+                       args: Seq[String] = Seq.empty[String])(implicit 
transid: TransactionId): Future[ContainerId] = {
+        runs += ((image, args))
+        Future.failed(BrokenDockerContainer(containerId, "Broken container"))
+      }
+    }
+    implicit val runc = stub[RuncApi]
+
+    val container = DockerContainer.create(
+      transid = transid,
+      image = "image",
+      userProvidedImage = false,
+      dockerRunParameters = parameters)
+    a[WhiskContainerStartupError] should be thrownBy await(container)
+
+    docker.pulls should have size 0
+    docker.runs should have size 1
+    docker.inspects should have size 0
+    docker.rms should have size 1
+  }
+
   it should "provide a proper error if inspect fails for blackbox containers" 
in {
     implicit val docker = new TestDockerClient {
       override def inspectIPAddress(id: ContainerId,


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
[email protected]


With regards,
Apache Git Services

Reply via email to