dubee closed pull request #2847: Treat action code as attachments for
created/updated actions
URL: https://github.com/apache/incubator-openwhisk/pull/2847
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ansible/files/runtimes.json b/ansible/files/runtimes.json
index 1866b69195..44cef222b3 100644
--- a/ansible/files/runtimes.json
+++ b/ansible/files/runtimes.json
@@ -6,7 +6,11 @@
"image": {
"name": "nodejsaction"
},
- "deprecated": true
+ "deprecated": true,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "nodejs:6",
@@ -14,7 +18,11 @@
"image": {
"name": "nodejs6action"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "nodejs:8",
@@ -22,7 +30,11 @@
"image": {
"name": "action-nodejs-v8"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
}
],
"python": [
@@ -31,7 +43,11 @@
"image": {
"name": "python2action"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "python:2",
@@ -39,14 +55,22 @@
"image": {
"name": "python2action"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "python:3",
"image": {
"name": "python3action"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
}
],
"swift": [
@@ -55,21 +79,33 @@
"image": {
"name": "swiftaction"
},
- "deprecated": true
+ "deprecated": true,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "swift:3",
"image": {
"name": "swift3action"
},
- "deprecated": true
+ "deprecated": true,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "swift:3.1.1",
"image": {
"name": "action-swift-v3.1.1"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
},
{
"kind": "swift:4.1",
@@ -77,7 +113,11 @@
"image": {
"name": "action-swift-v4.1"
},
- "deprecated": false
+ "deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ }
}
],
"java": [
@@ -89,8 +129,8 @@
},
"deprecated": false,
"attached": {
- "attachmentName": "jarfile",
- "attachmentType": "application/java-archive"
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
},
"sentinelledLogs": false,
"requireMain": true
@@ -101,6 +141,10 @@
"kind": "php:7.1",
"default": true,
"deprecated": false,
+ "attached": {
+ "attachmentName": "codefile",
+ "attachmentType": "text/plain"
+ },
"image": {
"name": "action-php-v7.1"
}
diff --git a/common/scala/src/main/scala/whisk/core/entity/Exec.scala
b/common/scala/src/main/scala/whisk/core/entity/Exec.scala
index 8f318129d0..77bc8cefe0 100644
--- a/common/scala/src/main/scala/whisk/core/entity/Exec.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/Exec.scala
@@ -30,6 +30,8 @@ import whisk.core.entity.size.SizeInt
import whisk.core.entity.size.SizeOptionString
import whisk.core.entity.size.SizeString
+import java.util.Base64
+
/**
* Exec encodes the executable details of an action. For black
* box container, an image name is required. For Javascript and Python
@@ -140,19 +142,24 @@ protected[core] case class
CodeExecMetaDataAsString(manifest: RuntimeManifest,
protected[core] case class CodeExecAsAttachment(manifest: RuntimeManifest,
override val code:
Attachment[String],
- override val entryPoint:
Option[String])
+ override val entryPoint:
Option[String],
+ override val binary: Boolean =
false)
extends CodeExec[Attachment[String]] {
override val kind = manifest.kind
override val image = manifest.image
override val sentinelledLogs = manifest.sentinelledLogs.getOrElse(true)
override val deprecated = manifest.deprecated.getOrElse(false)
override val pull = false
- override lazy val binary = true
override def codeAsJson = code.toJson
def inline(bytes: Array[Byte]): CodeExecAsAttachment = {
- val encoded = new String(bytes, StandardCharsets.UTF_8)
- copy(code = Inline(encoded))
+ val code = new String(bytes, StandardCharsets.UTF_8)
+
+ if (kind == "java" && !Exec.isBinaryCode(code)) {
+ val encoded = Base64.getEncoder.encodeToString(bytes)
+ copy(code = Inline(encoded))
+ } else
+ copy(code = Inline(code))
}
def attach: CodeExecAsAttachment = {
@@ -304,21 +311,29 @@ protected[core] object Exec extends ArgNormalizer[Exec]
with DefaultJsonProtocol
manifest.attached
.map { a =>
- val jar: Attachment[String] = {
- // java actions once stored the attachment in "jar" instead of
"code"
- obj.fields.get("code").orElse(obj.fields.get("jar"))
+ val code = obj.fields.get("code")
+ val binary: Boolean = code match {
+ case Some(JsString(c)) => isBinaryCode(c)
+ case _ =>
+ obj.fields.get("binary") match {
+ case Some(JsBoolean(b)) => b
+ case _ => false
+ }
+ }
+ val attachment: Attachment[String] = {
+ code
} map {
attFmt[String].read(_)
} getOrElse {
- throw new DeserializationException(
- s"'code' must be a valid base64 string in 'exec' for '$kind'
actions")
+ throw new DeserializationException(s"'code' must be a string
defined in 'exec' for '$kind' actions")
}
val main = optMainField.orElse {
if (manifest.requireMain.exists(identity)) {
throw new DeserializationException(s"'main' must be a string
defined in 'exec' for '$kind' actions")
} else None
}
- CodeExecAsAttachment(manifest, jar, main)
+
+ CodeExecAsAttachment(manifest, attachment, main, binary)
}
.getOrElse {
val code: String = obj.fields.get("code") match {
diff --git a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
index 97f98ae315..c96c51694f 100644
--- a/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
+++ b/common/scala/src/main/scala/whisk/core/entity/WhiskAction.scala
@@ -19,7 +19,6 @@ package whisk.core.entity
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
-import java.util.Base64
import scala.concurrent.ExecutionContext
import scala.concurrent.Future
@@ -326,14 +325,14 @@ object WhiskAction extends DocumentFactory[WhiskAction]
with WhiskEntityQueries[
require(doc != null, "doc undefined")
} map { _ =>
doc.exec match {
- case exec @ CodeExecAsAttachment(_, Inline(code), _) =>
+ case exec @ CodeExecAsAttachment(_, Inline(code), _, _) =>
implicit val logger = db.logging
implicit val ec = db.executionContext
val newDoc = doc.copy(exec = exec.attach)
newDoc.revision(doc.rev)
- val stream = new
ByteArrayInputStream(Base64.getDecoder().decode(code))
+ val stream = new ByteArrayInputStream(code.getBytes("UTF-8"))
val manifest = exec.manifest.attached.get
for (i1 <- super.put(db, newDoc);
@@ -362,12 +361,10 @@ object WhiskAction extends DocumentFactory[WhiskAction]
with WhiskEntityQueries[
fa.flatMap { action =>
action.exec match {
- case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _), _) =>
+ case exec @ CodeExecAsAttachment(_, Attached(attachmentName, _), _, _)
=>
val boas = new ByteArrayOutputStream()
- val b64s = Base64.getEncoder().wrap(boas)
- getAttachment[A](db, action.docinfo, attachmentName, b64s).map { _ =>
- b64s.close()
+ getAttachment[A](db, action.docinfo, attachmentName, boas).map { _ =>
val newAction = action.copy(exec = exec.inline(boas.toByteArray))
newAction.revision(action.rev)
newAction
diff --git
a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
index 380c5f0b76..3b0af6376b 100644
--- a/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
+++ b/tests/src/test/scala/whisk/core/controller/test/ActionsApiTests.scala
@@ -742,6 +742,147 @@ class ActionsApiTests extends ControllerTestCommon with
WhiskActionsApi {
}
}
+ it should "put an action and ensure its code is treated as an attachment" in
{
+ val javaAction =
+ WhiskAction(namespace, aname(), javaDefault("ZHViZWU=", Some("hello")),
annotations = Parameters("exec", "java"))
+ val nodeAction = WhiskAction(namespace, aname(), jsDefault("??"),
Parameters("x", "b"))
+ val swiftAction = WhiskAction(namespace, aname(), swift3("??"),
Parameters("x", "b"))
+ val actions = Seq((javaAction, JAVA_DEFAULT), (nodeAction, NODEJS6),
(swiftAction, SWIFT3))
+
+ actions.foreach {
+ case (action, kind) =>
+ val content = WhiskActionPut(
+ Some(action.exec),
+ Some(action.parameters),
+ Some(ActionLimitsOption(Some(action.limits.timeout),
Some(action.limits.memory), Some(action.limits.logs))))
+ val cacheKey = s"${CacheKey(action)}".replace("(", "\\(").replace(")",
"\\)")
+ val expectedPutLog = Seq(
+ s"caching $cacheKey",
+ s"uploading attachment 'codefile' of document 'id:
${action.namespace}/${action.name}")
+ .mkString("(?s).*")
+ val expectedGetLog =
+ Seq(s"finding attachment 'codefile' of document 'id:
${action.namespace}/${action.name}").mkString("(?s).*")
+
+ // first request invalidates any previous entries and caches new result
+ Put(s"$collectionPath/${action.name}", content) ~>
Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName,
kind)))
+ }
+ stream.toString should include regex (expectedPutLog)
+ stream.reset()
+
+ // second request should fetch from cache
+ Get(s"$collectionPath/${action.name}") ~>
Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName,
kind)))
+ }
+ stream.toString should include regex (expectedGetLog)
+ stream.reset()
+
+ // delete should invalidate cache
+ Delete(s"$collectionPath/${action.name}") ~>
Route.seal(routes(creds)(transid())) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ action.namespace,
+ action.name,
+ action.exec,
+ action.parameters,
+ action.limits,
+ action.version,
+ action.publish,
+ action.annotations ++ Parameters(WhiskAction.execFieldName,
kind)))
+ }
+ }
+ }
+
+ it should "ensure old and new action schemas are supported" in {
+ implicit val tid = transid()
+ val actionOldSchema = WhiskAction(namespace, aname(), js6Old("??"))
+ val actionNewSchema = WhiskAction(namespace, aname(), jsDefault("??"))
+ val content = WhiskActionPut(
+ Some(actionOldSchema.exec),
+ Some(actionOldSchema.parameters),
+ Some(
+ ActionLimitsOption(
+ Some(actionOldSchema.limits.timeout),
+ Some(actionOldSchema.limits.memory),
+ Some(actionOldSchema.limits.logs))))
+ val expectedPutLog =
+ Seq(s"uploading attachment 'codefile' of document 'id:
${actionOldSchema.namespace}/${actionOldSchema.name}")
+ .mkString("(?s).*")
+
+ put(entityStore, actionOldSchema)
+
+ stream.toString should not include regex(expectedPutLog)
+ stream.reset()
+
+ Post(s"$collectionPath/${actionOldSchema.name}") ~>
Route.seal(routes(creds)) ~> check {
+ status should be(Accepted)
+ val response = responseAs[JsObject]
+ response.fields("activationId") should not be None
+ }
+
+ Put(s"$collectionPath/${actionOldSchema.name}?overwrite=true", content) ~>
Route.seal(routes(creds)) ~> check {
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ actionOldSchema.namespace,
+ actionOldSchema.name,
+ actionNewSchema.exec,
+ actionOldSchema.parameters,
+ actionOldSchema.limits,
+ actionOldSchema.version.upPatch,
+ actionOldSchema.publish,
+ actionOldSchema.annotations ++ Parameters(WhiskAction.execFieldName,
NODEJS6)))
+ }
+
+ stream.toString should include regex (expectedPutLog)
+ stream.reset()
+
+ Post(s"$collectionPath/${actionOldSchema.name}") ~>
Route.seal(routes(creds)) ~> check {
+ status should be(Accepted)
+ val response = responseAs[JsObject]
+ response.fields("activationId") should not be None
+ }
+
+ Delete(s"$collectionPath/${actionOldSchema.name}") ~>
Route.seal(routes(creds)) ~> check {
+ status should be(OK)
+ val response = responseAs[WhiskAction]
+ response should be(
+ WhiskAction(
+ actionOldSchema.namespace,
+ actionOldSchema.name,
+ actionNewSchema.exec,
+ actionOldSchema.parameters,
+ actionOldSchema.limits,
+ actionOldSchema.version.upPatch,
+ actionOldSchema.publish,
+ actionOldSchema.annotations ++ Parameters(WhiskAction.execFieldName,
NODEJS6)))
+ }
+ }
+
it should "reject put with conflict for pre-existing action" in {
implicit val tid = transid()
val action = WhiskAction(namespace, aname(), jsDefault("??"),
Parameters("x", "b"))
diff --git
a/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
b/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
index d2cac40438..7c3a630976 100644
--- a/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
+++ b/tests/src/test/scala/whisk/core/database/test/CacheConcurrencyTests.scala
@@ -127,7 +127,11 @@ class CacheConcurrencyTests extends FlatSpec with
WskTestHelpers with BeforeAndA
para.tasksupport = new ForkJoinTaskSupport(new ForkJoinPool(nThreads))
para.map { i =>
if (i != 16) {
- wsk.action.get(name)
+ val rr = wsk.action.get(name, expectedExitCode = DONTCARE_EXIT)
+ withClue(s"expecting get to either succeed or fail with not found:
$rr") {
+ // some will succeed and some should fail with not found
+ rr.exitCode should (be(SUCCESS_EXIT) or be(NOT_FOUND))
+ }
} else {
wsk.action.create(name, None, parameters = Map("color" ->
JsString("blue")), update = true)
}
diff --git a/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
b/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
index 688fcd5b7e..3bee1dc8d7 100644
--- a/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
+++ b/tests/src/test/scala/whisk/core/entity/test/ExecHelpers.scala
@@ -53,33 +53,52 @@ trait ExecHelpers extends Matchers with WskActorSystem with
StreamLogging {
ExecManifest.ImageName(image, Some("openwhisk"), Some("latest"))
}
- protected def js(code: String, main: Option[String] = None) = {
+ protected def jsOld(code: String, main: Option[String] = None) = {
CodeExecAsString(RuntimeManifest(NODEJS, imagename(NODEJS), deprecated =
Some(true)), trim(code), main.map(_.trim))
}
- protected def js6(code: String, main: Option[String] = None) = {
+ protected def js(code: String, main: Option[String] = None) = {
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest =
ExecManifest.runtimesManifest.resolveDefaultRuntime(NODEJS).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim),
Exec.isBinaryCode(code))
+ }
+
+ protected def js6Old(code: String, main: Option[String] = None) = {
CodeExecAsString(
RuntimeManifest(NODEJS6, imagename(NODEJS6), default = Some(true),
deprecated = Some(false)),
trim(code),
main.map(_.trim))
}
+ protected def js6(code: String, main: Option[String] = None) = {
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest =
ExecManifest.runtimesManifest.resolveDefaultRuntime(NODEJS6).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim),
Exec.isBinaryCode(code))
+ }
protected def jsDefault(code: String, main: Option[String] = None) = {
js6(code, main)
}
- protected def js6MetaData(main: Option[String] = None, binary: Boolean) = {
+ protected def js6MetaDataOld(main: Option[String] = None, binary: Boolean) =
{
CodeExecMetaDataAsString(
RuntimeManifest(NODEJS6, imagename(NODEJS6), default = Some(true),
deprecated = Some(false)),
binary,
main.map(_.trim))
}
+ protected def js6MetaData(main: Option[String] = None, binary: Boolean) = {
+ val manifest =
ExecManifest.runtimesManifest.resolveDefaultRuntime(NODEJS6).get
+
+ CodeExecMetaDataAsAttachment(manifest, binary, main.map(_.trim))
+ }
+
protected def javaDefault(code: String, main: Option[String] = None) = {
val attachment = attFmt[String].read(code.trim.toJson)
val manifest =
ExecManifest.runtimesManifest.resolveDefaultRuntime(JAVA_DEFAULT).get
- CodeExecAsAttachment(manifest, attachment, main.map(_.trim))
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim),
Exec.isBinaryCode(code))
}
protected def javaMetaData(main: Option[String] = None, binary: Boolean) = {
@@ -88,16 +107,22 @@ trait ExecHelpers extends Matchers with WskActorSystem
with StreamLogging {
CodeExecMetaDataAsAttachment(manifest, binary, main.map(_.trim))
}
- protected def swift(code: String, main: Option[String] = None) = {
+ protected def swiftOld(code: String, main: Option[String] = None) = {
CodeExecAsString(RuntimeManifest(SWIFT, imagename(SWIFT), deprecated =
Some(true)), trim(code), main.map(_.trim))
}
+ protected def swift(code: String, main: Option[String] = None) = {
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest =
ExecManifest.runtimesManifest.resolveDefaultRuntime(SWIFT).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim),
Exec.isBinaryCode(code))
+ }
+
protected def swift3(code: String, main: Option[String] = None) = {
- val default =
ExecManifest.runtimesManifest.resolveDefaultRuntime(SWIFT3).flatMap(_.default)
- CodeExecAsString(
- RuntimeManifest(SWIFT3, imagename(SWIFT3), default = default, deprecated
= Some(false)),
- trim(code),
- main.map(_.trim))
+ val attachment = attFmt[String].read(code.trim.toJson)
+ val manifest =
ExecManifest.runtimesManifest.resolveDefaultRuntime(SWIFT3).get
+
+ CodeExecAsAttachment(manifest, attachment, main.map(_.trim),
Exec.isBinaryCode(code))
}
protected def sequence(components: Vector[FullyQualifiedEntityName]) =
SequenceExec(components)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
