This is an automated email from the ASF dual-hosted git repository.
mrutkowski pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-openwhisk-devtools.git
The following commit(s) were added to refs/heads/master by this push:
new 6c50a64 Package action source along with its dependencies (#266)
6c50a64 is described below
commit 6c50a64602bda13cf302382359d49cb8dc4713f9
Author: Priti Desai <[email protected]>
AuthorDate: Fri Jun 28 07:22:03 2019 -0700
Package action source along with its dependencies (#266)
* read file if exists locally before downloading it
* base64 for zip contents
* adding marshal resources
* adding more changes for packaging
* adding more changes to package node
* adding more changes to package node
* adding more changes to package node
* adding marshalresource
* adding marshalresource
* packaging npm actions
* packaging npm actions
* packaging npm actions
* adding marshalresource
* packaging npm actions
* adding zip to runtime
* adding init run json payload
* removing debug statements
* updating build template
* updating build template
* checking if path exists
* adding production mode during npm install
---
knative-build/runtimes/javascript/Dockerfile | 1 +
.../runtimes/javascript/buildtemplate.yaml | 23 +++++--
.../runtimes/javascript/platform/knative.js | 78 ++++++++++++++++++++++
.../javascript/tests/packageaction/build.yaml.tmpl | 29 ++++++++
.../tests/packageaction/knative-data-init-run.json | 20 ++++++
.../tests/packageaction/knative-data-init.json | 8 +++
.../tests/packageaction/knative-data-run.json | 13 ++++
.../tests/packageaction/service.yaml.tmpl | 12 ++++
8 files changed, 177 insertions(+), 7 deletions(-)
diff --git a/knative-build/runtimes/javascript/Dockerfile
b/knative-build/runtimes/javascript/Dockerfile
index 621fcc7..9e5a7a7 100644
--- a/knative-build/runtimes/javascript/Dockerfile
+++ b/knative-build/runtimes/javascript/Dockerfile
@@ -18,6 +18,7 @@
FROM node:10.15.0-stretch
RUN apt-get update && apt-get install -y \
imagemagick \
+ zip \
unzip \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /nodejsAction
diff --git a/knative-build/runtimes/javascript/buildtemplate.yaml
b/knative-build/runtimes/javascript/buildtemplate.yaml
index 0760ced..9de87e8 100644
--- a/knative-build/runtimes/javascript/buildtemplate.yaml
+++ b/knative-build/runtimes/javascript/buildtemplate.yaml
@@ -51,13 +51,21 @@ spec:
if [ -z ${OW_PROJECT_URL} ]; then
OW_ACTION_CODE="${OW_ACTION_CODE}"
else
- TEMPDIR="knative-"$((1 + RANDOM % 100))
- TEMPFILE=`basename "${OW_PROJECT_URL}"`
- mkdir $TEMPDIR
- cd $TEMPDIR
- wget -O $TEMPFILE "${OW_PROJECT_URL}"
- OW_ACTION_CODE=`cat $TEMPFILE`
- cd ..
+ if [ -f ${OW_PROJECT_URL} ]; then
+ if [ ${OW_ACTION_BINARY} ]; then
+ OW_ACTION_CODE=`base64 ${OW_PROJECT_URL}`
+ else
+ OW_ACTION_CODE=`cat ${OW_PROJECT_URL}`
+ fi
+ else
+ TEMPDIR="knative-"$((1 + RANDOM % 100))
+ TEMPFILE=`basename "${OW_PROJECT_URL}"`
+ mkdir $TEMPDIR
+ cd $TEMPDIR
+ wget -O $TEMPFILE "${OW_PROJECT_URL}"
+ OW_ACTION_CODE=`cat $TEMPFILE`
+ cd ..
+ fi
fi
cat <<EOF >> ${DOCKERFILE}
ENV __OW_RUNTIME_DEBUG "${OW_RUNTIME_DEBUG}"
@@ -68,6 +76,7 @@ spec:
ENV __OW_ACTION_BINARY "${OW_ACTION_BINARY}"
ENV __OW_HTTP_METHODS "${OW_HTTP_METHODS}"
ENV __OW_ACTION_RAW "${OW_ACTION_RAW}"
+ ENV __OW_PROJECT_URL "${OW_PROJECT_URL}"
EOF
- name: build-openwhisk-nodejs-runtime
image: "gcr.io/kaniko-project/executor:latest"
diff --git a/knative-build/runtimes/javascript/platform/knative.js
b/knative-build/runtimes/javascript/platform/knative.js
index 442381c..eb741fe 100644
--- a/knative-build/runtimes/javascript/platform/knative.js
+++ b/knative-build/runtimes/javascript/platform/knative.js
@@ -93,6 +93,7 @@ function createInitDataFromEnvironment(env) {
// TODO: default to empty?
initdata.actionName = (typeof env.__OW_ACTION_NAME === 'undefined') ?
"" : env.__OW_ACTION_NAME;
initdata.raw = (typeof env.__OW_ACTION_RAW === 'undefined') ? false :
env.__OW_ACTION_RAW.toLowerCase() === "true";
+ initdata.url = (typeof env.__OW_PROJECT_URL === 'undefined') ? "" :
env.__OW_PROJECT_URL;
DEBUG.dumpObject(initdata, "initdata");
return initdata;
@@ -135,6 +136,9 @@ function preProcessInitData(initdata, valuedata,
activationdata) {
throw ("Invalid Init. data; expected boolean for key
'raw'.");
}
}
+ if (initdata.url && typeof initdata.url === 'string') {
+ valuedata.url = initdata.url;
+ }
// Action name is a special case, as we have a key collision on
"name" between init. data and request
// param. data (as they both appear within "body.value") so we
must save it to its final location
@@ -249,6 +253,78 @@ function preProcessActivationData(env, activationdata) {
}
/**
+ * For actions having dependency on third party NPM modules, NodeJS runtime
would
+ * install those NPM modules and package them as part of the action source
using this function.
+ * In this function, runtime reads url from the initialization JSON payload,
+ * if the url points to an existing directory, run:
+ * cd initdata.url && npm install --production && zip -r action.zip . &&
base64 action.zip
+ * Packaging actions as NodeJS module with NPM libraries are treated as zip
actions by the runtime.
+ * Zipped actions must contain either package.json or index.js at the root.
+ * This validation is left to the user and not enforced while initializing the
action.
+ */
+function marshalResources(initData, valueData) {
+ DEBUG.functionStart();
+ try {
+ // check if url is set and assigned some string value
+ if (typeof initData.url === "string" && initData.url !== undefined) {
+ const fs = require('fs');
+ if (fs.existsSync(initData.url)) {
+ const stats = fs.lstatSync(initData.url);
+ // check if specified url is an existing directory
+ if (stats.isDirectory()) {
+ // import spawnSync routine from child_process NPM module
+ // spawnSync spawns a new process using the given command
and does not return until
+ // the child process is fully closed/finished executing.
+ // Note: Be very careful changing any of the child_process
and spwanSync functionality
+ DEBUG.dumpObject("Starting to install NPM modules from " +
initData.url, "process packaging - npm", "marshalResources")
+ const {spawnSync} = require('child_process'),
+ npm = spawnSync('npm', ['install', '--production'],
{cwd: initData.url});
+ DEBUG.dumpObject(`stdout:
${npm.stdout.toString().trim()}`, "npm install", "marshalResources");
+ DEBUG.dumpObject(`stderr:
${npm.stderr.toString().trim()}`, "npm install", "marshalResources");
+ if (npm.status !== 0) {
+ throw (npm.error);
+ }
+
+ var zipFile = "action.zip";
+ // note here we are using same instance of spwanSync as we
used for running npm install
+ // to make sure the execution of zip command follows right
after npm install closes (sequential)
+ DEBUG.dumpObject("Starting to compress action resources at
" + initData.url, "process packaging - zip", "marshalResources")
+ const compressFile = spawnSync('zip', ['-r', zipFile,
'.'], {cwd: initData.url});
+ DEBUG.dumpObject(`stdout:
${compressFile.stdout.toString().trim()}`, "zip -r action.zip *",
"marshalResources");
+ DEBUG.dumpObject(`stderr:
${compressFile.stderr.toString().trim()}`, "zip -r action.zip *",
"marshalResources");
+ if (compressFile.status !== 0) {
+ throw (compressFile.error);
+ }
+
+ // and same here, making sure base64 command follows right
after zip command finishes by
+ // using the same spawnSync instance of child_process.
+ DEBUG.dumpObject("Starting to decode compressed action
resource", "process packaging - base64", "marshalResources")
+ const code = spawnSync('base64', [initData.url + '/' +
zipFile]);
+ DEBUG.dumpObject(`stdout:
${code.stdout.toString().trim()}`, "base64 action.zip", "marshalResources");
+ DEBUG.dumpObject(`stderr:
${code.stderr.toString().trim()}`, "base64 action.zip", "marshalResources");
+ if (code.status !== 0) {
+ throw (code.error);
+ }
+
+ // assign base64 encoded zip file content to action code
before initializing the action
+ initData.code = code.stdout.toString().trim();
+ valueData.code = initData.code;
+
+ // mark this action as binary so that run routine can
decode it appropriately
+ initData.binary = true;
+ valueData.binary = true;
+ }
+ }
+ }
+ } catch (e) {
+ console.error(e);
+ DEBUG.functionEndError("");
+ throw("Unable to marshall NPM resources: ");
+ }
+ DEBUG.functionEnd()
+}
+
+/**
* Pre-process the incoming http request data, moving it to where the
* route handlers expect it to be for an openwhisk runtime.
*/
@@ -269,6 +345,8 @@ function preProcessRequest(req){
preProcessInitData(initData, valueData, activationData);
}
+ marshalResources(initData, valueData);
+
if(hasActivationData(req)) {
// process HTTP request header and body to make it available to
function as parameter data
preProcessHTTPContext(req, valueData);
diff --git
a/knative-build/runtimes/javascript/tests/packageaction/build.yaml.tmpl
b/knative-build/runtimes/javascript/tests/packageaction/build.yaml.tmpl
new file mode 100644
index 0000000..1cbe955
--- /dev/null
+++ b/knative-build/runtimes/javascript/tests/packageaction/build.yaml.tmpl
@@ -0,0 +1,29 @@
+apiVersion: build.knative.dev/v1alpha1
+kind: Build
+metadata:
+ name: nodejs-10-package-npm
+spec:
+ serviceAccountName: openwhisk-runtime-builder
+ sources:
+ - name: runtime
+ git:
+ url: "https://github.com/apache/incubator-openwhisk-devtools.git";
+ revision: "master"
+ - name: application
+ targetPath: app
+ git:
+ url: "https://github.com/apache/incubator-openwhisk-test.git";
+ revision: "master"
+ template:
+ name: openwhisk-nodejs-runtime
+ arguments:
+ - name: TARGET_IMAGE_NAME
+ value: "docker.io/${DOCKER_USERNAME}/nodejs-10-package-npm"
+ - name: DOCKERFILE
+ value: "./knative-build/runtimes/javascript/Dockerfile"
+ - name: OW_ACTION_BINARY
+ value: "true"
+ - name: OW_ACTION_NAME
+ value: "nodejs-package-npm"
+ - name: OW_PROJECT_URL
+ value: "./app/packages/left-pad/"
diff --git
a/knative-build/runtimes/javascript/tests/packageaction/knative-data-init-run.json
b/knative-build/runtimes/javascript/tests/packageaction/knative-data-init-run.json
new file mode 100644
index 0000000..96307b0
--- /dev/null
+++
b/knative-build/runtimes/javascript/tests/packageaction/knative-data-init-run.json
@@ -0,0 +1,20 @@
+{
+ "init": {
+ "name" : "nodejs-package-npm",
+ "main" : "main",
+ "binary": true,
+ "url" : "app/packages/left-pad/"
+ },
+ "activation": {
+ "namespace": "default",
+ "action_name": "nodejs-package-npm",
+ "api_host": "",
+ "api_key": "",
+ "activation_id": "",
+ "deadline": "4102498800000"
+ },
+ "value": {
+ "lines" : ["Hello","How are you?"]
+ }
+}
+
diff --git
a/knative-build/runtimes/javascript/tests/packageaction/knative-data-init.json
b/knative-build/runtimes/javascript/tests/packageaction/knative-data-init.json
new file mode 100644
index 0000000..badd5ec
--- /dev/null
+++
b/knative-build/runtimes/javascript/tests/packageaction/knative-data-init.json
@@ -0,0 +1,8 @@
+{
+ "init": {
+ "name" : "nodejs-package-npm",
+ "main" : "main",
+ "binary": true,
+ "url" : "app/packages/left-pad/"
+ }
+}
diff --git
a/knative-build/runtimes/javascript/tests/packageaction/knative-data-run.json
b/knative-build/runtimes/javascript/tests/packageaction/knative-data-run.json
new file mode 100644
index 0000000..93ad826
--- /dev/null
+++
b/knative-build/runtimes/javascript/tests/packageaction/knative-data-run.json
@@ -0,0 +1,13 @@
+{
+ "activation": {
+ "namespace": "default",
+ "action_name": "nodejs-package-npm",
+ "api_host": "",
+ "api_key": "",
+ "activation_id": "",
+ "deadline": "4102498800000"
+ },
+ "value": {
+ "lines" : ["Hello","How are you?"]
+ }
+}
diff --git
a/knative-build/runtimes/javascript/tests/packageaction/service.yaml.tmpl
b/knative-build/runtimes/javascript/tests/packageaction/service.yaml.tmpl
new file mode 100644
index 0000000..3c5a439
--- /dev/null
+++ b/knative-build/runtimes/javascript/tests/packageaction/service.yaml.tmpl
@@ -0,0 +1,12 @@
+apiVersion: serving.knative.dev/v1alpha1
+kind: Service
+metadata:
+ name: nodejs-package-npm
+ namespace: default
+spec:
+ runLatest:
+ configuration:
+ revisionTemplate:
+ spec:
+ container:
+ image: docker.io/${DOCKER_USERNAME}/nodejs-10-package-npm
