This is an automated email from the ASF dual-hosted git repository.
mrutkowski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/openwhisk-runtime-docker.git
The following commit(s) were added to refs/heads/master by this push:
new d7ea472 Enable multiple platforms to use actionproxy
new e02a5bb Merge pull request #82 from pwplusnick/platform
d7ea472 is described below
commit d7ea472c90339b40157e37329b6923f0b3ec47aa
Author: Will Plusnick <[email protected]>
AuthorDate: Fri Feb 7 20:14:21 2020 -0600
Enable multiple platforms to use actionproxy
* Add an ability to use action proxy with different platforms
* Add openwhisk platform implementation
* Add knative platform implementation
* Update dockerfile
* Note that there was a factory pattern implemented, but is currently
bypassed
---
core/actionProxy/Dockerfile | 5 +-
core/actionProxy/actionproxy.py | 75 ++++-
core/actionProxy/owplatform/__init__.py | 87 ++++++
core/actionProxy/owplatform/knative.py | 314 +++++++++++++++++++++
.../{Dockerfile => owplatform/openwhisk.py} | 26 +-
5 files changed, 471 insertions(+), 36 deletions(-)
diff --git a/core/actionProxy/Dockerfile b/core/actionProxy/Dockerfile
index c2cc072..173b050 100644
--- a/core/actionProxy/Dockerfile
+++ b/core/actionProxy/Dockerfile
@@ -29,8 +29,11 @@ RUN apk upgrade --update \
ENV FLASK_PROXY_PORT 8080
-RUN mkdir -p /actionProxy
+RUN mkdir -p /actionProxy/owplatform
ADD actionproxy.py /actionProxy/
+ADD owplatform/__init__.py /actionProxy/owplatform/
+ADD owplatform/knative.py /actionProxy/owplatform/
+ADD owplatform/openwhisk.py /actionProxy/owplatform/
RUN mkdir -p /action
ADD stub.sh /action/exec
diff --git a/core/actionProxy/actionproxy.py b/core/actionProxy/actionproxy.py
index 39e1ea5..1812918 100644
--- a/core/actionProxy/actionproxy.py
+++ b/core/actionProxy/actionproxy.py
@@ -27,17 +27,27 @@ code when required.
*/
"""
-import sys
-import os
+import base64
+import codecs
+import io
import json
+import os
import subprocess
-import codecs
+import sys
+import zipfile
+
import flask
from gevent.pywsgi import WSGIServer
-import zipfile
-import io
-import base64
+# The following import is only needed if we actually want to use the factory
pattern.
+# See comment below for reasons we decided to bypass it.
+#from owplatform import PlatformFactory, InvalidPlatformError
+from owplatform.knative import KnativeImpl
+from owplatform.openwhisk import OpenWhiskImpl
+
+PLATFORM_OPENWHISK = 'openwhisk'
+PLATFORM_KNATIVE = 'knative'
+DEFAULT_PLATFORM = PLATFORM_OPENWHISK
class ActionRunner:
"""ActionRunner."""
@@ -215,8 +225,7 @@ def setRunner(r):
runner = r
[email protected]('/init', methods=['POST'])
-def init():
+def init(message=None):
if proxy.rejectReinit is True and proxy.initialized is True:
msg = 'Cannot initialize the action more than once.'
sys.stderr.write(msg + '\n')
@@ -224,7 +233,7 @@ def init():
response.status_code = 403
return response
- message = flask.request.get_json(force=True, silent=True)
+ message = message or flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
flask.abort(404)
else:
@@ -247,14 +256,15 @@ def init():
return complete(response)
[email protected]('/run', methods=['POST'])
-def run():
+def run(message=None):
def error():
response = flask.jsonify({'error': 'The action did not receive a
dictionary as an argument.'})
response.status_code = 404
return complete(response)
- message = flask.request.get_json(force=True, silent=True)
+ # If we have a message use that, if not try using the request json if it
exists (returns None on no JSON)
+ # otherwise just make it an empty dictionary
+ message = message or flask.request.get_json(force=True, silent=True) or {}
if message and not isinstance(message, dict):
return error()
else:
@@ -264,9 +274,14 @@ def run():
if runner.verify():
try:
- code, result = runner.run(args, runner.env(message or {}))
- response = flask.jsonify(result)
- response.status_code = code
+ if 'activation' in message:
+ code, result = runner.run(args,
runner.env(message['activation'] or {}))
+ response = flask.jsonify(result)
+ response.status_code = code
+ else:
+ code, result = runner.run(args, runner.env(message or {}))
+ response = flask.jsonify(result)
+ response.status_code = code
except Exception as e:
response = flask.jsonify({'error': 'Internal error. {}'.format(e)})
response.status_code = 500
@@ -286,6 +301,36 @@ def complete(response):
def main():
+# This is for future users. If there ever comes a time where more platforms
are implemented or where
+# speed is less of a concern it is advisable to use the factory pattern
described below. As for now
+# we have decided the trade off in speed is not worth it. In runtimes,
milliseconds matter!
+#
+# platformImpl = None
+# PlatformFactory.addPlatform(PLATFORM_OPENWHISK, OpenWhiskImpl)
+# PlatformFactory.addPlatform(PLATFORM_KNATIVE, KnativeImpl)
+#
+# targetPlatform = os.getenv('__OW_RUNTIME_PLATFORM', DEFAULT_PLATFORM)
+# if not PlatformFactory.isSupportedPlatform(targetPlatform):
+# raise InvalidPlatformError(targetPlatform,
PlatformFactory.supportedPlatforms())
+# else:
+# platformFactory = PlatformFactory()
+# platformImpl = platformFactory.createPlatformImpl(targetPlatform,
proxy)
+# platformImpl.registerHandlers(init, run)
+
+ platformImpl = None
+ targetPlatform = os.getenv('__OW_RUNTIME_PLATFORM',
DEFAULT_PLATFORM).lower()
+ # Target Knative if it specified, otherwise just default to OpenWhisk.
+ if targetPlatform == PLATFORM_KNATIVE:
+ platformImpl = KnativeImpl(proxy)
+ else:
+ platformImpl = OpenWhiskImpl(proxy)
+ if targetPlatform != PLATFORM_OPENWHISK:
+ print(f"Invalid __OW_RUNTIME_PLATFORM {targetPlatform}! " +
+ f"Valid Platforms are {PLATFORM_OPENWHISK} and
{PLATFORM_KNATIVE}. " +
+ f"Defaulting to {PLATFORM_OPENWHISK}.", file=sys.stderr)
+
+ platformImpl.registerHandlers(init, run)
+
port = int(os.getenv('FLASK_PROXY_PORT', 8080))
server = WSGIServer(('0.0.0.0', port), proxy, log=None)
server.serve_forever()
diff --git a/core/actionProxy/owplatform/__init__.py
b/core/actionProxy/owplatform/__init__.py
new file mode 100644
index 0000000..1c3c2a3
--- /dev/null
+++ b/core/actionProxy/owplatform/__init__.py
@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class PlatformFactory:
+
+ _SUPPORTED_PLATFORMS = set()
+ _PLATFORM_IMPLEMENTATIONS = {}
+
+ def __init__(self):
+ pass
+
+ @classmethod
+ def supportedPlatforms(cls):
+ return cls._SUPPORTED_PLATFORMS
+
+ @classmethod
+ def isSupportedPlatform(cls, id):
+ return id.lower() in cls._SUPPORTED_PLATFORMS
+
+ @classmethod
+ def addPlatform(cls, platform, platformImp):
+ if platform.lower not in cls._SUPPORTED_PLATFORMS:
+ cls._SUPPORTED_PLATFORMS.add(platform.lower())
+ cls._PLATFORM_IMPLEMENTATIONS[platform.lower()] = platformImp
+ else:
+ raise DuplicatePlatform()
+ getterName = "PLATFORM_" + platform.upper()
+ setattr(cls, getterName, platform)
+
+ @classmethod
+ def createPlatformImpl(cls, id, proxy):
+ if cls.isSupportedPlatform(id):
+ return cls._PLATFORM_IMPLEMENTATIONS[id.lower()](proxy)
+ else:
+ raise InvalidPlatformError(id, self.supportedPlatforms())
+
+ @property
+ def app(self):
+ return self._app
+
+ @app.setter
+ def app(self, value):
+ raise ConstantError("app cannot be set outside of initialization")
+
+ @property
+ def config(self):
+ return self._config
+
+ @config.setter
+ def config(self, value):
+ raise ConstantError("config cannot be set outside of initialization")
+
+ @property
+ def service(self):
+ return self._service
+
+ @service.setter
+ def service(self, value):
+ raise ConstantError("service cannot be set outside of initialization")
+
+class ConstantError(Exception):
+ pass
+
+class DuplicatePlatformError(Exception):
+ pass
+
+class InvalidPlatformError(Exception):
+ def __init__(self, platform, supportedPlatforms):
+ self.platform = platform.lower()
+ self.supportedPlatforms = supportedPlatforms
+
+ def __str__(self):
+ return f"Invalid Platform: {self.platform} is not in supported
platforms {self.supportedPlatforms}."
diff --git a/core/actionProxy/owplatform/knative.py
b/core/actionProxy/owplatform/knative.py
new file mode 100644
index 0000000..fe89334
--- /dev/null
+++ b/core/actionProxy/owplatform/knative.py
@@ -0,0 +1,314 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import base64
+from json import dumps
+import os
+import sys
+
+import flask
+
+DEFAULT_METHOD = ['POST']
+VALID_METHODS = set(['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'])
+
+OW_ENV_PREFIX = '__OW_'
+
+# A stem cell is an openwhisk container that is not 'pre-initialized'
+# with the code in the environment variable '__OW_ACTION_CODE'
+# returns a boolean
+def isStemCell():
+ actionCode = os.getenv('__OW_ACTION_CODE', '')
+ return len(actionCode) == 0
+
+# Checks to see if the activation data is in the request
+# returns a boolean
+def hasActivationData(msg):
+ return 'activation' in msg and 'value' in msg
+
+# Checks to see if the initialization data is in the request
+# returns a boolean
+def hasInitData(msg):
+ return 'init' in msg
+
+def removeInitData(body):
+ def delIfPresent(d, key):
+ if key in d:
+ del d[key]
+ if body and 'value' in body:
+ delIfPresent(body['value'], 'code')
+ delIfPresent(body['value'], 'main')
+ delIfPresent(body['value'],'binary')
+ delIfPresent(body['value'], 'raw')
+ delIfPresent(body['value'], 'actionName')
+
+# create initialization data from environment variables
+# return dictionary
+def createInitDataFromEnvironment():
+ initData = {}
+ initData['main'] = os.getenv('__OW_ACTION_MAIN', 'main')
+ initData['code'] = os.getenv('__OW_ACTION_CODE', '')
+ initData['binary'] = os.getenv('__OW_ACTION_BINARY', 'false').lower() ==
'true'
+ initData['actionName'] = os.getenv('__OW_ACTION_NAME', '')
+ initData['raw'] = os.getenv('__OW_ACTION_RAW', 'false').lower() == 'true'
+ return initData
+
+def preProcessInitData(initData, valueData, activationData):
+ def presentAndType(mapping, key, dataType):
+ return key in mapping and isinstance(mapping[key], dataType)
+
+ if len(initData) > 0:
+ if presentAndType(initData, 'main', str):
+ valueData['main'] = initData['main']
+ if presentAndType(initData, 'code', str):
+ valueData['code'] = initData['code']
+
+ try:
+ if presentAndType(initData, 'binary', bool):
+ valueData['binary'] = initData['binary']
+ elif 'binary' in initData:
+ raise InvalidInitValueType('binary', 'boolean')
+
+ if presentAndType(initData, 'raw', bool):
+ valueData['raw'] = initData['raw']
+ elif 'raw' in initData:
+ raise InvalidInitValueType('raw', 'boolean')
+
+ except InvalidInitValueType as e:
+ print(e, file=sys.stderr)
+ raise InvalidInitData(e)
+
+ # Action name is a special case, as we have a key collision on "name"
between init. data and request
+ # param. data so we must save it to its final location as the default
Action name as part of the
+ # activation data
+ if presentAndType(initData, 'name', str):
+ if 'action_name' not in activationData or \
+ (isinstance(activationData['action_name'], str) and \
+ len(activationData['action_name']) == 0):
+ activationData['action_name'] = initData['name']
+
+def preProcessHTTPContext(msg, valueData):
+ if valueData.get('raw', False):
+ if isinstance(msg.get('value', {}), str):
+ valueData['__ow_body'] = msg.get('value')
+ else:
+ tmpBody = msg.get('value', {})
+ removeInitData(tmpBody)
+ bodyStr = str(tmpBody)
+ valueData['__ow_body'] = base64.b64encode(bodyStr)
+ valueData['__ow_query'] = flask.request.query_string
+
+ namespace = ''
+ if '__OW_NAMESPACE' in os.environ:
+ namespace = os.getenv('__OW_NAMESPACE')
+ valueData['__ow_user'] = namespace
+ valueData['__ow_method'] = flask.request.method
+ valueData['__ow_headers'] = flask.request.headers
+ valueData['__ow_path'] = ''
+
+def preProcessActivationData(activationData):
+ for k in activationData:
+ if isinstance(activationData[k], str):
+ environVar = OW_ENV_PREFIX + k.upper()
+ os.environ[environVar] = activationData[k]
+
+def preProcessRequest(msg):
+ valueData = msg.get('value', {})
+ if isinstance(valueData, str):
+ valueData = {}
+ initData = msg.get('init', {})
+ activationData = msg.get('activation', {})
+
+ if hasInitData(msg):
+ preProcessInitData(initData, valueData, activationData)
+
+ if hasActivationData(msg):
+ preProcessHTTPContext(msg, valueData)
+ preProcessActivationData(activationData)
+
+ msg['value'] = valueData
+ msg['init'] = initData
+ msg['activation'] = activationData
+
+def postProcessResponse(requestHeaders, response):
+ CONTENT_TYPE = 'Content-Type'
+ content_types = {
+ 'json': 'application/json',
+ 'html': 'text/html',
+ }
+
+ statusCode = response.status
+ headers = {}
+ body = response.get_json() or {}
+ contentTypeInHeaders = False
+
+ # if a status code is specified set and remove from the body
+ # of the response
+ if 'statusCode' in body:
+ statusCode = body['statusCode']
+ del body['statusCode']
+
+ if 'headers' in body:
+ headers = body['headers']
+ del body['headers']
+
+ # content-type vs Content-Type
+ # make Content-Type standard
+ if CONTENT_TYPE.lower() in headers:
+ headers[CONTENT_TYPE] = headers[CONTENT_TYPE.lower()]
+ del headers[CONTENT_TYPE.lower()]
+
+ # if there is no content type specified make it html for string bodies
+ # and json for non-string bodies
+ if not CONTENT_TYPE in headers:
+ if isinstance(body, str):
+ headers[CONTENT_TYPE] = content_types['html']
+ else:
+ headers[CONTENT_TYPE] = content_types['json']
+ else:
+ contentTypeInHeaders = True
+
+ # a json object containing statusCode, headers, and body is what we expect
from a web action
+ # so we only want to return the actual body
+ if 'body' in body:
+ body = body['body']
+
+ # if we are returning an image that is base64 encoded, we actually want to
return the image
+ if contentTypeInHeaders and 'image' in headers[CONTENT_TYPE]:
+ body = base64.b64decode(body)
+ headers['Content-Transfer-Encoding'] = 'binary'
+ else:
+ body = dumps(body)
+
+ if statusCode == 200 and len(body) == 0:
+ statusCode = 204 # no content status code
+
+ if 'Access-Control-Allow-Origin' not in headers:
+ headers['Access-Control-Allow-Origin'] = '*'
+
+ if 'Access-Control-Allow-Methods' not in headers:
+ headers['Access-Control-Allow-Methods'] = 'OPTIONS, GET, DELETE, POST,
PUT, HEAD, PATCH'
+
+ if 'Access-Control-Allow-Headers' not in headers:
+ headers['Access-Control-Allow-Headers'] = 'Authorization, Origin, X -
Requested - With, Content - Type, Accept, User - Agent'
+ if 'Access-Control-Request-Headers' in requestHeaders:
+ headers['Access-Control-Request-Headers'] =
requestHeaders['Access-Control-Request-Headers']
+ return flask.Response(body, statusCode, headers)
+
+class KnativeImpl:
+
+ def __init__(self, proxy):
+ self.proxy = proxy
+ self.initCode = None
+ self.runCode = None
+
+ def _run_error(self):
+ response = flask.jsonify({'error': 'The action did not receive a
dictionary as an argument.'})
+ response.status_code = 404
+ return response
+
+ def run(self):
+ response = None
+ message = flask.request.get_json(force=True, silent=True) or {}
+ request_headers = flask.request.headers
+ dedicated_runtime = False
+
+ if message and not isinstance(message, dict):
+ return self._run_error()
+
+ try:
+ # don't process init data if it is not a stem cell
+ if hasInitData(message) and not isStemCell():
+ raise NonStemCellInitError()
+
+ # if it is a dedicated runtime and is uninitialized, then init
from environment
+ if not isStemCell() and self.proxy.initialized is False:
+ message['init'] = createInitDataFromEnvironment()
+ dedicated_runtime = True
+
+ if hasInitData(message) and hasActivationData(message) and not
dedicated_runtime:
+ preProcessRequest(message)
+ self.initCode(message)
+ removeInitData(message)
+ response = self.runCode(message)
+ response = postProcessResponse(request_headers, response)
+ elif hasInitData(message) and not dedicated_runtime:
+ preProcessRequest(message)
+ response = self.initCode(message)
+ elif hasActivationData(message) and not dedicated_runtime:
+ preProcessRequest(message)
+ response = self.runCode(message)
+ response = postProcessResponse(request_headers, response)
+ else:
+ preProcessRequest(message)
+ # This is for the case when it is a dedicated runtime, but has
not yet been
+ # initialized from the environment
+ if dedicated_runtime and self.proxy.initialized is False:
+ self.initCode(message)
+ removeInitData(message)
+ response = self.runCode(message)
+ response = postProcessResponse(request_headers, response)
+ except Exception as e:
+ response = flask.jsonify({'error': str(e)})
+ response.status_code = 404
+
+ return response
+
+
+ def registerHandlers(self, initCodeImp, runCodeImp):
+
+ self.initCode = initCodeImp
+ self.runCode = runCodeImp
+
+ httpMethods = os.getenv('__OW_HTTP_METHODS', DEFAULT_METHOD)
+ # try to turn the environment variable into a list if it is in the
right format
+ if isinstance(httpMethods, str) and httpMethods[0] == '[' and
httpMethods[-1] == ']':
+ httpMethods = httpMethods[1:-1].split(',')
+ # otherwise just default if it is not a list
+ elif not isinstance(httpMethods, list):
+ httpMethods = DEFAULT_METHOD
+
+ httpMethods = {m.upper() for m in httpMethods}
+
+ # use some fancy set operations to make sure all the methods are valid
+ # and remove any that aren't
+ invalidMethods = httpMethods.difference(set(VALID_METHODS))
+ validMethods = list(httpMethods.intersection(set(VALID_METHODS)))
+ if len(invalidMethods) > 0:
+ for invalidMethod in invalidMethods:
+ print("Environment variable '__OW_HTTP_METHODS' has an
unrecognised value (" + invalidMethod + ").",
+ file=sys.stderr)
+
+ self.proxy.add_url_rule('/', 'run', self.run, methods=validMethods)
+
+class NonStemCellInitError(Exception):
+ def __str__(self):
+ return "Cannot initialize a runtime with a dedicated function."
+
+class InvalidInitValueType(Exception):
+ def __init__(self, key, valueType):
+ self.key = key
+ self.valueType = valueType
+
+ def __str__(self):
+ return f"Invalid Init. data; expected {self.valueType} for key
'{self.key}'."
+
+class InvalidInitData(Exception):
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return f"Unable to process Initialization data: {self.msg}"
diff --git a/core/actionProxy/Dockerfile
b/core/actionProxy/owplatform/openwhisk.py
similarity index 50%
copy from core/actionProxy/Dockerfile
copy to core/actionProxy/owplatform/openwhisk.py
index c2cc072..afb5752 100644
--- a/core/actionProxy/Dockerfile
+++ b/core/actionProxy/owplatform/openwhisk.py
@@ -15,25 +15,11 @@
# limitations under the License.
#
-# Dockerfile for docker skeleton (useful for running blackbox binaries,
scripts, or Python 3 actions) .
-FROM python:3.6-alpine
+class OpenWhiskImpl:
-# Upgrade and install basic Python dependencies.
-RUN apk upgrade --update \
- && apk add --no-cache bash perl jq zip git curl wget openssl ca-certificates
sed openssh-client \
- && update-ca-certificates \
- && apk add --no-cache --virtual .build-deps bzip2-dev gcc libc-dev \
- && pip install --upgrade pip setuptools six \
- && pip install --no-cache-dir gevent==1.3.6 flask==1.0.2 \
- && apk del .build-deps
+ def __init__(self, proxy):
+ self.proxy = proxy
-ENV FLASK_PROXY_PORT 8080
-
-RUN mkdir -p /actionProxy
-ADD actionproxy.py /actionProxy/
-
-RUN mkdir -p /action
-ADD stub.sh /action/exec
-RUN chmod +x /action/exec
-
-CMD ["/bin/bash", "-c", "cd actionProxy && python -u actionproxy.py"]
+ def registerHandlers(self, init, run):
+ self.proxy.add_url_rule('/init', 'init', init, methods=['POST'])
+ self.proxy.add_url_rule('/run', 'run', run, methods=['POST'])
