This is an automated email from the ASF dual-hosted git repository.
csantanapr pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-openwhisk-runtime-go.git
The following commit(s) were added to refs/heads/master by this push:
new c3e94d5 Static Build, Versioning and Support for More Languages (#72)
c3e94d5 is described below
commit c3e94d5f755b7b52b2a6e09e53154d06bde74505
Author: Michele Sciabarra <[email protected]>
AuthorDate: Mon Mar 4 17:42:01 2019 +0100
Static Build, Versioning and Support for More Languages (#72)
---
.gitignore | 6 +-
openwhisk/_test/build.sh => CHANGES.md | 57 ++++-------
CREDITS.txt | 4 +-
LICENSE-filetype.txt | 25 -----
README.md | 4 +
{actionProxyLoop => actionloop}/Dockerfile | 0
{actionProxyLoop => actionloop}/build.gradle | 16 +++-
build.gradle | 7 +-
examples/Makefile | 3 +-
examples/bash-hello/Makefile | 2 +-
examples/golang-hello-single/Makefile | 4 +-
examples/golang-hello-vendor/Makefile | 8 +-
examples/golang-main-package/Makefile | 8 +-
examples/golang-main-single/Makefile | 4 +-
examples/golang-main-standalone/Makefile | 2 +-
examples/golang-main-vendor/Makefile | 4 +-
golang1.11/build.gradle | 3 +-
main/proxy.go | 33 +------
openwhisk/_test/build.sh | 6 +-
openwhisk/_test/compile.py | 11 +++
openwhisk/_test/{build.sh => find.sh} | 40 +-------
openwhisk/_test/pysample/exec | 6 ++
openwhisk/_test/pysample/lib/action/__init__.py | 2 +
openwhisk/_test/pysample/lib/action/main.py | 7 ++
openwhisk/_test/pysample/lib/exec.py | 20 ++++
openwhisk/actionProxy.go | 34 ++++++-
openwhisk/actionProxy_test.go | 44 +++++++++
openwhisk/compiler_test.go | 2 +-
openwhisk/extractor.go | 56 +----------
openwhisk/extractor_test.go | 2 +-
openwhisk/filetype.go | 10 +-
openwhisk/filetype_test.go | 6 ++
openwhisk/initHandler.go | 24 ++---
openwhisk/initHandler_test.go | 17 ++++
openwhisk/util_test.go | 19 ++--
openwhisk/version.go | 2 +-
openwhisk/zip.go | 104 +++++++++++++++++++++
openwhisk/{version.go => zip_test.go} | 26 +++++-
settings.gradle | 3 +-
.../actionContainers/ActionLoopBasicTests.scala | 2 +-
.../ActionLoopContainerTests.scala | 6 +-
41 files changed, 379 insertions(+), 260 deletions(-)
diff --git a/.gitignore b/.gitignore
index d490e01..7eacee2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,8 +11,9 @@ tests/build/
vendor/
# Go binary proxy
-actionProxyLoop/proxy
-golang1.10/proxy
+common/proxy
+actionloop/proxy
+golang1.11/proxy
# Go test transient files
openwhisk/_test/exec
@@ -26,6 +27,7 @@ openwhisk/_test/output/
openwhisk/action/
openwhisk/compile/
openwhisk/debug.test
+*.pyc
# Eclipse
tests/bin/
diff --git a/openwhisk/_test/build.sh b/CHANGES.md
old mode 100755
new mode 100644
similarity index 54%
copy from openwhisk/_test/build.sh
copy to CHANGES.md
index 58f80e8..c9a557b
--- a/openwhisk/_test/build.sh
+++ b/CHANGES.md
@@ -1,4 +1,4 @@
-#!/bin/bash
+<!--
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,42 +15,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-cd "$(dirname $0)"
-
-function build {
- test -e exec && rm exec
- cp $1.src $1.go
- GOPATH=$PWD go build -a -o exec $1.go
- rm $1.go
-}
-
-function build_main {
- test -e exec && rm exec
- cp ../../common/gobuild.py.launcher.go $1.go
- cat $1.src >>$1.go
- go build -a -o exec $1.go
- rm $1.go
-}
-
-
-build hi
-zip hi.zip exec
-cp exec hi
-
-build_main hello_message
-zip hello_message.zip exec
-cp exec hello_message
-
-build_main hello_greeting
-zip hello_greeting.zip exec
-cp exec hello_greeting
-
-test -e hello.zip && rm hello.zip
-cd src
-zip -q -r ../hello.zip main.go hello
-cd ..
-
-build exec
-test -e exec.zip && rm exec.zip
-zip -q -r exec.zip exec etc dir
+-->
+# ActionLoop v1.0.1
+- embedded file type detection
+- now showing the commend
+- librdkafka in golang image
+- showing version numbuer with -debug
+
+# Actionloop v2
+Versioning
+- renamed actionloop docker image to actionloop-v2
+Docker Images Support
+- static build of the executable docker image, so actionloop can be used also
in alpine images
+ActionLoop for Scripting Languages
+- any script starting with '#!' is recognized as executable
+- now the -compile will zip the entire directory of the `bin` directory after
compilation
+- if you upload a folder `src/exec` the entire directory is moved to `bin`,
including other uploaded files
diff --git a/CREDITS.txt b/CREDITS.txt
index f3d5e6a..c47fb41 100644
--- a/CREDITS.txt
+++ b/CREDITS.txt
@@ -1,6 +1,6 @@
-Michele Sciabarra <[email protected]>
+Michele Sciabarra <[email protected]>
-Carlos Santana
+Carlos Santana <[email protected]>
Rodric Rabbah <[email protected]>
diff --git a/LICENSE-filetype.txt b/LICENSE-filetype.txt
deleted file mode 100644
index c03a05c..0000000
--- a/LICENSE-filetype.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-The MIT License
-
-Copyright (c) Tomas Aparicio
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
diff --git a/README.md b/README.md
index 186221c..88c634b 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,10 @@ This repository containst both the OpenWhisk runtime for
Golang Actions, as well
- Deployment for [Generic](docs/DEPLOY.md#generic) actions
- The [ActionLoop](docs/ACTION.md#actionloop) protocol for generic actions
+# Change Log
+
+[CHANGES.md](Here.)
+
# License
[Apache 2.0](LICENSE.txt)
diff --git a/actionProxyLoop/Dockerfile b/actionloop/Dockerfile
similarity index 100%
rename from actionProxyLoop/Dockerfile
rename to actionloop/Dockerfile
diff --git a/actionProxyLoop/build.gradle b/actionloop/build.gradle
similarity index 74%
rename from actionProxyLoop/build.gradle
rename to actionloop/build.gradle
index b0586ce..4daba91 100644
--- a/actionProxyLoop/build.gradle
+++ b/actionloop/build.gradle
@@ -15,5 +15,19 @@
* limitations under the License.
*/
-ext.dockerImageName = 'actionloop'
+ext.dockerImageName = 'actionloop-v2'
apply from: '../gradle/docker.gradle'
+
+distDocker.dependsOn 'copyProxy'
+distDocker.finalizedBy('cleanup')
+
+task copyProxy(type: Copy) {
+ from '../common/proxy'
+ into '.'
+}
+
+task cleanup(type: Delete) {
+ delete 'proxy'
+ delete 'gobuild.py'
+ delete 'gobuild.py.launcher.go'
+}
diff --git a/build.gradle b/build.gradle
index 18cebe0..149426f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -46,10 +46,13 @@ golang {
goVersion = '1.11.5'
}
-
build.dependsOn vendor
build {
targetPlatform = ['linux-amd64']
- go 'build -o actionProxyLoop/proxy main/proxy.go'
+ go """build -o common/proxy -ldflags '-extldflags "-static"'
main/proxy.go"""
+}
+
+task cleanup(type: Delete) {
+ delete 'common/proxy'
}
diff --git a/examples/Makefile b/examples/Makefile
index 8774e4d..538ec75 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -3,6 +3,8 @@ IMAGES?=openwhisk
all: .PHONY
+.PHONY: bash-hello golang-hello-single golang-main-single
golang-main-standalone golang-main-package golang-main-vendor
golang-hello-vendor
+
bash-hello:
cd $@ && OW_USER=$(IMAGES) make clean deploy test
@@ -30,5 +32,4 @@ golang-hello-vendor:
cd $@ && OW_USER=$(IMAGES) make clean devel test
cd $@ && OW_USER=$(IMAGES) make clean deploy test
-.PHONY: bash-hello golang-hello-single golang-main-single
golang-main-standalone golang-main-package golang-main-vendor
golang-hello-vendor
diff --git a/examples/bash-hello/Makefile b/examples/bash-hello/Makefile
index f0e4914..29c12de 100644
--- a/examples/bash-hello/Makefile
+++ b/examples/bash-hello/Makefile
@@ -1,6 +1,6 @@
WSK?=wsk
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
NAME=bash-hello
PACKAGE=test
SRC=hello.sh
diff --git a/examples/golang-hello-single/Makefile
b/examples/golang-hello-single/Makefile
index c88906a..c74fdba 100644
--- a/examples/golang-hello-single/Makefile
+++ b/examples/golang-hello-single/Makefile
@@ -1,5 +1,5 @@
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
OW_COMPILER?=$(OW_USER)/actionloop-golang-v1.11
WSK?=wsk
MAIN=hello
@@ -19,7 +19,7 @@ $(ZIP): $(SRC)
clean:
-$(WSK) action delete $(PACKAGE)/$(NAME)
- -rm $(ZIP) package.done test.json
+ -rm $(ZIP) package.done test.json 2>/dev/null
test: test.json
$(WSK) action invoke test/$(NAME) -r
diff --git a/examples/golang-hello-vendor/Makefile
b/examples/golang-hello-vendor/Makefile
index 120dcd9..9d417fb 100644
--- a/examples/golang-hello-vendor/Makefile
+++ b/examples/golang-hello-vendor/Makefile
@@ -1,5 +1,5 @@
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
OW_COMPILER?=$(OW_USER)/actionloop-golang-v1.11
WSK?=wsk
MAIN=hello
@@ -23,14 +23,14 @@ $(BINZIP): $(SRCS) $(VENDORS) $(SRCZIP)
docker run -i $(OW_COMPILER) -compile $(MAIN) <$(SRCZIP) >$(BINZIP)
$(SRCZIP): $(SRCS) $(VENDORS)
- cd src ; zip ../$(SRCZIP) -r *
+ cd src ; zip ../$(SRCZIP) -qr *
clean:
-$(WSK) action delete $(PACKAGE)/$(NAME)
- -rm $(BINZIP) $(SRCZIP) package.done test.json
+ -rm $(BINZIP) $(SRCZIP) package.done test.json 2>/dev/null
clean_vendor:
- -rm -r $(VENDORS)
+ -rm -r $(VENDORS)
test: test.json
$(WSK) action invoke test/$(NAME) -r
diff --git a/examples/golang-main-package/Makefile
b/examples/golang-main-package/Makefile
index c097852..d42a9df 100644
--- a/examples/golang-main-package/Makefile
+++ b/examples/golang-main-package/Makefile
@@ -1,5 +1,5 @@
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
OW_COMPILER?=$(OW_USER)/actionloop-golang-v1.11
WSK?=wsk
MAIN=main
@@ -16,14 +16,14 @@ devel: package.done $(SRCZIP)
$(WSK) action update $(PACKAGE)/$(NAME) $(SRCZIP) --main $(MAIN)
--docker $(OW_COMPILER)
$(BINZIP): $(SRCS)
- cd src ; zip - -r * | docker run -i $(OW_COMPILER) -compile $(MAIN)
>../$(BINZIP)
+ cd src ; zip - -qr * | docker run -i $(OW_COMPILER) -compile $(MAIN)
>../$(BINZIP)
$(SRCZIP): $(SRCS)
- cd src ; zip ../$(SRCZIP) -r *
+ cd src ; zip ../$(SRCZIP) -qr *
clean:
-$(WSK) action delete $(PACKAGE)/$(NAME)
- -rm $(BINZIP) $(SRCZIP) package.done test.json
+ -rm $(BINZIP) $(SRCZIP) package.done test.json 2>/dev/null
test: test.json
$(WSK) action invoke test/$(NAME) -r
diff --git a/examples/golang-main-single/Makefile
b/examples/golang-main-single/Makefile
index 64e2415..e861905 100644
--- a/examples/golang-main-single/Makefile
+++ b/examples/golang-main-single/Makefile
@@ -1,5 +1,5 @@
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
OW_COMPILER?=$(OW_USER)/actionloop-golang-v1.11
WSK?=wsk
MAIN=main
@@ -19,7 +19,7 @@ $(ZIP): $(SRC)
clean:
-$(WSK) action delete $(PACKAGE)/$(NAME)
- -rm $(ZIP) package.done test.json
+ -rm $(ZIP) package.done test.json 2>/dev/null
test: test.json
$(WSK) action invoke test/$(NAME) -r
diff --git a/examples/golang-main-standalone/Makefile
b/examples/golang-main-standalone/Makefile
index f7a8ec4..4918436 100644
--- a/examples/golang-main-standalone/Makefile
+++ b/examples/golang-main-standalone/Makefile
@@ -1,5 +1,5 @@
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
OW_COMPILER?=$(OW_USER)/actionloop-golang-v1.11
WSK?=wsk
MAIN=main
diff --git a/examples/golang-main-vendor/Makefile
b/examples/golang-main-vendor/Makefile
index 06ea478..bca6b37 100644
--- a/examples/golang-main-vendor/Makefile
+++ b/examples/golang-main-vendor/Makefile
@@ -1,5 +1,5 @@
OW_USER?=openwhisk
-OW_RUNTIME?=$(OW_USER)/actionloop
+OW_RUNTIME?=$(OW_USER)/actionloop-v2
OW_COMPILER?=$(OW_USER)/actionloop-golang-v1.11
WSK?=wsk
MAIN=main
@@ -23,7 +23,7 @@ $(BINZIP): $(SRCS) $(VENDORS) $(SRCZIP)
docker run -i $(OW_COMPILER) -compile $(MAIN) <$(SRCZIP) >$(BINZIP)
$(SRCZIP): $(SRCS) $(VENDORS)
- cd src ; zip ../$(SRCZIP) -r *
+ cd src ; zip ../$(SRCZIP) -qr *
clean:
-$(WSK) action delete $(PACKAGE)/$(NAME)
diff --git a/golang1.11/build.gradle b/golang1.11/build.gradle
index ea0731c..cfe8e61 100644
--- a/golang1.11/build.gradle
+++ b/golang1.11/build.gradle
@@ -23,9 +23,8 @@ distDocker.dependsOn 'copyCompiler'
distDocker.dependsOn 'copyEpilogue'
distDocker.finalizedBy('cleanup')
-
task copyProxy(type: Copy) {
- from '../actionProxyLoop/proxy'
+ from '../common/proxy'
into '.'
}
diff --git a/main/proxy.go b/main/proxy.go
index 1fb9cdf..7f35914 100644
--- a/main/proxy.go
+++ b/main/proxy.go
@@ -17,11 +17,8 @@
package main
import (
- "archive/zip"
- "bytes"
"flag"
"fmt"
- "io/ioutil"
"log"
"os"
@@ -44,38 +41,12 @@ func fatalIf(err error) {
}
}
-// use the runtime as a compiler "on-the-fly"
-func extractAndCompile(ap *openwhisk.ActionProxy) {
-
- // read the std input
- in, err := ioutil.ReadAll(os.Stdin)
- fatalIf(err)
-
- // extract and compile it
- file, err := ap.ExtractAndCompile(&in, *compile)
- fatalIf(err)
-
- // read the file, zip it and write it to stdout
- buf := new(bytes.Buffer)
- zwr := zip.NewWriter(buf)
- zf, err := zwr.Create("exec")
- fatalIf(err)
- filedata, err := ioutil.ReadFile(file)
- fatalIf(err)
- _, err = zf.Write(filedata)
- fatalIf(err)
- fatalIf(zwr.Flush())
- fatalIf(zwr.Close())
- _, err = os.Stdout.Write(buf.Bytes())
- fatalIf(err)
-}
-
func main() {
flag.Parse()
// show version number
if *version {
- fmt.Println("OpenWhisk ActionLoop Proxy", openwhisk.Version)
+ fmt.Printf("OpenWhisk ActionLoop Proxy v%s\n",
openwhisk.Version)
return
}
@@ -91,7 +62,7 @@ func main() {
// compile on the fly upon request
if *compile != "" {
- extractAndCompile(ap)
+ ap.ExtractAndCompileIO(os.Stdin, os.Stdout, *compile)
return
}
diff --git a/openwhisk/_test/build.sh b/openwhisk/_test/build.sh
index 58f80e8..42707f8 100755
--- a/openwhisk/_test/build.sh
+++ b/openwhisk/_test/build.sh
@@ -35,15 +35,15 @@ function build_main {
build hi
-zip hi.zip exec
+zip -q hi.zip exec
cp exec hi
build_main hello_message
-zip hello_message.zip exec
+zip -q hello_message.zip exec
cp exec hello_message
build_main hello_greeting
-zip hello_greeting.zip exec
+zip -q hello_greeting.zip exec
cp exec hello_greeting
test -e hello.zip && rm hello.zip
diff --git a/openwhisk/_test/compile.py b/openwhisk/_test/compile.py
new file mode 100755
index 0000000..e52486a
--- /dev/null
+++ b/openwhisk/_test/compile.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+# license agreements; and to You under the Apache License, Version 2.0.
+import os, sys
+os.rename(sys.argv[2], sys.argv[3]+"/action")
+with open(sys.argv[3]+"/exec", "w") as f:
+ f.write("""#!/bin/bash
+cd "$(dirname $0)"
+export PYTHONPATH=$PWD/action
+python action/exec.py
+""")
diff --git a/openwhisk/_test/build.sh b/openwhisk/_test/find.sh
similarity index 54%
copy from openwhisk/_test/build.sh
copy to openwhisk/_test/find.sh
index 58f80e8..20255ba 100755
--- a/openwhisk/_test/build.sh
+++ b/openwhisk/_test/find.sh
@@ -15,42 +15,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-cd "$(dirname $0)"
-
-function build {
- test -e exec && rm exec
- cp $1.src $1.go
- GOPATH=$PWD go build -a -o exec $1.go
- rm $1.go
-}
-
-function build_main {
- test -e exec && rm exec
- cp ../../common/gobuild.py.launcher.go $1.go
- cat $1.src >>$1.go
- go build -a -o exec $1.go
- rm $1.go
-}
-
-
-build hi
-zip hi.zip exec
-cp exec hi
-
-build_main hello_message
-zip hello_message.zip exec
-cp exec hello_message
-
-build_main hello_greeting
-zip hello_greeting.zip exec
-cp exec hello_greeting
-
-test -e hello.zip && rm hello.zip
-cd src
-zip -q -r ../hello.zip main.go hello
-cd ..
-
-build exec
-test -e exec.zip && rm exec.zip
-zip -q -r exec.zip exec etc dir
+find "$1" | sort
diff --git a/openwhisk/_test/pysample/exec b/openwhisk/_test/pysample/exec
new file mode 100755
index 0000000..eedf580
--- /dev/null
+++ b/openwhisk/_test/pysample/exec
@@ -0,0 +1,6 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+# license agreements; and to You under the Apache License, Version 2.0.
+cd "$(dirname $0)"
+export PYTHONPATH=$PWD/lib
+python lib/exec.py
diff --git a/openwhisk/_test/pysample/lib/action/__init__.py
b/openwhisk/_test/pysample/lib/action/__init__.py
new file mode 100644
index 0000000..e4e2475
--- /dev/null
+++ b/openwhisk/_test/pysample/lib/action/__init__.py
@@ -0,0 +1,2 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+# license agreements; and to You under the Apache License, Version 2.0.
diff --git a/openwhisk/_test/pysample/lib/action/main.py
b/openwhisk/_test/pysample/lib/action/main.py
new file mode 100644
index 0000000..1796dbf
--- /dev/null
+++ b/openwhisk/_test/pysample/lib/action/main.py
@@ -0,0 +1,7 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+# license agreements; and to You under the Apache License, Version 2.0.
+def main(args):
+ name = "world"
+ if "name" in args:
+ name = args["name"]
+ return {"python": "Hello, %s" % name }
diff --git a/openwhisk/_test/pysample/lib/exec.py
b/openwhisk/_test/pysample/lib/exec.py
new file mode 100644
index 0000000..4bd6ac1
--- /dev/null
+++ b/openwhisk/_test/pysample/lib/exec.py
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
contributor
+# license agreements; and to You under the Apache License, Version 2.0.
+
+from __future__ import print_function
+import os, json
+from action.main import main
+inp = os.fdopen(0, "rb")
+out = os.fdopen(3, "wb")
+while True:
+ while True:
+ line = inp.readline()
+ args = json.loads(line)
+ payload = {}
+ if "value" in args:
+ payload = args["value"]
+ res = main(payload)
+ out.write(json.dumps(res, ensure_ascii=False).encode('utf-8'))
+ out.write("\n")
+ out.flush()
+
diff --git a/openwhisk/actionProxy.go b/openwhisk/actionProxy.go
index cb6e441..22bb31c 100644
--- a/openwhisk/actionProxy.go
+++ b/openwhisk/actionProxy.go
@@ -19,9 +19,12 @@ package openwhisk
import (
"fmt"
+ "io"
+ "io/ioutil"
"log"
"net/http"
"os"
+ "path/filepath"
)
// ActionProxy is the container of the data specific to a server
@@ -100,7 +103,6 @@ func (ap *ActionProxy) StartLatestAction() error {
Debug("removing the failed action in %s", exeDir)
os.RemoveAll(exeDir)
}
-
return err
}
@@ -115,9 +117,33 @@ func (ap *ActionProxy) ServeHTTP(w http.ResponseWriter, r
*http.Request) {
// Start creates a proxy to execute actions
func (ap *ActionProxy) Start(port int) {
-
// listen and start
- //http.HandleFunc("/init", func(w http.ResponseWriter, r *http.Request)
{ ap.initHandler(w, r) })
- //http.HandleFunc("/run", func(w http.ResponseWriter, r *http.Request)
{ ap.runHandler(w, r) })
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), ap))
}
+
+// ExtractAndCompileIO read in input and write in output to use the runtime as
a compiler "on-the-fly"
+func (ap *ActionProxy) ExtractAndCompileIO(r io.Reader, w io.Writer, main
string) {
+
+ // read the std input
+ in, err := ioutil.ReadAll(r)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // extract and compile it
+ file, err := ap.ExtractAndCompile(&in, main)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // zip the directory containing the file and write output
+ zip, err := Zip(filepath.Dir(file))
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ _, err = w.Write(zip)
+ if err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/openwhisk/actionProxy_test.go b/openwhisk/actionProxy_test.go
index 2d767af..31901c0 100644
--- a/openwhisk/actionProxy_test.go
+++ b/openwhisk/actionProxy_test.go
@@ -18,8 +18,10 @@
package openwhisk
import (
+ "bytes"
"fmt"
"io/ioutil"
+ "log"
"os"
"testing"
@@ -81,3 +83,45 @@ func TestStartLatestAction_emit2(t *testing.T) {
/**/
ap.theExecutor.Stop()
}
+
+func Example_compile_bin() {
+ os.RemoveAll("./action/c1")
+ logf, _ := ioutil.TempFile("/tmp", "log")
+ ap := NewActionProxy("./action/c1", "_test/compile.py", logf, logf)
+ dat, _ := Zip("_test/pysample")
+ inp := bytes.NewBuffer(dat)
+ out := new(bytes.Buffer)
+ ap.ExtractAndCompileIO(inp, out, "main")
+ Unzip(out.Bytes(), "./action/c1/out")
+ sys("_test/find.sh", "./action/c1/out")
+ // Output:
+ // ./action/c1/out
+ // ./action/c1/out/exec
+ // ./action/c1/out/lib
+ // ./action/c1/out/lib/action
+ // ./action/c1/out/lib/action/__init__.py
+ // ./action/c1/out/lib/action/main.py
+ // ./action/c1/out/lib/exec.py
+}
+
+func Example_compile_src() {
+ os.RemoveAll("./action/c2")
+ logf, _ := ioutil.TempFile("/tmp", "log")
+ ap := NewActionProxy("./action/c2", "_test/compile.py", logf, logf)
+ log.Println(ioutil.ReadAll(logf))
+ dat, _ := Zip("_test/pysample/lib")
+ inp := bytes.NewBuffer(dat)
+ out := new(bytes.Buffer)
+ ap.ExtractAndCompileIO(inp, out, "main")
+ Unzip(out.Bytes(), "./action/c2/out")
+ sys("_test/find.sh", "./action/c2/out")
+ // Output:
+ // ./action/c2/out
+ // ./action/c2/out/action
+ // ./action/c2/out/action/action
+ // ./action/c2/out/action/action/__init__.py
+ // ./action/c2/out/action/action/main.py
+ // ./action/c2/out/action/exec.py
+ // ./action/c2/out/exec
+
+}
diff --git a/openwhisk/compiler_test.go b/openwhisk/compiler_test.go
index ca43b2a..ed91ad4 100644
--- a/openwhisk/compiler_test.go
+++ b/openwhisk/compiler_test.go
@@ -50,7 +50,7 @@ const (
)
// compile a main
-func Example_compile() {
+func Example_cli_compiler() {
sys(PREP, "hello.src", "0", "exec")
ap := NewActionProxy(TMP, COMP, os.Stdout, os.Stderr)
fmt.Println(isCompiled(TMP + "0/src/exec"))
diff --git a/openwhisk/extractor.go b/openwhisk/extractor.go
index 8140151..f750fc6 100644
--- a/openwhisk/extractor.go
+++ b/openwhisk/extractor.go
@@ -18,66 +18,12 @@
package openwhisk
import (
- "archive/zip"
- "bytes"
"fmt"
- "io"
"io/ioutil"
"os"
- "path/filepath"
"strconv"
)
-func unzip(src []byte, dest string) error {
- reader := bytes.NewReader(src)
- r, err := zip.NewReader(reader, int64(len(src)))
- if err != nil {
- return err
- }
-
- os.MkdirAll(dest, 0755)
-
- // Closure to address file descriptors issue with all the deferred
.Close() methods
- extractAndWriteFile := func(f *zip.File) error {
- rc, err := f.Open()
- if err != nil {
- return err
- }
- defer func() {
- rc.Close()
- }()
-
- path := filepath.Join(dest, f.Name)
-
- if f.FileInfo().IsDir() {
- os.MkdirAll(path, f.Mode())
- } else {
- os.MkdirAll(filepath.Dir(path), f.Mode())
- f, err := os.OpenFile(path,
os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
- if err != nil {
- return err
- }
- defer func() {
- f.Close()
- }()
-
- _, err = io.Copy(f, rc)
- if err != nil {
- return err
- }
- }
- return nil
- }
-
- for _, f := range r.File {
- err := extractAndWriteFile(f)
- if err != nil {
- return err
- }
- }
- return nil
-}
-
// higherDir will find the highest numeric name a sub directory has
// 0 if no numeric dir names found
func highestDir(dir string) int {
@@ -111,7 +57,7 @@ func (ap *ActionProxy) ExtractAction(buf *[]byte, suffix
string) (string, error)
file := newDir + "/exec"
if IsZip(*buf) {
Debug("Extract Action, assuming a zip")
- return file, unzip(*buf, newDir)
+ return file, Unzip(*buf, newDir)
}
return file, ioutil.WriteFile(file, *buf, 0755)
}
diff --git a/openwhisk/extractor_test.go b/openwhisk/extractor_test.go
index 891d7d2..c3009ad 100644
--- a/openwhisk/extractor_test.go
+++ b/openwhisk/extractor_test.go
@@ -74,7 +74,7 @@ func TestHighestDir(t *testing.T) {
}
// Issue #62 sample zip
-func ExampleBadZip() {
+func Example_badZip() {
buf := []byte{
0x50, 0x4b, 0x03, 0x04, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00,
0x81, 0xb0, 0x81, 0x4d, 0x2d, 0xf6,
0xa5, 0x66, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00,
0x09, 0x00, 0x1c, 0x00, 0x69, 0x6e,
diff --git a/openwhisk/filetype.go b/openwhisk/filetype.go
index f4d6c3d..4825837 100644
--- a/openwhisk/filetype.go
+++ b/openwhisk/filetype.go
@@ -37,13 +37,19 @@ func IsMach64(buf []byte) bool {
buf[2] == 0xed && buf[3] == 0xfe
}
+// IsBangPath checks for a shell executable
+func IsBangPath(buf []byte) bool {
+ return len(buf) > 2 &&
+ buf[0] == '#' && buf[1] == '!'
+}
+
// IsExecutable check if it is an executable, according the current runtime
func IsExecutable(buf []byte, runtime string) bool {
switch runtime {
case "darwin":
- return IsMach64(buf)
+ return IsMach64(buf) || IsBangPath(buf)
case "linux":
- return IsElf(buf)
+ return IsElf(buf) || IsBangPath(buf)
case "windows":
return IsExe(buf)
default:
diff --git a/openwhisk/filetype_test.go b/openwhisk/filetype_test.go
index 5f016c6..1ee6986 100644
--- a/openwhisk/filetype_test.go
+++ b/openwhisk/filetype_test.go
@@ -37,6 +37,8 @@ var windowsFile = []byte{
0x4d, 0x5a, 0x90, 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0x00, 0x00,
}
+var shellFile = []byte("#!/bin/sh\necho hello\n")
+
func Example_filetype() {
fmt.Printf("%t\n%t\n", IsElf(linuxFile), IsElf(zipFile))
fmt.Printf("%t\n%t\n", IsMach64(darwinFile), IsMach64(zipFile))
@@ -45,6 +47,7 @@ func Example_filetype() {
fmt.Printf("%t\n%t\n", IsExecutable(linuxFile, "linux"),
IsExecutable(zipFile, "linux"))
fmt.Printf("%t\n%t\n", IsExecutable(windowsFile, "windows"),
IsExecutable(zipFile, "windows"))
fmt.Printf("%t\n%t\n", IsExecutable(darwinFile, "darwin"),
IsExecutable(zipFile, "darwin"))
+ fmt.Printf("%t\n%t\n%t\n", IsExecutable(shellFile, "darwin"),
IsExecutable(shellFile, "linux"), IsExecutable(shellFile, "windows"))
// Output:
// true
// false
@@ -60,5 +63,8 @@ func Example_filetype() {
// false
// true
// false
+ // true
+ // true
+ // false
}
diff --git a/openwhisk/initHandler.go b/openwhisk/initHandler.go
index 73b5ffd..474e85c 100644
--- a/openwhisk/initHandler.go
+++ b/openwhisk/initHandler.go
@@ -127,14 +127,9 @@ func (ap *ActionProxy) initHandler(w http.ResponseWriter,
r *http.Request) {
// ExtractAndCompile decode the buffer and if a compiler is defined, compile
it also
func (ap *ActionProxy) ExtractAndCompile(buf *[]byte, main string) (string,
error) {
- // extract in "bin" or in "src" if the runtime can compile
- suffix := "bin"
- if ap.compiler != "" {
- suffix = "src"
- }
- // extract action
- file, err := ap.ExtractAction(buf, suffix)
+ // extract action in src folder
+ file, err := ap.ExtractAction(buf, "src")
if err != nil {
return "", err
}
@@ -148,26 +143,21 @@ func (ap *ActionProxy) ExtractAndCompile(buf *[]byte,
main string) (string, erro
srcDir := filepath.Join(parent, "src")
binDir := filepath.Join(parent, "bin")
binFile := filepath.Join(binDir, "exec")
- os.Mkdir(binDir, 0755)
-
- // if the file is already compiled just move it from src to bin
- if isCompiled(file) {
- os.Rename(file, binFile)
- return binFile, nil
- }
- // no compiler, move it anyway
- if ap.compiler == "" {
- os.Rename(file, binFile)
+ // if the file is already compiled or there is no compiler just move it
from src to bin
+ if ap.compiler == "" || isCompiled(file) {
+ os.Rename(srcDir, binDir)
return binFile, nil
}
// ok let's try to compile
Debug("compiling: %s main: %s", file, main)
+ os.Mkdir(binDir, 0755)
err = ap.CompileAction(main, srcDir, binDir)
if err != nil {
return "", err
}
+
// check only if the file exist
if _, err := os.Stat(binFile); os.IsNotExist(err) {
return "", fmt.Errorf("cannot compile")
diff --git a/openwhisk/initHandler_test.go b/openwhisk/initHandler_test.go
index de71f6d..c76a86c 100644
--- a/openwhisk/initHandler_test.go
+++ b/openwhisk/initHandler_test.go
@@ -226,3 +226,20 @@ func Example_badinit_nocompiler() {
// 500 {"error":"no action defined yet"}
// hi
}
+
+func Example_zip_init() {
+ ts, cur, log := startTestServer("")
+ buf, _ := Zip("_test/pysample")
+ doInit(ts, initBytes(buf, ""))
+ doRun(ts, ``)
+ doRun(ts, `{"name":"World"}`)
+ stopTestServer(ts, cur, log)
+ // Output:
+ // 200 {"ok":true}
+ // 200 {"python": "Hello, Mike"}
+ // 200 {"python": "Hello, World"}
+ // XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX
+ // XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX
+ // XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX
+ // XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX
+}
diff --git a/openwhisk/util_test.go b/openwhisk/util_test.go
index 59fd5f3..19a0d4c 100644
--- a/openwhisk/util_test.go
+++ b/openwhisk/util_test.go
@@ -110,8 +110,7 @@ func initCode(file string, main string) string {
return string(j)
}
-func initBinary(file string, main string) string {
- dat, _ := ioutil.ReadFile(file)
+func initBytes(dat []byte, main string) string {
enc := base64.StdEncoding.EncodeToString(dat)
body := initBodyRequest{Binary: true, Code: enc}
if main != "" {
@@ -121,6 +120,11 @@ func initBinary(file string, main string) string {
return string(j)
}
+func initBinary(file string, main string) string {
+ dat, _ := ioutil.ReadFile(file)
+ return initBytes(dat, main)
+}
+
func abs(in string) string {
out, _ := filepath.Abs(in)
return out
@@ -166,17 +170,18 @@ func removeLineNr(out string) string {
return re.ReplaceAllString(out, "::")
}
func TestMain(m *testing.M) {
- // Debugging = true // enable debug
- // silence those annoying logs
- if !Debugging {
+ var Debug = false // enable debug of tests
+ if !Debug {
+ // silence those annoying tests
log.SetOutput(ioutil.Discard)
+ // build support files
+ sys("_test/build.sh")
+ sys("_test/zips.sh")
}
// increase timeouts for init
DefaultTimeoutStart = 1000 * time.Millisecond
// build some test stuff
- sys("_test/build.sh")
- sys("_test/zips.sh")
// go ahead
code := m.Run()
os.Exit(code)
diff --git a/openwhisk/version.go b/openwhisk/version.go
index 27af90b..3640360 100644
--- a/openwhisk/version.go
+++ b/openwhisk/version.go
@@ -17,4 +17,4 @@
package openwhisk
// Version number - internal
-var Version = "1.0.1"
+var Version = "2"
diff --git a/openwhisk/zip.go b/openwhisk/zip.go
new file mode 100644
index 0000000..36ae387
--- /dev/null
+++ b/openwhisk/zip.go
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package openwhisk
+
+import (
+ "archive/zip"
+ "bytes"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// Unzip extracts file and directories in the given destination folder
+func Unzip(src []byte, dest string) error {
+ reader := bytes.NewReader(src)
+ r, err := zip.NewReader(reader, int64(len(src)))
+ if err != nil {
+ return err
+ }
+ os.MkdirAll(dest, 0755)
+ // Closure to address file descriptors issue with all the deferred
.Close() methods
+ extractAndWriteFile := func(f *zip.File) error {
+ rc, err := f.Open()
+ defer rc.Close()
+ if err != nil {
+ return err
+ }
+ path := filepath.Join(dest, f.Name)
+ if f.FileInfo().IsDir() {
+ return os.MkdirAll(path, f.Mode())
+ }
+ err = os.MkdirAll(filepath.Dir(path), 0755)
+ if err != nil {
+ return err
+ }
+ file, err := os.OpenFile(path,
os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+ defer file.Close()
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(file, rc)
+ return err
+ }
+ for _, f := range r.File {
+ err := extractAndWriteFile(f)
+ if err != nil {
+ log.Println(err)
+ }
+ }
+ return nil
+}
+
+// Zip a directory
+func Zip(dir string) ([]byte, error) {
+ buf := new(bytes.Buffer)
+ zwr := zip.NewWriter(buf)
+ dir = filepath.Clean(dir)
+ err := filepath.Walk(dir, func(filePath string, info os.FileInfo, err
error) error {
+ if info.IsDir() {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ relPath := strings.TrimPrefix(filePath, dir)[1:]
+ zipFile, err := zwr.Create(relPath)
+ if err != nil {
+ return err
+ }
+ fsFile, err := os.Open(filePath)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(zipFile, fsFile)
+ if err != nil {
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ err = zwr.Close()
+ if err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
diff --git a/openwhisk/version.go b/openwhisk/zip_test.go
similarity index 62%
copy from openwhisk/version.go
copy to openwhisk/zip_test.go
index 27af90b..a2c8eef 100644
--- a/openwhisk/version.go
+++ b/openwhisk/zip_test.go
@@ -16,5 +16,27 @@
*/
package openwhisk
-// Version number - internal
-var Version = "1.0.1"
+import (
+ "fmt"
+ "os"
+)
+
+func Example() {
+ os.RemoveAll("./action/unzip")
+ os.Mkdir("./action/unzip", 0755)
+ buf, err := Zip("_test/pysample")
+ fmt.Println(err)
+ err = Unzip(buf, "./action/unzip")
+ sys("_test/find.sh", "./action/unzip")
+ fmt.Println(err)
+ // Output:
+ // <nil>
+ // ./action/unzip
+ // ./action/unzip/exec
+ // ./action/unzip/lib
+ // ./action/unzip/lib/action
+ // ./action/unzip/lib/action/__init__.py
+ // ./action/unzip/lib/action/main.py
+ // ./action/unzip/lib/exec.py
+ // <nil>
+}
diff --git a/settings.gradle b/settings.gradle
index 01343e2..15eaa52 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -17,7 +17,7 @@
include 'tests'
-include 'actionProxyLoop'
+include 'actionloop'
include 'golang1.11'
rootProject.name = 'runtime-golang'
@@ -35,4 +35,3 @@ gradle.ext.scalafmt = [
version: '1.5.0',
config: new File(rootProject.projectDir, '.scalafmt.conf')
]
-
diff --git
a/tests/src/test/scala/runtime/actionContainers/ActionLoopBasicTests.scala
b/tests/src/test/scala/runtime/actionContainers/ActionLoopBasicTests.scala
index 7936af4..1a346ce 100644
--- a/tests/src/test/scala/runtime/actionContainers/ActionLoopBasicTests.scala
+++ b/tests/src/test/scala/runtime/actionContainers/ActionLoopBasicTests.scala
@@ -25,7 +25,7 @@ import org.scalatest.junit.JUnitRunner
@RunWith(classOf[JUnitRunner])
class ActionLoopBasicTests extends BasicActionRunnerTests with WskActorSystem {
- val image = "actionloop"
+ val image = "actionloop-v2"
override def withActionContainer(env: Map[String, String] = Map.empty)(
code: ActionContainer => Unit) = {
diff --git
a/tests/src/test/scala/runtime/actionContainers/ActionLoopContainerTests.scala
b/tests/src/test/scala/runtime/actionContainers/ActionLoopContainerTests.scala
index 0e35784..59ff555 100644
---
a/tests/src/test/scala/runtime/actionContainers/ActionLoopContainerTests.scala
+++
b/tests/src/test/scala/runtime/actionContainers/ActionLoopContainerTests.scala
@@ -32,12 +32,12 @@ class ActionLoopContainerTests
import GoResourceHelpers._
- val image = "actionloop"
+ val image = "actionloop-v2"
def withActionLoopContainer(code: ActionContainer => Unit) =
- withContainer("actionloop")(code)
+ withContainer(image)(code)
- behavior of "actionloop"
+ behavior of image
def shCodeHello(main: String) = Seq(
Seq(main) ->
