This patch enhances OSv build scripts to allow exporting files in addition
of uploading them to an image. It addresses 3 usage scenarios described in
 #900 and demostrated by examples below:

- ./scripts/build image=empty export=all - export files that are specified is 
usr.manifest.skel in order to be able to produce capstan osv.bootstrap package

- ./scripts/build image=java-isolated export=all usrskel=none - export files 
that are part of OSv modules (./modules directory) along with all other 
dependent modules minus what is part of usr.manifest.skel

- ./scripts/build image=openjdk8-zulu-compact1 export=selected usrskel=none - 
export files that are part of an app (most applicable to apps/openjdk8-****) 
without any dependent modules

The export logic is enabled by passing new parameter export 
[=none|all|selected] to scripts/build. 
Exported files are placed under build/export directory. 

Please note that the changes are backwards compatible. However changes to 
scripts/upload_manifest.py
 will break patches that are part of 
https://github.com/mikelangelo-project/capstan-packages/tree/master/docker_files/common.
 Hopefully this patch will make some of these patches obsolete.

Fixed #900

Signed-off-by: Waldemar Kozaczuk <jwkozac...@gmail.com>
---
 modules/empty/module.py    |  0
 scripts/build              | 16 +++++++-
 scripts/export_manifest.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++
 scripts/manifest_common.py | 78 +++++++++++++++++++++++++++++++++++++++
 scripts/module.py          |  8 +++-
 scripts/upload_manifest.py | 79 +---------------------------------------
 6 files changed, 192 insertions(+), 80 deletions(-)
 create mode 100644 modules/empty/module.py
 create mode 100755 scripts/export_manifest.py
 create mode 100644 scripts/manifest_common.py

diff --git a/modules/empty/module.py b/modules/empty/module.py
new file mode 100644
index 0000000..e69de29
diff --git a/scripts/build b/scripts/build
index 7cee19c..14b4fe1 100755
--- a/scripts/build
+++ b/scripts/build
@@ -62,6 +62,7 @@ then
        exit 2
 fi
 OUT=build/`readlink build/last`
+EXPORT_DIR=`realpath build/export`
 
 # Look for arguments that look like "name=value", and set them in an
 # assoative array "vars". Also look for the "-j ..." parallelism option,
@@ -69,6 +70,7 @@ OUT=build/`readlink build/last`
 declare -A vars
 j_arg=
 checknumber=
+vars[export]=none
 for i
 do
        if test -n "$checknumber"
@@ -147,7 +149,12 @@ then
        usrskel_arg="--usrskel ${vars[usrskel]}"
 fi
 
-jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC 
OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules 
$usrskel_arg
+export=${vars[export]}
+if [ "$export" != "none" ]
+then
+    rm -rf $EXPORT_DIR
+fi
+jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC 
OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules 
$usrskel_arg --export $export
 
 bootfs_manifest=$manifest make "${args[@]}" | tee -a build.out
 # check exit status of make
@@ -180,7 +187,12 @@ zfs)
        qemu-img convert -f raw -O qcow2 bare.raw usr.img
        qemu-img resize usr.img ${fs_size}b >/dev/null 2>&1
 
-       $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D 
jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D 
miscbase=$miscbase
+       if [ "$export" == "none" ]
+       then
+               $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D 
jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D 
miscbase=$miscbase
+    else
+               $SRC/scripts/export_manifest.py -e $EXPORT_DIR -m usr.manifest 
-D jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D 
miscbase=$miscbase
+       fi
        ;;
 ramfs)
        qemu-img convert -f raw -O qcow2 loader.img usr.img
diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py
new file mode 100755
index 0000000..f5a957a
--- /dev/null
+++ b/scripts/export_manifest.py
@@ -0,0 +1,91 @@
+#!/usr/bin/python
+
+import optparse, os, shutil
+from manifest_common import add_var, expand, unsymlink, read_manifest, 
defines, strip_file
+
+# This will export the package based on the provided manifest file. It uses 
the same mechanism to
+# get the files that need copying as the actual upload process. The only 
current limitation is
+# support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts.
+def export_package(manifest, dest):
+    abs_dest = os.path.abspath(dest)
+
+    files = list(expand(manifest))
+    files = [(x, unsymlink(y % defines)) for (x, y) in files]
+
+    # Create the base directory where we are going to put all package files.
+    if not os.path.exists(abs_dest):
+        os.makedirs(abs_dest)
+
+    for name, hostname in files:
+        name = name[1:] if name.startswith("/") else name
+        name = os.path.join(abs_dest, name)
+
+        if hostname.startswith("->"):
+            link_source = hostname[2:]
+            target_dir = os.path.dirname(name)
+
+            if link_source.startswith("/"):
+                link_source = os.path.join(abs_dest, link_source[1:])
+            else:
+                link_source = os.path.abspath(os.path.join(target_dir, 
link_source))
+
+            link_source = os.path.relpath(link_source, target_dir)
+
+            if not os.path.exists(target_dir):
+                os.makedirs(target_dir)
+
+            os.symlink(link_source, name)
+            print "[INFO] added link %s -> %s" % (name, link_source)
+
+        else:
+            # If it is a file, copy it to the target directory.
+            if os.path.isfile(hostname):
+                # Make sure the target dir exists
+                dirname = os.path.dirname(name)
+                if not os.path.exists(dirname):
+                    os.makedirs(dirname)
+
+                if hostname.endswith("-stripped.so"):
+                    continue
+                hostname = strip_file(hostname)
+
+                shutil.copy(hostname, name)
+                print "[INFO] exported %s" % name
+            elif os.path.isdir(hostname):
+                # If hostname is a dir, it is only a request to create the 
folder on guest. Nothing to copy.
+                if not os.path.exists(name):
+                    os.makedirs(name)
+                print "[INFO] created dir %s" % name
+
+            else:
+                # Inform the user that the rule cannot be applied. For 
example, this happens for links in OSv.
+                print "[ERR] unable to export %s" % hostname
+
+
+def main():
+    make_option = optparse.make_option
+
+    opt = optparse.OptionParser(option_list=[
+            make_option('-m',
+                        dest='manifest',
+                        help='read manifest from FILE',
+                        metavar='FILE'),
+            make_option('-D',
+                        type='string',
+                        help='define VAR=DATA',
+                        metavar='VAR=DATA',
+                        action='callback',
+                        callback=add_var),
+            make_option('-e',
+                        dest='export',
+                        help='exports the contents of the usr.manifest into a 
given folder',
+                        metavar='FILE'),
+    ])
+
+    (options, args) = opt.parse_args()
+
+    manifest = read_manifest(options.manifest)
+    export_package(manifest, options.export)
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py
new file mode 100644
index 0000000..414cce3
--- /dev/null
+++ b/scripts/manifest_common.py
@@ -0,0 +1,78 @@
+#!/usr/bin/python
+
+import os, io, re, subprocess
+
+defines = {}
+
+def add_var(option, opt, value, parser):
+    var, val = value.split('=')
+    defines[var] = val
+
+def expand(items):
+    for name, hostname in items:
+        if name.endswith('/**') and hostname.endswith('/**'):
+            name = name[:-2]
+            hostname = hostname[:-2]
+            for dirpath, dirnames, filenames in os.walk(hostname):
+                for filename in filenames:
+                    relpath = dirpath[len(hostname):]
+                    if relpath != "":
+                        relpath += "/"
+                    yield (name + relpath + filename,
+                           hostname + relpath + filename)
+        elif '/&/' in name and hostname.endswith('/&'):
+            prefix, suffix = name.split('/&/', 1)
+            yield (prefix + '/' + suffix, hostname[:-1] + suffix)
+        else:
+            yield (name, hostname)
+
+def unsymlink(f):
+    if f.startswith('!'):
+        return f[1:]
+    if f.startswith('->'):
+        return f
+    try:
+        link = os.readlink(f)
+        if link.startswith('/'):
+            # try to find a match
+            base = os.path.dirname(f)
+            while not os.path.exists(base + link):
+                if base == '/':
+                    return f
+                base = os.path.dirname(base)
+        else:
+            base = os.path.dirname(f) + '/'
+        return unsymlink(base + link)
+    except Exception:
+        return f
+
+# Reads the manifest and returns it as a list of pairs (guestpath, hostpath).
+def read_manifest(fn):
+    ret = []
+    comment = re.compile("^[ \t]*(|#.*|\[manifest])$")
+    with open(fn, 'r') as f:
+        for line in f:
+            line = line.rstrip();
+            if comment.match(line): continue
+            components = line.split(": ", 2)
+            guestpath = components[0].strip();
+            hostpath = components[1].strip()
+            ret.append((guestpath, hostpath))
+    return ret
+
+def strip_file(filename):
+    def to_strip(filename):
+        ff = os.path.abspath(filename)
+        osvdir = os.path.abspath('../..')
+        return ff.startswith(os.getcwd()) or \
+               ff.startswith(osvdir + "/modules") or \
+               ff.startswith(osvdir + "/apps")
+
+    stripped_filename = filename
+    if filename.endswith(".so") and to_strip(filename):
+        stripped_filename = filename[:-3] + "-stripped.so"
+        if not os.path.exists(stripped_filename) \
+                or (os.path.getmtime(stripped_filename) < \
+                            os.path.getmtime(filename)):
+            subprocess.call(["strip", "-o", stripped_filename, filename])
+    return stripped_filename
diff --git a/scripts/module.py b/scripts/module.py
index ec96e89..f7baf72 100755
--- a/scripts/module.py
+++ b/scripts/module.py
@@ -170,6 +170,7 @@ def build(args):
         print("Using image config: %s" % image_config_file)
         config = resolve.local_import(image_config_file)
         run_list = config.get('run', [])
+        selected_modules = []
     else:
         # If images/image_config doesn't exist, assume image_config is a
         # comma-separated list of module names, and build an image from those
@@ -206,7 +207,10 @@ def build(args):
         # Add moduless thare are implictly required if others are present
         resolve.resolve_required_modules_if_other_is_present()
 
-    modules = resolve.get_required_modules()
+    if args.export == 'selected':
+        modules = list(module for module in resolve.get_required_modules() if 
module.name in selected_modules)
+    else:
+        modules = resolve.get_required_modules()
     modules_to_run = resolve.get_modules_to_run()
 
     print("Modules:")
@@ -260,6 +264,8 @@ if __name__ == "__main__":
                         help="image configuration name. Looked up in " + 
image_configs_dir)
     build_cmd.add_argument("--usrskel", action="store", default="default",
                         help="override default usr.manifest.skel")
+    build_cmd.add_argument("--export", action="store", default="false",
+                           help="export files instead of producing image")
     build_cmd.set_defaults(func=build)
 
     clean_cmd = subparsers.add_parser("clean", help="Clean modules")
diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py
index de570f0..736cd34 100755
--- a/scripts/upload_manifest.py
+++ b/scripts/upload_manifest.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
-import os, optparse, io, subprocess, socket, threading, stat, sys, re
+import optparse, os, subprocess, socket, threading, stat, sys
+from manifest_common import add_var, expand, unsymlink, read_manifest, 
defines, strip_file
 
 try:
     import StringIO
@@ -10,64 +11,6 @@ except ImportError:
     # This works on Python 3
     StringIO = io.StringIO
 
-defines = {}
-
-def add_var(option, opt, value, parser):
-    var, val = value.split('=')
-    defines[var] = val
-
-def expand(items):
-    for name, hostname in items:
-        if name.endswith('/**') and hostname.endswith('/**'):
-            name = name[:-2]
-            hostname = hostname[:-2]
-            for dirpath, dirnames, filenames in os.walk(hostname):
-                for filename in filenames:
-                    relpath = dirpath[len(hostname):]
-                    if relpath != "":
-                        relpath += "/"
-                    yield (name + relpath + filename,
-                           hostname + relpath + filename)
-        elif '/&/' in name and hostname.endswith('/&'):
-            prefix, suffix = name.split('/&/', 1)
-            yield (prefix + '/' + suffix, hostname[:-1] + suffix)
-        else:
-            yield (name, hostname)
-
-def unsymlink(f):
-    if f.startswith('!'):
-        return f[1:]
-    if f.startswith('->'):
-        return f
-    try:
-        link = os.readlink(f)
-        if link.startswith('/'):
-            # try to find a match
-            base = os.path.dirname(f)
-            while not os.path.exists(base + link):
-                if base == '/':
-                    return f
-                base = os.path.dirname(base)
-        else:
-            base = os.path.dirname(f) + '/'
-        return unsymlink(base + link)
-    except Exception:
-        return f
-
-# Reads the manifest and returns it as a list of pairs (guestpath, hostpath).
-def read_manifest(fn):
-    ret = []
-    comment = re.compile("^[ \t]*(|#.*|\[manifest])$")
-    with open(fn, 'r') as f:
-        for line in f:
-            line = line.rstrip();
-            if comment.match(line): continue
-            components = line.split(": ", 2)
-            guestpath = components[0].strip();
-            hostpath = components[1].strip()
-            ret.append((guestpath, hostpath))
-    return ret
-
 def upload(osv, manifest, depends):
     manifest = [(x, y % defines) for (x, y) in manifest]
     files = list(expand(manifest))
@@ -118,24 +61,6 @@ def upload(osv, manifest, depends):
                 + cpio_field(0, 8)                # check
                 + filename + b'\0')
 
-    def to_strip(filename):
-        ff = os.path.abspath(filename);
-        osvdir = os.path.abspath('../..');
-        return ff.startswith(os.getcwd()) or \
-            ff.startswith(osvdir + "/modules") or \
-            ff.startswith(osvdir + "/apps")
-
-    def strip_file(filename):
-        stripped_filename = filename
-        if filename.endswith(".so") and to_strip(filename):
-            stripped_filename = filename[:-3] + "-stripped.so"
-            if not os.path.exists(stripped_filename) \
-                    or (os.path.getmtime(stripped_filename) < \
-                        os.path.getmtime(filename)):
-                subprocess.call(["strip", "-o", stripped_filename, filename])
-        return stripped_filename
-
-
     # Send the files to the guest
     for name, hostname in files:
         if hostname.startswith("->"):
-- 
2.7.4

-- 
You received this message because you are subscribed to the Google Groups "OSv 
Development" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to osv-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

Reply via email to