Thank you Waldek for trying to upstream this, I'm looking forward to be removing the current patches from the capstan-packages build platform in favor of using upstreamed code! :D
Unfortunately I'm not very familiar with the scripts that we're patching here, the patches that we're currently using on capstan-packages build platform were prepared by Gregor (I'm CC-ing him in this comment, hopefully he will be able to review your changes). So I'll not be able to provide a helpful review of your code - however, I'll definetly test it as a user. I've applied your patch locally and tested the `./scripts/build image=empty export=all` command. I have two comments already: 1) I compare directory structure of current osv.bootstrap package with that of build/export and I can see all files are identical except the tools/cpiod.so file. In current osv.bootstrap packages size of tools/cpiod.so file is 1.8 MB, while the cpiod.so provided by `./scripts/build image=empty export=all` is only 220 KB. Do you perhaps have any idea what could cause the difference? 2) When running `./scripts/build image=empty export=all` I've noticed that not only build/export directory is filled with content, but also build/last/loader.img is created. I'm thinking if it would be possible to prevent building actual image in case when `export` mode is enabled - hopefully this would reduce build time. Right now I'm modifying the capstan-packages platform locally to test it with your patches, I'll let you know how it works. I'm looking forward to be hearing your opinion about the two questions above. Dne sreda, 09. avgust 2017 13.47.46 UTC+2 je oseba Waldek Kozaczuk napisala: > > This patch enhances OSv build scripts to allow exporting files in addition > of uploading them to an image. It addresses 3 usage scenarios described in > #900 and demostrated by examples below: > > - ./scripts/build image=empty export=all - export files that are specified > is usr.manifest.skel in order to be able to produce capstan osv.bootstrap > package > > - ./scripts/build image=java-isolated export=all usrskel=none - export > files that are part of OSv modules (./modules directory) along with all > other dependent modules minus what is part of usr.manifest.skel > > - ./scripts/build image=openjdk8-zulu-compact1 export=selected > usrskel=none - export files that are part of an app (most applicable to > apps/openjdk8-****) without any dependent modules > > The export logic is enabled by passing new parameter export > [=none|all|selected] to scripts/build. > Exported files are placed under build/export directory. > > Please note that the changes are backwards compatible. However changes to > scripts/upload_manifest.py > will break patches that are part of > https://github.com/mikelangelo-project/capstan-packages/tree/master/docker_files/common. > > Hopefully this patch will make some of these patches obsolete. > > Fixed #900 > > Signed-off-by: Waldemar Kozaczuk <jwkoz...@gmail.com <javascript:>> > --- > modules/empty/module.py | 0 > scripts/build | 16 +++++++- > scripts/export_manifest.py | 91 > ++++++++++++++++++++++++++++++++++++++++++++++ > scripts/manifest_common.py | 78 +++++++++++++++++++++++++++++++++++++++ > scripts/module.py | 8 +++- > scripts/upload_manifest.py | 79 +--------------------------------------- > 6 files changed, 192 insertions(+), 80 deletions(-) > create mode 100644 modules/empty/module.py > create mode 100755 scripts/export_manifest.py > create mode 100644 scripts/manifest_common.py > > diff --git a/modules/empty/module.py b/modules/empty/module.py > new file mode 100644 > index 0000000..e69de29 > diff --git a/scripts/build b/scripts/build > index 7cee19c..14b4fe1 100755 > --- a/scripts/build > +++ b/scripts/build > @@ -62,6 +62,7 @@ then > exit 2 > fi > OUT=build/`readlink build/last` > +EXPORT_DIR=`realpath build/export` > > # Look for arguments that look like "name=value", and set them in an > # assoative array "vars". Also look for the "-j ..." parallelism option, > @@ -69,6 +70,7 @@ OUT=build/`readlink build/last` > declare -A vars > j_arg= > checknumber= > +vars[export]=none > for i > do > if test -n "$checknumber" > @@ -147,7 +149,12 @@ then > usrskel_arg="--usrskel ${vars[usrskel]}" > fi > > -jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC > OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules > $usrskel_arg > +export=${vars[export]} > +if [ "$export" != "none" ] > +then > + rm -rf $EXPORT_DIR > +fi > +jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC > OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules > $usrskel_arg --export $export > > bootfs_manifest=$manifest make "${args[@]}" | tee -a build.out > # check exit status of make > @@ -180,7 +187,12 @@ zfs) > qemu-img convert -f raw -O qcow2 bare.raw usr.img > qemu-img resize usr.img ${fs_size}b >/dev/null 2>&1 > > - $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D > jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D > miscbase=$miscbase > + if [ "$export" == "none" ] > + then > + $SRC/scripts/upload_manifest.py -o usr.img -m > usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D > glibcbase=$glibcbase -D miscbase=$miscbase > + else > + $SRC/scripts/export_manifest.py -e $EXPORT_DIR -m > usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D > glibcbase=$glibcbase -D miscbase=$miscbase > + fi > ;; > ramfs) > qemu-img convert -f raw -O qcow2 loader.img usr.img > diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py > new file mode 100755 > index 0000000..f5a957a > --- /dev/null > +++ b/scripts/export_manifest.py > @@ -0,0 +1,91 @@ > +#!/usr/bin/python > + > +import optparse, os, shutil > +from manifest_common import add_var, expand, unsymlink, read_manifest, > defines, strip_file > + > +# This will export the package based on the provided manifest file. It > uses the same mechanism to > +# get the files that need copying as the actual upload process. The only > current limitation is > +# support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. > +def export_package(manifest, dest): > + abs_dest = os.path.abspath(dest) > + > + files = list(expand(manifest)) > + files = [(x, unsymlink(y % defines)) for (x, y) in files] > + > + # Create the base directory where we are going to put all package > files. > + if not os.path.exists(abs_dest): > + os.makedirs(abs_dest) > + > + for name, hostname in files: > + name = name[1:] if name.startswith("/") else name > + name = os.path.join(abs_dest, name) > + > + if hostname.startswith("->"): > + link_source = hostname[2:] > + target_dir = os.path.dirname(name) > + > + if link_source.startswith("/"): > + link_source = os.path.join(abs_dest, link_source[1:]) > + else: > + link_source = os.path.abspath(os.path.join(target_dir, > link_source)) > + > + link_source = os.path.relpath(link_source, target_dir) > + > + if not os.path.exists(target_dir): > + os.makedirs(target_dir) > + > + os.symlink(link_source, name) > + print "[INFO] added link %s -> %s" % (name, link_source) > + > + else: > + # If it is a file, copy it to the target directory. > + if os.path.isfile(hostname): > + # Make sure the target dir exists > + dirname = os.path.dirname(name) > + if not os.path.exists(dirname): > + os.makedirs(dirname) > + > + if hostname.endswith("-stripped.so"): > + continue > + hostname = strip_file(hostname) > + > + shutil.copy(hostname, name) > + print "[INFO] exported %s" % name > + elif os.path.isdir(hostname): > + # If hostname is a dir, it is only a request to create > the folder on guest. Nothing to copy. > + if not os.path.exists(name): > + os.makedirs(name) > + print "[INFO] created dir %s" % name > + > + else: > + # Inform the user that the rule cannot be applied. For > example, this happens for links in OSv. > + print "[ERR] unable to export %s" % hostname > + > + > +def main(): > + make_option = optparse.make_option > + > + opt = optparse.OptionParser(option_list=[ > + make_option('-m', > + dest='manifest', > + help='read manifest from FILE', > + metavar='FILE'), > + make_option('-D', > + type='string', > + help='define VAR=DATA', > + metavar='VAR=DATA', > + action='callback', > + callback=add_var), > + make_option('-e', > + dest='export', > + help='exports the contents of the usr.manifest > into a given folder', > + metavar='FILE'), > + ]) > + > + (options, args) = opt.parse_args() > + > + manifest = read_manifest(options.manifest) > + export_package(manifest, options.export) > + > +if __name__ == "__main__": > + main() > diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py > new file mode 100644 > index 0000000..414cce3 > --- /dev/null > +++ b/scripts/manifest_common.py > @@ -0,0 +1,78 @@ > +#!/usr/bin/python > + > +import os, io, re, subprocess > + > +defines = {} > + > +def add_var(option, opt, value, parser): > + var, val = value.split('=') > + defines[var] = val > + > +def expand(items): > + for name, hostname in items: > + if name.endswith('/**') and hostname.endswith('/**'): > + name = name[:-2] > + hostname = hostname[:-2] > + for dirpath, dirnames, filenames in os.walk(hostname): > + for filename in filenames: > + relpath = dirpath[len(hostname):] > + if relpath != "": > + relpath += "/" > + yield (name + relpath + filename, > + hostname + relpath + filename) > + elif '/&/' in name and hostname.endswith('/&'): > + prefix, suffix = name.split('/&/', 1) > + yield (prefix + '/' + suffix, hostname[:-1] + suffix) > + else: > + yield (name, hostname) > + > +def unsymlink(f): > + if f.startswith('!'): > + return f[1:] > + if f.startswith('->'): > + return f > + try: > + link = os.readlink(f) > + if link.startswith('/'): > + # try to find a match > + base = os.path.dirname(f) > + while not os.path.exists(base + link): > + if base == '/': > + return f > + base = os.path.dirname(base) > + else: > + base = os.path.dirname(f) + '/' > + return unsymlink(base + link) > + except Exception: > + return f > + > +# Reads the manifest and returns it as a list of pairs (guestpath, > hostpath). > +def read_manifest(fn): > + ret = [] > + comment = re.compile("^[ \t]*(|#.*|\[manifest])$") > + with open(fn, 'r') as f: > + for line in f: > + line = line.rstrip(); > + if comment.match(line): continue > + components = line.split(": ", 2) > + guestpath = components[0].strip(); > + hostpath = components[1].strip() > + ret.append((guestpath, hostpath)) > + return ret > + > +def strip_file(filename): > + def to_strip(filename): > + ff = os.path.abspath(filename) > + osvdir = os.path.abspath('../..') > + return ff.startswith(os.getcwd()) or \ > + ff.startswith(osvdir + "/modules") or \ > + ff.startswith(osvdir + "/apps") > + > + stripped_filename = filename > + if filename.endswith(".so") and to_strip(filename): > + stripped_filename = filename[:-3] + "-stripped.so" > + if not os.path.exists(stripped_filename) \ > + or (os.path.getmtime(stripped_filename) < \ > + os.path.getmtime(filename)): > + subprocess.call(["strip", "-o", stripped_filename, filename]) > + return stripped_filename > diff --git a/scripts/module.py b/scripts/module.py > index ec96e89..f7baf72 100755 > --- a/scripts/module.py > +++ b/scripts/module.py > @@ -170,6 +170,7 @@ def build(args): > print("Using image config: %s" % image_config_file) > config = resolve.local_import(image_config_file) > run_list = config.get('run', []) > + selected_modules = [] > else: > # If images/image_config doesn't exist, assume image_config is a > # comma-separated list of module names, and build an image from > those > @@ -206,7 +207,10 @@ def build(args): > # Add moduless thare are implictly required if others are present > resolve.resolve_required_modules_if_other_is_present() > > - modules = resolve.get_required_modules() > + if args.export == 'selected': > + modules = list(module for module in > resolve.get_required_modules() if module.name in selected_modules) > + else: > + modules = resolve.get_required_modules() > modules_to_run = resolve.get_modules_to_run() > > print("Modules:") > @@ -260,6 +264,8 @@ if __name__ == "__main__": > help="image configuration name. Looked up in " + > image_configs_dir) > build_cmd.add_argument("--usrskel", action="store", > default="default", > help="override default usr.manifest.skel") > + build_cmd.add_argument("--export", action="store", default="false", > + help="export files instead of producing > image") > build_cmd.set_defaults(func=build) > > clean_cmd = subparsers.add_parser("clean", help="Clean modules") > diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py > index de570f0..736cd34 100755 > --- a/scripts/upload_manifest.py > +++ b/scripts/upload_manifest.py > @@ -1,6 +1,7 @@ > #!/usr/bin/python > > -import os, optparse, io, subprocess, socket, threading, stat, sys, re > +import optparse, os, subprocess, socket, threading, stat, sys > +from manifest_common import add_var, expand, unsymlink, read_manifest, > defines, strip_file > > try: > import StringIO > @@ -10,64 +11,6 @@ except ImportError: > # This works on Python 3 > StringIO = io.StringIO > > -defines = {} > - > -def add_var(option, opt, value, parser): > - var, val = value.split('=') > - defines[var] = val > - > -def expand(items): > - for name, hostname in items: > - if name.endswith('/**') and hostname.endswith('/**'): > - name = name[:-2] > - hostname = hostname[:-2] > - for dirpath, dirnames, filenames in os.walk(hostname): > - for filename in filenames: > - relpath = dirpath[len(hostname):] > - if relpath != "": > - relpath += "/" > - yield (name + relpath + filename, > - hostname + relpath + filename) > - elif '/&/' in name and hostname.endswith('/&'): > - prefix, suffix = name.split('/&/', 1) > - yield (prefix + '/' + suffix, hostname[:-1] + suffix) > - else: > - yield (name, hostname) > - > -def unsymlink(f): > - if f.startswith('!'): > - return f[1:] > - if f.startswith('->'): > - return f > - try: > - link = os.readlink(f) > - if link.startswith('/'): > - # try to find a match > - base = os.path.dirname(f) > - while not os.path.exists(base + link): > - if base == '/': > - return f > - base = os.path.dirname(base) > - else: > - base = os.path.dirname(f) + '/' > - return unsymlink(base + link) > - except Exception: > - return f > - > -# Reads the manifest and returns it as a list of pairs (guestpath, > hostpath). > -def read_manifest(fn): > - ret = [] > - comment = re.compile("^[ \t]*(|#.*|\[manifest])$") > - with open(fn, 'r') as f: > - for line in f: > - line = line.rstrip(); > - if comment.match(line): continue > - components = line.split(": ", 2) > - guestpath = components[0].strip(); > - hostpath = components[1].strip() > - ret.append((guestpath, hostpath)) > - return ret > - > def upload(osv, manifest, depends): > manifest = [(x, y % defines) for (x, y) in manifest] > files = list(expand(manifest)) > @@ -118,24 +61,6 @@ def upload(osv, manifest, depends): > + cpio_field(0, 8) # check > + filename + b'\0') > > - def to_strip(filename): > - ff = os.path.abspath(filename); > - osvdir = os.path.abspath('../..'); > - return ff.startswith(os.getcwd()) or \ > - ff.startswith(osvdir + "/modules") or \ > - ff.startswith(osvdir + "/apps") > - > - def strip_file(filename): > - stripped_filename = filename > - if filename.endswith(".so") and to_strip(filename): > - stripped_filename = filename[:-3] + "-stripped.so" > - if not os.path.exists(stripped_filename) \ > - or (os.path.getmtime(stripped_filename) < \ > - os.path.getmtime(filename)): > - subprocess.call(["strip", "-o", stripped_filename, > filename]) > - return stripped_filename > - > - > # Send the files to the guest > for name, hostname in files: > if hostname.startswith("->"): > -- > 2.7.4 > > -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. For more options, visit https://groups.google.com/d/optout.
