I have addressed all suggestions by Nadav and added handling of new parameter export_dir requested by Miha.
Waldek On Saturday, August 19, 2017 at 12:51:38 PM UTC-4, Waldek Kozaczuk wrote: > > This patch enhances OSv build scripts to allow exporting files in addition > of uploading them to an image. It addresses 3 usage scenarios described in > #900 and demostrated by examples below: > > - ./scripts/build image=empty export=all - export files that are specified > is usr.manifest.skel in order to be able to produce capstan osv.bootstrap > package > > - ./scripts/build image=java-isolated export=all usrskel=none - export > files that are part of OSv modules (./modules directory) along with all > other dependent modules minus what is part of usr.manifest.skel > > - ./scripts/build image=openjdk8-zulu-compact1 export=selected > usrskel=none - export files that are part of an app (most applicable to > apps/openjdk8-****) without any dependent modules > > The export logic is enabled by passing new parameter export > [=none|all|selected] to scripts/build. > Exported files are placed under build/export directory or directory > indocated by export_dir parameter. > > Please note that the changes are backwards compatible. However changes to > scripts/upload_manifest.py > will break patches that are part of > https://github.com/mikelangelo-project/capstan-packages/tree/master/docker_files/common. > > Hopefully this patch will make some of these patches obsolete. > > Fixed #900 > > Signed-off-by: Waldemar Kozaczuk <[email protected]> > > --- > modules/empty/module.py | 0 > scripts/build | 15 +++++++- > scripts/export_manifest.py | 93 > ++++++++++++++++++++++++++++++++++++++++++++++ > scripts/manifest_common.py | 78 ++++++++++++++++++++++++++++++++++++++ > scripts/module.py | 15 ++++++-- > scripts/upload_manifest.py | 79 +-------------------------------------- > 6 files changed, 198 insertions(+), 82 deletions(-) > create mode 100644 modules/empty/module.py > create mode 100755 scripts/export_manifest.py > create mode 100644 scripts/manifest_common.py > > diff --git a/modules/empty/module.py b/modules/empty/module.py > new file mode 100644 > index 0000000..e69de29 > diff --git a/scripts/build b/scripts/build > index 7cee19c..3d42f78 100755 > --- a/scripts/build > +++ b/scripts/build > @@ -147,7 +147,12 @@ then > usrskel_arg="--usrskel ${vars[usrskel]}" > fi > > -jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC > OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules > $usrskel_arg > +export=${vars[export]-none} > +if [ "$export" == "selected" ] > +then > + no_required_arg="--no-required" > +fi > +jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC > OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules > $usrskel_arg $no_required_arg > > bootfs_manifest=$manifest make "${args[@]}" | tee -a build.out > # check exit status of make > @@ -180,7 +185,13 @@ zfs) > qemu-img convert -f raw -O qcow2 bare.raw usr.img > qemu-img resize usr.img ${fs_size}b >/dev/null 2>&1 > > - $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D > jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D > miscbase=$miscbase > + if [ "$export" == "none" ] > + then > + $SRC/scripts/upload_manifest.py -o usr.img -m > usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D > glibcbase=$glibcbase -D miscbase=$miscbase > + else > + export_dir=${vars[export_dir]-$SRC/build/export} > + $SRC/scripts/export_manifest.py -e $export_dir -m > usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D > glibcbase=$glibcbase -D miscbase=$miscbase > + fi > ;; > ramfs) > qemu-img convert -f raw -O qcow2 loader.img usr.img > diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py > new file mode 100755 > index 0000000..6bade92 > --- /dev/null > +++ b/scripts/export_manifest.py > @@ -0,0 +1,93 @@ > +#!/usr/bin/python > + > +import optparse, os, shutil > +from manifest_common import add_var, expand, unsymlink, read_manifest, > defines, strip_file > + > +# This will export the package based on the provided manifest file. It > uses the same mechanism to > +# get the files that need copying as the actual upload process. The only > current limitation is > +# support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. > +def export_package(manifest, dest): > + abs_dest = os.path.abspath(dest) > + print "[INFO] exporting into directory %s" % abs_dest > + > + # Remove and create the base directory where we are going to put all > package files. > + if os.path.exists(abs_dest): > + shutil.rmtree(abs_dest) > + os.makedirs(abs_dest) > + > + files = list(expand(manifest)) > + files = [(x, unsymlink(y % defines)) for (x, y) in files] > + > + for name, hostname in files: > + name = name[1:] if name.startswith("/") else name > + name = os.path.join(abs_dest, name) > + > + if hostname.startswith("->"): > + link_source = hostname[2:] > + target_dir = os.path.dirname(name) > + > + if link_source.startswith("/"): > + link_source = os.path.join(abs_dest, link_source[1:]) > + else: > + link_source = os.path.abspath(os.path.join(target_dir, > link_source)) > + > + link_source = os.path.relpath(link_source, target_dir) > + > + if not os.path.exists(target_dir): > + os.makedirs(target_dir) > + > + os.symlink(link_source, name) > + print "[INFO] added link %s -> %s" % (name, link_source) > + > + else: > + # If it is a file, copy it to the target directory. > + if os.path.isfile(hostname): > + # Make sure the target dir exists > + dirname = os.path.dirname(name) > + if not os.path.exists(dirname): > + os.makedirs(dirname) > + > + if hostname.endswith("-stripped.so"): > + continue > + hostname = strip_file(hostname) > + > + shutil.copy(hostname, name) > + print "[INFO] exported %s" % name > + elif os.path.isdir(hostname): > + # If hostname is a dir, it is only a request to create > the folder on guest. Nothing to copy. > + if not os.path.exists(name): > + os.makedirs(name) > + print "[INFO] created dir %s" % name > + > + else: > + # Inform the user that the rule cannot be applied. For > example, this happens for links in OSv. > + print "[ERR] unable to export %s" % hostname > + > + > +def main(): > + make_option = optparse.make_option > + > + opt = optparse.OptionParser(option_list=[ > + make_option('-m', > + dest='manifest', > + help='read manifest from FILE', > + metavar='FILE'), > + make_option('-D', > + type='string', > + help='define VAR=DATA', > + metavar='VAR=DATA', > + action='callback', > + callback=add_var), > + make_option('-e', > + dest='export', > + help='exports the contents of the usr.manifest > into a given folder', > + metavar='FILE'), > + ]) > + > + (options, args) = opt.parse_args() > + > + manifest = read_manifest(options.manifest) > + export_package(manifest, options.export) > + > +if __name__ == "__main__": > + main() > diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py > new file mode 100644 > index 0000000..414cce3 > --- /dev/null > +++ b/scripts/manifest_common.py > @@ -0,0 +1,78 @@ > +#!/usr/bin/python > + > +import os, io, re, subprocess > + > +defines = {} > + > +def add_var(option, opt, value, parser): > + var, val = value.split('=') > + defines[var] = val > + > +def expand(items): > + for name, hostname in items: > + if name.endswith('/**') and hostname.endswith('/**'): > + name = name[:-2] > + hostname = hostname[:-2] > + for dirpath, dirnames, filenames in os.walk(hostname): > + for filename in filenames: > + relpath = dirpath[len(hostname):] > + if relpath != "": > + relpath += "/" > + yield (name + relpath + filename, > + hostname + relpath + filename) > + elif '/&/' in name and hostname.endswith('/&'): > + prefix, suffix = name.split('/&/', 1) > + yield (prefix + '/' + suffix, hostname[:-1] + suffix) > + else: > + yield (name, hostname) > + > +def unsymlink(f): > + if f.startswith('!'): > + return f[1:] > + if f.startswith('->'): > + return f > + try: > + link = os.readlink(f) > + if link.startswith('/'): > + # try to find a match > + base = os.path.dirname(f) > + while not os.path.exists(base + link): > + if base == '/': > + return f > + base = os.path.dirname(base) > + else: > + base = os.path.dirname(f) + '/' > + return unsymlink(base + link) > + except Exception: > + return f > + > +# Reads the manifest and returns it as a list of pairs (guestpath, > hostpath). > +def read_manifest(fn): > + ret = [] > + comment = re.compile("^[ \t]*(|#.*|\[manifest])$") > + with open(fn, 'r') as f: > + for line in f: > + line = line.rstrip(); > + if comment.match(line): continue > + components = line.split(": ", 2) > + guestpath = components[0].strip(); > + hostpath = components[1].strip() > + ret.append((guestpath, hostpath)) > + return ret > + > +def strip_file(filename): > + def to_strip(filename): > + ff = os.path.abspath(filename) > + osvdir = os.path.abspath('../..') > + return ff.startswith(os.getcwd()) or \ > + ff.startswith(osvdir + "/modules") or \ > + ff.startswith(osvdir + "/apps") > + > + stripped_filename = filename > + if filename.endswith(".so") and to_strip(filename): > + stripped_filename = filename[:-3] + "-stripped.so" > + if not os.path.exists(stripped_filename) \ > + or (os.path.getmtime(stripped_filename) < \ > + os.path.getmtime(filename)): > + subprocess.call(["strip", "-o", stripped_filename, filename]) > + return stripped_filename > diff --git a/scripts/module.py b/scripts/module.py > index ec96e89..548dd0c 100755 > --- a/scripts/module.py > +++ b/scripts/module.py > @@ -170,6 +170,7 @@ def build(args): > print("Using image config: %s" % image_config_file) > config = resolve.local_import(image_config_file) > run_list = config.get('run', []) > + selected_modules = [] > else: > # If images/image_config doesn't exist, assume image_config is a > # comma-separated list of module names, and build an image from > those > @@ -203,10 +204,16 @@ def build(args): > else: > api.require_running(name) > > - # Add moduless thare are implictly required if others are present > + # Add modules that are implicitly required if others are present > resolve.resolve_required_modules_if_other_is_present() > > - modules = resolve.get_required_modules() > + # By default append manifests from all modules resolved through > api.require() > + # otherwise (add_required_to_manifest=False) only append manifests > from the selected_modules > + if args.add_required_to_manifest: > + modules = resolve.get_required_modules() > + else: > + modules = list(module for module in > resolve.get_required_modules() if module.name in selected_modules) > + > modules_to_run = resolve.get_modules_to_run() > > print("Modules:") > @@ -260,7 +267,9 @@ if __name__ == "__main__": > help="image configuration name. Looked up in " + > image_configs_dir) > build_cmd.add_argument("--usrskel", action="store", > default="default", > help="override default usr.manifest.skel") > - build_cmd.set_defaults(func=build) > + build_cmd.add_argument("--no-required", > dest="add_required_to_manifest", action="store_false", > + help="do not add files to usr.manifest from > modules implicitly resolved through api.require()") > + build_cmd.set_defaults(func=build,add_required_to_manifest=True) > > clean_cmd = subparsers.add_parser("clean", help="Clean modules") > clean_cmd.add_argument("-q", "--quiet", action="store_true") > diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py > index de570f0..736cd34 100755 > --- a/scripts/upload_manifest.py > +++ b/scripts/upload_manifest.py > @@ -1,6 +1,7 @@ > #!/usr/bin/python > > -import os, optparse, io, subprocess, socket, threading, stat, sys, re > +import optparse, os, subprocess, socket, threading, stat, sys > +from manifest_common import add_var, expand, unsymlink, read_manifest, > defines, strip_file > > try: > import StringIO > @@ -10,64 +11,6 @@ except ImportError: > # This works on Python 3 > StringIO = io.StringIO > > -defines = {} > - > -def add_var(option, opt, value, parser): > - var, val = value.split('=') > - defines[var] = val > - > -def expand(items): > - for name, hostname in items: > - if name.endswith('/**') and hostname.endswith('/**'): > - name = name[:-2] > - hostname = hostname[:-2] > - for dirpath, dirnames, filenames in os.walk(hostname): > - for filename in filenames: > - relpath = dirpath[len(hostname):] > - if relpath != "": > - relpath += "/" > - yield (name + relpath + filename, > - hostname + relpath + filename) > - elif '/&/' in name and hostname.endswith('/&'): > - prefix, suffix = name.split('/&/', 1) > - yield (prefix + '/' + suffix, hostname[:-1] + suffix) > - else: > - yield (name, hostname) > - > -def unsymlink(f): > - if f.startswith('!'): > - return f[1:] > - if f.startswith('->'): > - return f > - try: > - link = os.readlink(f) > - if link.startswith('/'): > - # try to find a match > - base = os.path.dirname(f) > - while not os.path.exists(base + link): > - if base == '/': > - return f > - base = os.path.dirname(base) > - else: > - base = os.path.dirname(f) + '/' > - return unsymlink(base + link) > - except Exception: > - return f > - > -# Reads the manifest and returns it as a list of pairs (guestpath, > hostpath). > -def read_manifest(fn): > - ret = [] > - comment = re.compile("^[ \t]*(|#.*|\[manifest])$") > - with open(fn, 'r') as f: > - for line in f: > - line = line.rstrip(); > - if comment.match(line): continue > - components = line.split(": ", 2) > - guestpath = components[0].strip(); > - hostpath = components[1].strip() > - ret.append((guestpath, hostpath)) > - return ret > - > def upload(osv, manifest, depends): > manifest = [(x, y % defines) for (x, y) in manifest] > files = list(expand(manifest)) > @@ -118,24 +61,6 @@ def upload(osv, manifest, depends): > + cpio_field(0, 8) # check > + filename + b'\0') > > - def to_strip(filename): > - ff = os.path.abspath(filename); > - osvdir = os.path.abspath('../..'); > - return ff.startswith(os.getcwd()) or \ > - ff.startswith(osvdir + "/modules") or \ > - ff.startswith(osvdir + "/apps") > - > - def strip_file(filename): > - stripped_filename = filename > - if filename.endswith(".so") and to_strip(filename): > - stripped_filename = filename[:-3] + "-stripped.so" > - if not os.path.exists(stripped_filename) \ > - or (os.path.getmtime(stripped_filename) < \ > - os.path.getmtime(filename)): > - subprocess.call(["strip", "-o", stripped_filename, > filename]) > - return stripped_filename > - > - > # Send the files to the guest > for name, hostname in files: > if hostname.startswith("->"): > -- > 2.7.4 > > -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
