This patch enhances OSv build scripts to allow exporting files in addition of uploading them to an image. It addresses 3 usage scenarios described in #900 and demostrated by examples below:
- ./scripts/build image=empty export=all - export files that are specified is usr.manifest.skel in order to be able to produce capstan osv.bootstrap package - ./scripts/build image=java-isolated export=all usrskel=none - export files that are part of OSv modules (./modules directory) along with all other dependent modules minus what is part of usr.manifest.skel - ./scripts/build image=openjdk8-zulu-compact1 export=selected usrskel=none - export files that are part of an app (most applicable to apps/openjdk8-****) without any dependent modules The export logic is enabled by passing new parameter export [=none|all|selected] to scripts/build. Exported files are placed under build/export directory or directory indocated by export_dir parameter. Please note that the changes are backwards compatible. However changes to scripts/upload_manifest.py will break patches that are part of https://github.com/mikelangelo-project/capstan-packages/tree/master/docker_files/common. Hopefully this patch will make some of these patches obsolete. Fixed #900 Signed-off-by: Waldemar Kozaczuk <[email protected]> --- modules/empty/module.py | 0 scripts/build | 15 +++++++- scripts/export_manifest.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/manifest_common.py | 78 ++++++++++++++++++++++++++++++++++++++ scripts/module.py | 15 ++++++-- scripts/upload_manifest.py | 79 +-------------------------------------- 6 files changed, 198 insertions(+), 82 deletions(-) create mode 100644 modules/empty/module.py create mode 100755 scripts/export_manifest.py create mode 100644 scripts/manifest_common.py diff --git a/modules/empty/module.py b/modules/empty/module.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/build b/scripts/build index 7cee19c..3d42f78 100755 --- a/scripts/build +++ b/scripts/build @@ -147,7 +147,12 @@ then usrskel_arg="--usrskel ${vars[usrskel]}" fi -jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules $usrskel_arg +export=${vars[export]-none} +if [ "$export" == "selected" ] +then + no_required_arg="--no-required" +fi +jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules $usrskel_arg $no_required_arg bootfs_manifest=$manifest make "${args[@]}" | tee -a build.out # check exit status of make @@ -180,7 +185,13 @@ zfs) qemu-img convert -f raw -O qcow2 bare.raw usr.img qemu-img resize usr.img ${fs_size}b >/dev/null 2>&1 - $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D miscbase=$miscbase + if [ "$export" == "none" ] + then + $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D miscbase=$miscbase + else + export_dir=${vars[export_dir]-$SRC/build/export} + $SRC/scripts/export_manifest.py -e $export_dir -m usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D miscbase=$miscbase + fi ;; ramfs) qemu-img convert -f raw -O qcow2 loader.img usr.img diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py new file mode 100755 index 0000000..6bade92 --- /dev/null +++ b/scripts/export_manifest.py @@ -0,0 +1,93 @@ +#!/usr/bin/python + +import optparse, os, shutil +from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file + +# This will export the package based on the provided manifest file. It uses the same mechanism to +# get the files that need copying as the actual upload process. The only current limitation is +# support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. +def export_package(manifest, dest): + abs_dest = os.path.abspath(dest) + print "[INFO] exporting into directory %s" % abs_dest + + # Remove and create the base directory where we are going to put all package files. + if os.path.exists(abs_dest): + shutil.rmtree(abs_dest) + os.makedirs(abs_dest) + + files = list(expand(manifest)) + files = [(x, unsymlink(y % defines)) for (x, y) in files] + + for name, hostname in files: + name = name[1:] if name.startswith("/") else name + name = os.path.join(abs_dest, name) + + if hostname.startswith("->"): + link_source = hostname[2:] + target_dir = os.path.dirname(name) + + if link_source.startswith("/"): + link_source = os.path.join(abs_dest, link_source[1:]) + else: + link_source = os.path.abspath(os.path.join(target_dir, link_source)) + + link_source = os.path.relpath(link_source, target_dir) + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + os.symlink(link_source, name) + print "[INFO] added link %s -> %s" % (name, link_source) + + else: + # If it is a file, copy it to the target directory. + if os.path.isfile(hostname): + # Make sure the target dir exists + dirname = os.path.dirname(name) + if not os.path.exists(dirname): + os.makedirs(dirname) + + if hostname.endswith("-stripped.so"): + continue + hostname = strip_file(hostname) + + shutil.copy(hostname, name) + print "[INFO] exported %s" % name + elif os.path.isdir(hostname): + # If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy. + if not os.path.exists(name): + os.makedirs(name) + print "[INFO] created dir %s" % name + + else: + # Inform the user that the rule cannot be applied. For example, this happens for links in OSv. + print "[ERR] unable to export %s" % hostname + + +def main(): + make_option = optparse.make_option + + opt = optparse.OptionParser(option_list=[ + make_option('-m', + dest='manifest', + help='read manifest from FILE', + metavar='FILE'), + make_option('-D', + type='string', + help='define VAR=DATA', + metavar='VAR=DATA', + action='callback', + callback=add_var), + make_option('-e', + dest='export', + help='exports the contents of the usr.manifest into a given folder', + metavar='FILE'), + ]) + + (options, args) = opt.parse_args() + + manifest = read_manifest(options.manifest) + export_package(manifest, options.export) + +if __name__ == "__main__": + main() diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py new file mode 100644 index 0000000..414cce3 --- /dev/null +++ b/scripts/manifest_common.py @@ -0,0 +1,78 @@ +#!/usr/bin/python + +import os, io, re, subprocess + +defines = {} + +def add_var(option, opt, value, parser): + var, val = value.split('=') + defines[var] = val + +def expand(items): + for name, hostname in items: + if name.endswith('/**') and hostname.endswith('/**'): + name = name[:-2] + hostname = hostname[:-2] + for dirpath, dirnames, filenames in os.walk(hostname): + for filename in filenames: + relpath = dirpath[len(hostname):] + if relpath != "": + relpath += "/" + yield (name + relpath + filename, + hostname + relpath + filename) + elif '/&/' in name and hostname.endswith('/&'): + prefix, suffix = name.split('/&/', 1) + yield (prefix + '/' + suffix, hostname[:-1] + suffix) + else: + yield (name, hostname) + +def unsymlink(f): + if f.startswith('!'): + return f[1:] + if f.startswith('->'): + return f + try: + link = os.readlink(f) + if link.startswith('/'): + # try to find a match + base = os.path.dirname(f) + while not os.path.exists(base + link): + if base == '/': + return f + base = os.path.dirname(base) + else: + base = os.path.dirname(f) + '/' + return unsymlink(base + link) + except Exception: + return f + +# Reads the manifest and returns it as a list of pairs (guestpath, hostpath). +def read_manifest(fn): + ret = [] + comment = re.compile("^[ \t]*(|#.*|\[manifest])$") + with open(fn, 'r') as f: + for line in f: + line = line.rstrip(); + if comment.match(line): continue + components = line.split(": ", 2) + guestpath = components[0].strip(); + hostpath = components[1].strip() + ret.append((guestpath, hostpath)) + return ret + +def strip_file(filename): + def to_strip(filename): + ff = os.path.abspath(filename) + osvdir = os.path.abspath('../..') + return ff.startswith(os.getcwd()) or \ + ff.startswith(osvdir + "/modules") or \ + ff.startswith(osvdir + "/apps") + + stripped_filename = filename + if filename.endswith(".so") and to_strip(filename): + stripped_filename = filename[:-3] + "-stripped.so" + if not os.path.exists(stripped_filename) \ + or (os.path.getmtime(stripped_filename) < \ + os.path.getmtime(filename)): + subprocess.call(["strip", "-o", stripped_filename, filename]) + return stripped_filename diff --git a/scripts/module.py b/scripts/module.py index ec96e89..548dd0c 100755 --- a/scripts/module.py +++ b/scripts/module.py @@ -170,6 +170,7 @@ def build(args): print("Using image config: %s" % image_config_file) config = resolve.local_import(image_config_file) run_list = config.get('run', []) + selected_modules = [] else: # If images/image_config doesn't exist, assume image_config is a # comma-separated list of module names, and build an image from those @@ -203,10 +204,16 @@ def build(args): else: api.require_running(name) - # Add moduless thare are implictly required if others are present + # Add modules that are implicitly required if others are present resolve.resolve_required_modules_if_other_is_present() - modules = resolve.get_required_modules() + # By default append manifests from all modules resolved through api.require() + # otherwise (add_required_to_manifest=False) only append manifests from the selected_modules + if args.add_required_to_manifest: + modules = resolve.get_required_modules() + else: + modules = list(module for module in resolve.get_required_modules() if module.name in selected_modules) + modules_to_run = resolve.get_modules_to_run() print("Modules:") @@ -260,7 +267,9 @@ if __name__ == "__main__": help="image configuration name. Looked up in " + image_configs_dir) build_cmd.add_argument("--usrskel", action="store", default="default", help="override default usr.manifest.skel") - build_cmd.set_defaults(func=build) + build_cmd.add_argument("--no-required", dest="add_required_to_manifest", action="store_false", + help="do not add files to usr.manifest from modules implicitly resolved through api.require()") + build_cmd.set_defaults(func=build,add_required_to_manifest=True) clean_cmd = subparsers.add_parser("clean", help="Clean modules") clean_cmd.add_argument("-q", "--quiet", action="store_true") diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py index de570f0..736cd34 100755 --- a/scripts/upload_manifest.py +++ b/scripts/upload_manifest.py @@ -1,6 +1,7 @@ #!/usr/bin/python -import os, optparse, io, subprocess, socket, threading, stat, sys, re +import optparse, os, subprocess, socket, threading, stat, sys +from manifest_common import add_var, expand, unsymlink, read_manifest, defines, strip_file try: import StringIO @@ -10,64 +11,6 @@ except ImportError: # This works on Python 3 StringIO = io.StringIO -defines = {} - -def add_var(option, opt, value, parser): - var, val = value.split('=') - defines[var] = val - -def expand(items): - for name, hostname in items: - if name.endswith('/**') and hostname.endswith('/**'): - name = name[:-2] - hostname = hostname[:-2] - for dirpath, dirnames, filenames in os.walk(hostname): - for filename in filenames: - relpath = dirpath[len(hostname):] - if relpath != "": - relpath += "/" - yield (name + relpath + filename, - hostname + relpath + filename) - elif '/&/' in name and hostname.endswith('/&'): - prefix, suffix = name.split('/&/', 1) - yield (prefix + '/' + suffix, hostname[:-1] + suffix) - else: - yield (name, hostname) - -def unsymlink(f): - if f.startswith('!'): - return f[1:] - if f.startswith('->'): - return f - try: - link = os.readlink(f) - if link.startswith('/'): - # try to find a match - base = os.path.dirname(f) - while not os.path.exists(base + link): - if base == '/': - return f - base = os.path.dirname(base) - else: - base = os.path.dirname(f) + '/' - return unsymlink(base + link) - except Exception: - return f - -# Reads the manifest and returns it as a list of pairs (guestpath, hostpath). -def read_manifest(fn): - ret = [] - comment = re.compile("^[ \t]*(|#.*|\[manifest])$") - with open(fn, 'r') as f: - for line in f: - line = line.rstrip(); - if comment.match(line): continue - components = line.split(": ", 2) - guestpath = components[0].strip(); - hostpath = components[1].strip() - ret.append((guestpath, hostpath)) - return ret - def upload(osv, manifest, depends): manifest = [(x, y % defines) for (x, y) in manifest] files = list(expand(manifest)) @@ -118,24 +61,6 @@ def upload(osv, manifest, depends): + cpio_field(0, 8) # check + filename + b'\0') - def to_strip(filename): - ff = os.path.abspath(filename); - osvdir = os.path.abspath('../..'); - return ff.startswith(os.getcwd()) or \ - ff.startswith(osvdir + "/modules") or \ - ff.startswith(osvdir + "/apps") - - def strip_file(filename): - stripped_filename = filename - if filename.endswith(".so") and to_strip(filename): - stripped_filename = filename[:-3] + "-stripped.so" - if not os.path.exists(stripped_filename) \ - or (os.path.getmtime(stripped_filename) < \ - os.path.getmtime(filename)): - subprocess.call(["strip", "-o", stripped_filename, filename]) - return stripped_filename - - # Send the files to the guest for name, hostname in files: if hostname.startswith("->"): -- 2.7.4 -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
