This comment is from the original MikelAngelo version of export_manifest.py that I re-purposed. I am also not sure what the meaning of this is. Maybe symlinks of symlinks?
Maybe Miha or Gregor can shed light. On Sunday, August 20, 2017 at 8:37:17 AM UTC-4, Nadav Har'El wrote: > > Thanks. I committed this patch. > One thing I didn't understand is the comment: "The only current limitation > is support for links in OSv". I see you did handle symlinks... Is this an > old comment? > > > -- > Nadav Har'El > [email protected] <javascript:> > > On Sat, Aug 19, 2017 at 7:51 PM, Waldemar Kozaczuk <[email protected] > <javascript:>> wrote: > >> This patch enhances OSv build scripts to allow exporting files in addition >> of uploading them to an image. It addresses 3 usage scenarios described in >> #900 and demostrated by examples below: >> >> - ./scripts/build image=empty export=all - export files that are >> specified is usr.manifest.skel in order to be able to produce capstan >> osv.bootstrap package >> >> - ./scripts/build image=java-isolated export=all usrskel=none - export >> files that are part of OSv modules (./modules directory) along with all >> other dependent modules minus what is part of usr.manifest.skel >> >> - ./scripts/build image=openjdk8-zulu-compact1 export=selected >> usrskel=none - export files that are part of an app (most applicable to >> apps/openjdk8-****) without any dependent modules >> >> The export logic is enabled by passing new parameter export >> [=none|all|selected] to scripts/build. >> Exported files are placed under build/export directory or directory >> indocated by export_dir parameter. >> >> Please note that the changes are backwards compatible. However changes to >> scripts/upload_manifest.py >> will break patches that are part of >> https://github.com/mikelangelo-project/capstan-packages/tree/master/docker_files/common. >> >> Hopefully this patch will make some of these patches obsolete. >> >> Fixed #900 >> >> Signed-off-by: Waldemar Kozaczuk <[email protected] <javascript:>> >> >> --- >> modules/empty/module.py | 0 >> scripts/build | 15 +++++++- >> scripts/export_manifest.py | 93 >> ++++++++++++++++++++++++++++++++++++++++++++++ >> scripts/manifest_common.py | 78 ++++++++++++++++++++++++++++++++++++++ >> scripts/module.py | 15 ++++++-- >> scripts/upload_manifest.py | 79 +-------------------------------------- >> 6 files changed, 198 insertions(+), 82 deletions(-) >> create mode 100644 modules/empty/module.py >> create mode 100755 scripts/export_manifest.py >> create mode 100644 scripts/manifest_common.py >> >> diff --git a/modules/empty/module.py b/modules/empty/module.py >> new file mode 100644 >> index 0000000..e69de29 >> diff --git a/scripts/build b/scripts/build >> index 7cee19c..3d42f78 100755 >> --- a/scripts/build >> +++ b/scripts/build >> @@ -147,7 +147,12 @@ then >> usrskel_arg="--usrskel ${vars[usrskel]}" >> fi >> >> -jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC >> OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules >> $usrskel_arg >> +export=${vars[export]-none} >> +if [ "$export" == "selected" ] >> +then >> + no_required_arg="--no-required" >> +fi >> +jdkbase=$jdkbase ARCH=$arch mode=$mode OSV_BASE=$SRC >> OSV_BUILD_PATH=$OSV_BUILD_PATH scripts/module.py $j_arg build -c $modules >> $usrskel_arg $no_required_arg >> >> bootfs_manifest=$manifest make "${args[@]}" | tee -a build.out >> # check exit status of make >> @@ -180,7 +185,13 @@ zfs) >> qemu-img convert -f raw -O qcow2 bare.raw usr.img >> qemu-img resize usr.img ${fs_size}b >/dev/null 2>&1 >> >> - $SRC/scripts/upload_manifest.py -o usr.img -m usr.manifest -D >> jdkbase=$jdkbase -D gccbase=$gccbase -D glibcbase=$glibcbase -D >> miscbase=$miscbase >> + if [ "$export" == "none" ] >> + then >> + $SRC/scripts/upload_manifest.py -o usr.img -m >> usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D >> glibcbase=$glibcbase -D miscbase=$miscbase >> + else >> + export_dir=${vars[export_dir]-$SRC/build/export} >> + $SRC/scripts/export_manifest.py -e $export_dir -m >> usr.manifest -D jdkbase=$jdkbase -D gccbase=$gccbase -D >> glibcbase=$glibcbase -D miscbase=$miscbase >> + fi >> ;; >> ramfs) >> qemu-img convert -f raw -O qcow2 loader.img usr.img >> diff --git a/scripts/export_manifest.py b/scripts/export_manifest.py >> new file mode 100755 >> index 0000000..6bade92 >> --- /dev/null >> +++ b/scripts/export_manifest.py >> @@ -0,0 +1,93 @@ >> +#!/usr/bin/python >> + >> +import optparse, os, shutil >> +from manifest_common import add_var, expand, unsymlink, read_manifest, >> defines, strip_file >> + >> +# This will export the package based on the provided manifest file. It >> uses the same mechanism to >> +# get the files that need copying as the actual upload process. The only >> current limitation is >> +# support for links in OSv, e.g., /etc/mnttab: ->/proc/mounts. >> +def export_package(manifest, dest): >> + abs_dest = os.path.abspath(dest) >> + print "[INFO] exporting into directory %s" % abs_dest >> + >> + # Remove and create the base directory where we are going to put all >> package files. >> + if os.path.exists(abs_dest): >> + shutil.rmtree(abs_dest) >> + os.makedirs(abs_dest) >> + >> + files = list(expand(manifest)) >> + files = [(x, unsymlink(y % defines)) for (x, y) in files] >> + >> + for name, hostname in files: >> + name = name[1:] if name.startswith("/") else name >> + name = os.path.join(abs_dest, name) >> + >> + if hostname.startswith("->"): >> + link_source = hostname[2:] >> + target_dir = os.path.dirname(name) >> + >> + if link_source.startswith("/"): >> + link_source = os.path.join(abs_dest, link_source[1:]) >> + else: >> + link_source = os.path.abspath(os.path.join(target_dir, >> link_source)) >> + >> + link_source = os.path.relpath(link_source, target_dir) >> + >> + if not os.path.exists(target_dir): >> + os.makedirs(target_dir) >> + >> + os.symlink(link_source, name) >> + print "[INFO] added link %s -> %s" % (name, link_source) >> + >> + else: >> + # If it is a file, copy it to the target directory. >> + if os.path.isfile(hostname): >> + # Make sure the target dir exists >> + dirname = os.path.dirname(name) >> + if not os.path.exists(dirname): >> + os.makedirs(dirname) >> + >> + if hostname.endswith("-stripped.so"): >> + continue >> + hostname = strip_file(hostname) >> + >> + shutil.copy(hostname, name) >> + print "[INFO] exported %s" % name >> + elif os.path.isdir(hostname): >> + # If hostname is a dir, it is only a request to create >> the folder on guest. Nothing to copy. >> + if not os.path.exists(name): >> + os.makedirs(name) >> + print "[INFO] created dir %s" % name >> + >> + else: >> + # Inform the user that the rule cannot be applied. For >> example, this happens for links in OSv. >> + print "[ERR] unable to export %s" % hostname >> + >> + >> +def main(): >> + make_option = optparse.make_option >> + >> + opt = optparse.OptionParser(option_list=[ >> + make_option('-m', >> + dest='manifest', >> + help='read manifest from FILE', >> + metavar='FILE'), >> + make_option('-D', >> + type='string', >> + help='define VAR=DATA', >> + metavar='VAR=DATA', >> + action='callback', >> + callback=add_var), >> + make_option('-e', >> + dest='export', >> + help='exports the contents of the usr.manifest >> into a given folder', >> + metavar='FILE'), >> + ]) >> + >> + (options, args) = opt.parse_args() >> + >> + manifest = read_manifest(options.manifest) >> + export_package(manifest, options.export) >> + >> +if __name__ == "__main__": >> + main() >> diff --git a/scripts/manifest_common.py b/scripts/manifest_common.py >> new file mode 100644 >> index 0000000..414cce3 >> --- /dev/null >> +++ b/scripts/manifest_common.py >> @@ -0,0 +1,78 @@ >> +#!/usr/bin/python >> + >> +import os, io, re, subprocess >> + >> +defines = {} >> + >> +def add_var(option, opt, value, parser): >> + var, val = value.split('=') >> + defines[var] = val >> + >> +def expand(items): >> + for name, hostname in items: >> + if name.endswith('/**') and hostname.endswith('/**'): >> + name = name[:-2] >> + hostname = hostname[:-2] >> + for dirpath, dirnames, filenames in os.walk(hostname): >> + for filename in filenames: >> + relpath = dirpath[len(hostname):] >> + if relpath != "": >> + relpath += "/" >> + yield (name + relpath + filename, >> + hostname + relpath + filename) >> + elif '/&/' in name and hostname.endswith('/&'): >> + prefix, suffix = name.split('/&/', 1) >> + yield (prefix + '/' + suffix, hostname[:-1] + suffix) >> + else: >> + yield (name, hostname) >> + >> +def unsymlink(f): >> + if f.startswith('!'): >> + return f[1:] >> + if f.startswith('->'): >> + return f >> + try: >> + link = os.readlink(f) >> + if link.startswith('/'): >> + # try to find a match >> + base = os.path.dirname(f) >> + while not os.path.exists(base + link): >> + if base == '/': >> + return f >> + base = os.path.dirname(base) >> + else: >> + base = os.path.dirname(f) + '/' >> + return unsymlink(base + link) >> + except Exception: >> + return f >> + >> +# Reads the manifest and returns it as a list of pairs (guestpath, >> hostpath). >> +def read_manifest(fn): >> + ret = [] >> + comment = re.compile("^[ \t]*(|#.*|\[manifest])$") >> + with open(fn, 'r') as f: >> + for line in f: >> + line = line.rstrip(); >> + if comment.match(line): continue >> + components = line.split(": ", 2) >> + guestpath = components[0].strip(); >> + hostpath = components[1].strip() >> + ret.append((guestpath, hostpath)) >> + return ret >> + >> +def strip_file(filename): >> + def to_strip(filename): >> + ff = os.path.abspath(filename) >> + osvdir = os.path.abspath('../..') >> + return ff.startswith(os.getcwd()) or \ >> + ff.startswith(osvdir + "/modules") or \ >> + ff.startswith(osvdir + "/apps") >> + >> + stripped_filename = filename >> + if filename.endswith(".so") and to_strip(filename): >> + stripped_filename = filename[:-3] + "-stripped.so" >> + if not os.path.exists(stripped_filename) \ >> + or (os.path.getmtime(stripped_filename) < \ >> + os.path.getmtime(filename)): >> + subprocess.call(["strip", "-o", stripped_filename, filename]) >> + return stripped_filename >> diff --git a/scripts/module.py b/scripts/module.py >> index ec96e89..548dd0c 100755 >> --- a/scripts/module.py >> +++ b/scripts/module.py >> @@ -170,6 +170,7 @@ def build(args): >> print("Using image config: %s" % image_config_file) >> config = resolve.local_import(image_config_file) >> run_list = config.get('run', []) >> + selected_modules = [] >> else: >> # If images/image_config doesn't exist, assume image_config is a >> # comma-separated list of module names, and build an image from >> those >> @@ -203,10 +204,16 @@ def build(args): >> else: >> api.require_running(name) >> >> - # Add moduless thare are implictly required if others are present >> + # Add modules that are implicitly required if others are present >> resolve.resolve_required_modules_if_other_is_present() >> >> - modules = resolve.get_required_modules() >> + # By default append manifests from all modules resolved through >> api.require() >> + # otherwise (add_required_to_manifest=False) only append manifests >> from the selected_modules >> + if args.add_required_to_manifest: >> + modules = resolve.get_required_modules() >> + else: >> + modules = list(module for module in >> resolve.get_required_modules() if module.name in selected_modules) >> + >> modules_to_run = resolve.get_modules_to_run() >> >> print("Modules:") >> @@ -260,7 +267,9 @@ if __name__ == "__main__": >> help="image configuration name. Looked up in " + >> image_configs_dir) >> build_cmd.add_argument("--usrskel", action="store", >> default="default", >> help="override default usr.manifest.skel") >> - build_cmd.set_defaults(func=build) >> + build_cmd.add_argument("--no-required", >> dest="add_required_to_manifest", action="store_false", >> + help="do not add files to usr.manifest from >> modules implicitly resolved through api.require()") >> + build_cmd.set_defaults(func=build,add_required_to_manifest=True) >> >> clean_cmd = subparsers.add_parser("clean", help="Clean modules") >> clean_cmd.add_argument("-q", "--quiet", action="store_true") >> diff --git a/scripts/upload_manifest.py b/scripts/upload_manifest.py >> index de570f0..736cd34 100755 >> --- a/scripts/upload_manifest.py >> +++ b/scripts/upload_manifest.py >> @@ -1,6 +1,7 @@ >> #!/usr/bin/python >> >> -import os, optparse, io, subprocess, socket, threading, stat, sys, re >> +import optparse, os, subprocess, socket, threading, stat, sys >> +from manifest_common import add_var, expand, unsymlink, read_manifest, >> defines, strip_file >> >> try: >> import StringIO >> @@ -10,64 +11,6 @@ except ImportError: >> # This works on Python 3 >> StringIO = io.StringIO >> >> -defines = {} >> - >> -def add_var(option, opt, value, parser): >> - var, val = value.split('=') >> - defines[var] = val >> - >> -def expand(items): >> - for name, hostname in items: >> - if name.endswith('/**') and hostname.endswith('/**'): >> - name = name[:-2] >> - hostname = hostname[:-2] >> - for dirpath, dirnames, filenames in os.walk(hostname): >> - for filename in filenames: >> - relpath = dirpath[len(hostname):] >> - if relpath != "": >> - relpath += "/" >> - yield (name + relpath + filename, >> - hostname + relpath + filename) >> - elif '/&/' in name and hostname.endswith('/&'): >> - prefix, suffix = name.split('/&/', 1) >> - yield (prefix + '/' + suffix, hostname[:-1] + suffix) >> - else: >> - yield (name, hostname) >> - >> -def unsymlink(f): >> - if f.startswith('!'): >> - return f[1:] >> - if f.startswith('->'): >> - return f >> - try: >> - link = os.readlink(f) >> - if link.startswith('/'): >> - # try to find a match >> - base = os.path.dirname(f) >> - while not os.path.exists(base + link): >> - if base == '/': >> - return f >> - base = os.path.dirname(base) >> - else: >> - base = os.path.dirname(f) + '/' >> - return unsymlink(base + link) >> - except Exception: >> - return f >> - >> -# Reads the manifest and returns it as a list of pairs (guestpath, >> hostpath). >> -def read_manifest(fn): >> - ret = [] >> - comment = re.compile("^[ \t]*(|#.*|\[manifest])$") >> - with open(fn, 'r') as f: >> - for line in f: >> - line = line.rstrip(); >> - if comment.match(line): continue >> - components = line.split(": ", 2) >> - guestpath = components[0].strip(); >> - hostpath = components[1].strip() >> - ret.append((guestpath, hostpath)) >> - return ret >> - >> def upload(osv, manifest, depends): >> manifest = [(x, y % defines) for (x, y) in manifest] >> files = list(expand(manifest)) >> @@ -118,24 +61,6 @@ def upload(osv, manifest, depends): >> + cpio_field(0, 8) # check >> + filename + b'\0') >> >> - def to_strip(filename): >> - ff = os.path.abspath(filename); >> - osvdir = os.path.abspath('../..'); >> - return ff.startswith(os.getcwd()) or \ >> - ff.startswith(osvdir + "/modules") or \ >> - ff.startswith(osvdir + "/apps") >> - >> - def strip_file(filename): >> - stripped_filename = filename >> - if filename.endswith(".so") and to_strip(filename): >> - stripped_filename = filename[:-3] + "-stripped.so" >> - if not os.path.exists(stripped_filename) \ >> - or (os.path.getmtime(stripped_filename) < \ >> - os.path.getmtime(filename)): >> - subprocess.call(["strip", "-o", stripped_filename, >> filename]) >> - return stripped_filename >> - >> - >> # Send the files to the guest >> for name, hostname in files: >> if hostname.startswith("->"): >> -- >> 2.7.4 >> >> -- >> You received this message because you are subscribed to the Google Groups >> "OSv Development" group. >> To unsubscribe from this group and stop receiving emails from it, send an >> email to [email protected] <javascript:>. >> For more options, visit https://groups.google.com/d/optout. >> > > -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
