Factor the size checks out to separate functions. Besides, move the definitions of functions called therein before the definition of the new functions, for better readability.
Signed-off-by: Roman Kagan <[email protected]> --- v2v/v2v.ml | 378 +++++++++++++++++++++++++++++++------------------------------ 1 file changed, 190 insertions(+), 188 deletions(-) diff --git a/v2v/v2v.ml b/v2v/v2v.ml index c20cbf0..53456ea 100644 --- a/v2v/v2v.ml +++ b/v2v/v2v.ml @@ -183,6 +183,194 @@ let init_targets overlays source output output_format = output#prepare_targets source targets +(* Conversion can fail if there is no space on the guest filesystems + * (RHBZ#1139543). To avoid this situation, check there is some + * headroom. Mainly we care about the root filesystem. + *) +let check_free_space mpstats = + message (f_"Checking for sufficient free disk space in the guest"); + List.iter ( + fun { mp_path = mp; + mp_statvfs = { G.bfree = bfree; blocks = blocks; bsize = bsize } } -> + (* Ignore small filesystems. *) + let total_size = blocks *^ bsize in + if total_size > 100_000_000L then ( + (* bfree = free blocks for root user *) + let free_bytes = bfree *^ bsize in + let needed_bytes = + match mp with + | "/" -> + (* We may install some packages, and they would usually go + * on the root filesystem. + *) + 20_000_000L + | "/boot" -> + (* We usually regenerate the initramfs, which has a + * typical size of 20-30MB. Hence: + *) + 50_000_000L + | _ -> + (* For everything else, just make sure there is some free space. *) + 10_000_000L in + + if free_bytes < needed_bytes then + error (f_"not enough free space for conversion on filesystem '%s'. %Ld bytes free < %Ld bytes needed") + mp free_bytes needed_bytes + ) + ) mpstats + +(* Estimate the space required on the target for each disk. It is the + * maximum space that might be required, but in reasonable cases much + * less space would actually be needed. + * + * As a starting point we could take ov_virtual_size (plus a tiny + * overhead for qcow2 headers etc) as the maximum. However that's not + * very useful. Other information we have available is: + * + * - The list of filesystems across the source disk(s). + * + * - The disk used/free of each of those filesystems, and the + * filesystem type. + * + * Note that we do NOT have the used size of the source disk (because + * it may be remote). + * + * How do you attribute filesystem usage through to backing disks, + * since one filesystem might span multiple disks? + * + * How do you account for non-filesystem usage (eg. swap, partitions + * that libguestfs cannot read, the space between LVs/partitions)? + * + * Another wildcard is that although we try to run {c fstrim} on each + * source filesystem, it can fail in some common scenarios. Also + * qemu-img will do zero detection. Both of these can be big wins when + * they work. + * + * The algorithm used here is this: + * + * (1) Calculate the total virtual size of all guest filesystems. + * eg: [ "/boot" = 500 MB, "/" = 2.5 GB ], total = 3 GB + * + * (2) Calculate the total virtual size of all source disks. + * eg: [ sda = 1 GB, sdb = 3 GB ], total = 4 GB + * + * (3) The ratio of (1):(2) is the maximum that could be freed up if + * all filesystems were effectively zeroed out during the conversion. + * eg. ratio = 3/4 + * + * (4) Work out how much filesystem space we are likely to save if + * fstrim works, but exclude a few cases where fstrim will probably + * fail (eg. filesystems that don't support fstrim). This is the + * conversion saving. + * eg. [ "/boot" = 200 MB used, "/" = 1 GB used ], saving = 3 - 1.2 = 1.8 + * + * (5) Scale the conversion saving (4) by the ratio (3), and allocate + * that saving across all source disks in proportion to their + * virtual size. + * eg. scaled saving is 1.8 * 3/4 = 1.35 GB + * sda has 1/4 of total virtual size, so it gets a saving of 1.35/4 + * sda final estimated size = 1 - (1.35/4) = 0.6625 GB + * sdb has 3/4 of total virtual size, so it gets a saving of 3 * 1.35 / 4 + * sdb final estimate size = 3 - (3*1.35/4) = 1.9875 GB + *) +let estimate_target_size mpstats targets = + let sum = List.fold_left (+^) 0L in + + (* (1) *) + let fs_total_size = + sum ( + List.map (fun { mp_statvfs = s } -> s.G.blocks *^ s.G.bsize) mpstats + ) in + if verbose () then + printf "estimate_target_size: fs_total_size = %Ld [%s]\n%!" + fs_total_size (human_size fs_total_size); + + (* (2) *) + let source_total_size = + sum (List.map (fun t -> t.target_overlay.ov_virtual_size) targets) in + if verbose () then + printf "estimate_target_size: source_total_size = %Ld [%s]\n%!" + source_total_size (human_size source_total_size); + + if source_total_size = 0L then (* Avoid divide by zero error. *) + targets + else ( + (* (3) Store the ratio as a float to avoid overflows later. *) + let ratio = + Int64.to_float fs_total_size /. Int64.to_float source_total_size in + if verbose () then + printf "estimate_target_size: ratio = %.3f\n%!" ratio; + + (* (4) *) + let fs_free = + sum ( + List.map ( + function + (* On filesystems supported by fstrim, assume we can save all + * the free space. + *) + | { mp_vfs = "ext2"|"ext3"|"ext4"|"xfs"; mp_statvfs = s } -> + s.G.bfree *^ s.G.bsize + + (* fstrim is only supported on NTFS very recently, and has a + * lot of limitations. So make the safe assumption for now + * that it's not going to work. + *) + | { mp_vfs = "ntfs" } -> 0L + + (* For other filesystems, sorry we can't free anything :-/ *) + | _ -> 0L + ) mpstats + ) in + if verbose () then + printf "estimate_target_size: fs_free = %Ld [%s]\n%!" + fs_free (human_size fs_free); + let scaled_saving = Int64.of_float (Int64.to_float fs_free *. ratio) in + if verbose () then + printf "estimate_target_size: scaled_saving = %Ld [%s]\n%!" + scaled_saving (human_size scaled_saving); + + (* (5) *) + let targets = List.map ( + fun ({ target_overlay = ov } as t) -> + let size = ov.ov_virtual_size in + let proportion = + Int64.to_float size /. Int64.to_float source_total_size in + let estimated_size = + size -^ Int64.of_float (proportion *. Int64.to_float scaled_saving) in + if verbose () then + printf "estimate_target_size: %s: %Ld [%s]\n%!" + ov.ov_sd estimated_size (human_size estimated_size); + { t with target_estimated_size = Some estimated_size } + ) targets in + + targets + ) + +let get_mpstats g = + (* Collect statvfs information from the guest mountpoints. *) + let mpstats = List.map ( + fun (dev, path) -> + let statvfs = g#statvfs path in + let vfs = g#vfs_type dev in + { mp_dev = dev; mp_path = path; mp_statvfs = statvfs; mp_vfs = vfs } + ) (g#mountpoints ()) in + + if verbose () then ( + (* This is useful for debugging speed / fstrim issues. *) + printf "mpstats:\n"; + List.iter (print_mpstat Pervasives.stdout) mpstats + ); + + mpstats + +let check_target_free_space mpstats source targets output = + (* Estimate space required on target for each disk. Note this is a max. *) + message (f_"Estimating space required on target for each disk"); + let targets = estimate_target_size mpstats targets in + + output#check_target_free_space source targets + let rec main () = (* Handle the command line. *) let input, output, @@ -222,32 +410,9 @@ let rec main () = message (f_"Inspecting the overlay"); let inspect = inspect_source g root_choice in - (* The guest free disk space check and the target free space - * estimation both require statvfs information from mountpoints, so - * get that information first. - *) - let mpstats = List.map ( - fun (dev, path) -> - let statvfs = g#statvfs path in - let vfs = g#vfs_type dev in - { mp_dev = dev; mp_path = path; mp_statvfs = statvfs; mp_vfs = vfs } - ) (g#mountpoints ()) in - - if verbose () then ( - (* This is useful for debugging speed / fstrim issues. *) - printf "mpstats:\n"; - List.iter (print_mpstat Pervasives.stdout) mpstats - ); - - (* Check there is enough free space to perform conversion. *) - message (f_"Checking for sufficient free disk space in the guest"); + let mpstats = get_mpstats g in check_free_space mpstats; - - (* Estimate space required on target for each disk. Note this is a max. *) - message (f_"Estimating space required on target for each disk"); - let targets = estimate_target_size mpstats targets in - - output#check_target_free_space source targets; + check_target_free_space mpstats source targets output; (* Conversion. *) let guestcaps = @@ -597,41 +762,6 @@ and inspect_source g root_choice = if verbose () then printf "%s%!" (string_of_inspect inspect); inspect -(* Conversion can fail if there is no space on the guest filesystems - * (RHBZ#1139543). To avoid this situation, check there is some - * headroom. Mainly we care about the root filesystem. - *) -and check_free_space mpstats = - List.iter ( - fun { mp_path = mp; - mp_statvfs = { G.bfree = bfree; blocks = blocks; bsize = bsize } } -> - (* Ignore small filesystems. *) - let total_size = blocks *^ bsize in - if total_size > 100_000_000L then ( - (* bfree = free blocks for root user *) - let free_bytes = bfree *^ bsize in - let needed_bytes = - match mp with - | "/" -> - (* We may install some packages, and they would usually go - * on the root filesystem. - *) - 20_000_000L - | "/boot" -> - (* We usually regenerate the initramfs, which has a - * typical size of 20-30MB. Hence: - *) - 50_000_000L - | _ -> - (* For everything else, just make sure there is some free space. *) - 10_000_000L in - - if free_bytes < needed_bytes then - error (f_"not enough free space for conversion on filesystem '%s'. %Ld bytes free < %Ld bytes needed") - mp free_bytes needed_bytes - ) - ) mpstats - (* Perform the fstrim. The trimming bit is easy. Dealing with the * [--no-trim] parameter .. not so much. *) @@ -691,134 +821,6 @@ and do_fstrim g no_trim inspect = ) ) fses -(* Estimate the space required on the target for each disk. It is the - * maximum space that might be required, but in reasonable cases much - * less space would actually be needed. - * - * As a starting point we could take ov_virtual_size (plus a tiny - * overhead for qcow2 headers etc) as the maximum. However that's not - * very useful. Other information we have available is: - * - * - The list of filesystems across the source disk(s). - * - * - The disk used/free of each of those filesystems, and the - * filesystem type. - * - * Note that we do NOT have the used size of the source disk (because - * it may be remote). - * - * How do you attribute filesystem usage through to backing disks, - * since one filesystem might span multiple disks? - * - * How do you account for non-filesystem usage (eg. swap, partitions - * that libguestfs cannot read, the space between LVs/partitions)? - * - * Another wildcard is that although we try to run {c fstrim} on each - * source filesystem, it can fail in some common scenarios. Also - * qemu-img will do zero detection. Both of these can be big wins when - * they work. - * - * The algorithm used here is this: - * - * (1) Calculate the total virtual size of all guest filesystems. - * eg: [ "/boot" = 500 MB, "/" = 2.5 GB ], total = 3 GB - * - * (2) Calculate the total virtual size of all source disks. - * eg: [ sda = 1 GB, sdb = 3 GB ], total = 4 GB - * - * (3) The ratio of (1):(2) is the maximum that could be freed up if - * all filesystems were effectively zeroed out during the conversion. - * eg. ratio = 3/4 - * - * (4) Work out how much filesystem space we are likely to save if - * fstrim works, but exclude a few cases where fstrim will probably - * fail (eg. filesystems that don't support fstrim). This is the - * conversion saving. - * eg. [ "/boot" = 200 MB used, "/" = 1 GB used ], saving = 3 - 1.2 = 1.8 - * - * (5) Scale the conversion saving (4) by the ratio (3), and allocate - * that saving across all source disks in proportion to their - * virtual size. - * eg. scaled saving is 1.8 * 3/4 = 1.35 GB - * sda has 1/4 of total virtual size, so it gets a saving of 1.35/4 - * sda final estimated size = 1 - (1.35/4) = 0.6625 GB - * sdb has 3/4 of total virtual size, so it gets a saving of 3 * 1.35 / 4 - * sdb final estimate size = 3 - (3*1.35/4) = 1.9875 GB - *) -and estimate_target_size mpstats targets = - let sum = List.fold_left (+^) 0L in - - (* (1) *) - let fs_total_size = - sum ( - List.map (fun { mp_statvfs = s } -> s.G.blocks *^ s.G.bsize) mpstats - ) in - if verbose () then - printf "estimate_target_size: fs_total_size = %Ld [%s]\n%!" - fs_total_size (human_size fs_total_size); - - (* (2) *) - let source_total_size = - sum (List.map (fun t -> t.target_overlay.ov_virtual_size) targets) in - if verbose () then - printf "estimate_target_size: source_total_size = %Ld [%s]\n%!" - source_total_size (human_size source_total_size); - - if source_total_size = 0L then (* Avoid divide by zero error. *) - targets - else ( - (* (3) Store the ratio as a float to avoid overflows later. *) - let ratio = - Int64.to_float fs_total_size /. Int64.to_float source_total_size in - if verbose () then - printf "estimate_target_size: ratio = %.3f\n%!" ratio; - - (* (4) *) - let fs_free = - sum ( - List.map ( - function - (* On filesystems supported by fstrim, assume we can save all - * the free space. - *) - | { mp_vfs = "ext2"|"ext3"|"ext4"|"xfs"; mp_statvfs = s } -> - s.G.bfree *^ s.G.bsize - - (* fstrim is only supported on NTFS very recently, and has a - * lot of limitations. So make the safe assumption for now - * that it's not going to work. - *) - | { mp_vfs = "ntfs" } -> 0L - - (* For other filesystems, sorry we can't free anything :-/ *) - | _ -> 0L - ) mpstats - ) in - if verbose () then - printf "estimate_target_size: fs_free = %Ld [%s]\n%!" - fs_free (human_size fs_free); - let scaled_saving = Int64.of_float (Int64.to_float fs_free *. ratio) in - if verbose () then - printf "estimate_target_size: scaled_saving = %Ld [%s]\n%!" - scaled_saving (human_size scaled_saving); - - (* (5) *) - let targets = List.map ( - fun ({ target_overlay = ov } as t) -> - let size = ov.ov_virtual_size in - let proportion = - Int64.to_float size /. Int64.to_float source_total_size in - let estimated_size = - size -^ Int64.of_float (proportion *. Int64.to_float scaled_saving) in - if verbose () then - printf "estimate_target_size: %s: %Ld [%s]\n%!" - ov.ov_sd estimated_size (human_size estimated_size); - { t with target_estimated_size = Some estimated_size } - ) targets in - - targets - ) - (* Update the target_actual_size field in the target structure. *) and actual_target_size target = { target with target_actual_size = du target.target_file } -- 2.4.3 _______________________________________________ Libguestfs mailing list [email protected] https://www.redhat.com/mailman/listinfo/libguestfs
