Commit: 735469fea36d98281af33f32222f08b1931bbf80 Author: Sergey Sharybin Date: Fri Dec 13 15:58:07 2019 +0100 Branches: codesign https://developer.blender.org/rB735469fea36d98281af33f32222f08b1931bbf80
Codesign: Implement buildbot worker side DMG bundling Is based on bundle.sh but is adopted to an environment where codesigning is happening in a dedicated VM. Also fixed some code from previous commits: - Removed debug only early output when running commands - Ensured that path to git is always valid (wasn't a case when __file__ is relative to current directory). - Fixed simple code signer which couldn't have imported util. =================================================================== M build_files/buildbot/codesign/base_code_signer.py M build_files/buildbot/codesign/config_builder.py M build_files/buildbot/codesign/config_common.py M build_files/buildbot/codesign/config_server_template.py A build_files/buildbot/slave_bundle_dmg.py =================================================================== diff --git a/build_files/buildbot/codesign/base_code_signer.py b/build_files/buildbot/codesign/base_code_signer.py index 0481bcf2564..0505905c6f4 100644 --- a/build_files/buildbot/codesign/base_code_signer.py +++ b/build_files/buildbot/codesign/base_code_signer.py @@ -410,7 +410,7 @@ class BaseCodeSigner(metaclass=abc.ABCMeta): to verify logic of the code signing process. """ - if platform != self.platform or True: + if platform != self.platform: logger_server.info( f'Will run command for {platform}: {command}') return diff --git a/build_files/buildbot/codesign/config_builder.py b/build_files/buildbot/codesign/config_builder.py index 0cb83aba41e..1fa725ed28b 100644 --- a/build_files/buildbot/codesign/config_builder.py +++ b/build_files/buildbot/codesign/config_builder.py @@ -25,7 +25,7 @@ import sys from pathlib import Path -import util +import codesign.util as util from codesign.config_common import * @@ -34,6 +34,8 @@ if platform == util.Platform.LINUX: SHARED_STORAGE_DIR = Path('/data/codesign') elif platform == util.Platform.WINDOWS: SHARED_STORAGE_DIR = Path('Z:\\codesign') +elif platform == util.Platform.MACOS: + SHARED_STORAGE_DIR = Path('/Users/sergey/Developer/blender/codesign') # https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema LOGGING = { diff --git a/build_files/buildbot/codesign/config_common.py b/build_files/buildbot/codesign/config_common.py index 3710286c777..a37bc731dc0 100644 --- a/build_files/buildbot/codesign/config_common.py +++ b/build_files/buildbot/codesign/config_common.py @@ -24,7 +24,10 @@ from pathlib import Path # # This is how long buildbot packing step will wait signing server to # perform signing. -TIMEOUT_IN_SECONDS = 240 +# +# NOTE: Notarization could take a long time, hence the rather high value +# here. Might consider using different timeout for different platforms. +TIMEOUT_IN_SECONDS = 45 * 60 * 60 # Directory which is shared across buildbot worker and signing server. # diff --git a/build_files/buildbot/codesign/config_server_template.py b/build_files/buildbot/codesign/config_server_template.py index 1d6ddc54380..ff97ed15fa5 100644 --- a/build_files/buildbot/codesign/config_server_template.py +++ b/build_files/buildbot/codesign/config_server_template.py @@ -27,7 +27,7 @@ from pathlib import Path from codesign.config_common import * -CODESIGN_DIRECTORY = Path(__file__).parent +CODESIGN_DIRECTORY = Path(__file__).absolute().parent BLENDER_GIT_ROOT_DIRECTORY = CODESIGN_DIRECTORY.parent.parent.parent ################################################################################ diff --git a/build_files/buildbot/slave_bundle_dmg.py b/build_files/buildbot/slave_bundle_dmg.py new file mode 100755 index 00000000000..8a632d2ff3b --- /dev/null +++ b/build_files/buildbot/slave_bundle_dmg.py @@ -0,0 +1,529 @@ +#!/usr/bin/env python3 + +# ##### BEGIN GPL LICENSE BLOCK ##### +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# ##### END GPL LICENSE BLOCK ##### + +import argparse +import re +import shutil +import subprocess +import sys +import time + +from pathlib import Path +from tempfile import TemporaryDirectory, NamedTemporaryFile +from typing import List + +BUILDBOT_DIRECTORY = Path(__file__).absolute().parent +CODESIGN_SCRIPT = BUILDBOT_DIRECTORY / 'slave_codesign.py' +BLENDER_GIT_ROOT_DIRECTORY = BUILDBOT_DIRECTORY.parent.parent +DARWIN_DIRECTORY = BLENDER_GIT_ROOT_DIRECTORY / 'release' / 'darwin' + + +# Extra size which is added on top of actual files size when estimating size +# of destination DNG. +EXTRA_DMG_SIZE_IN_BYTES = 800 * 1024 * 1024 + +################################################################################ +# Common utilities + + +def get_directory_size(root_directory: Path) -> int: + """ + Get size of directory on disk + """ + + total_size = 0 + for file in root_directory.glob('**/*'): + total_size += file.lstat().st_size + return total_size + + +################################################################################ +# DMG bundling specific logic + +def create_argument_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'source_dir', + type=Path, + help='Source directory which points to either existing .app bundle' + 'or to a directory with .app bundles.') + parser.add_argument( + '--background-image', + type=Path, + help="Optional background picture which will be set on the DMG." + "If not provided default Blender's one is used.") + parser.add_argument( + '--volume-name', + type=str, + help='Optional name of a volume which will be used for DMG.') + parser.add_argument( + '--dmg', + type=Path, + help='Optional argument which points to a final DMG file name.') + parser.add_argument( + '--applescript', + type=Path, + help="Optional path to applescript to set up folder looks of DMG." + "If not provided default Blender's one is used.") + return parser + + +def collect_app_bundles(source_dir: Path) -> List[Path]: + """ + Collect all app bundles which are to be put into DMG + + If the source directory points to FOO.app it will be the only app bundle + packed. + + Otherwise all .app bundles from given directory are placed to a single + DMG. + """ + + if source_dir.name.endswith('.app'): + return [source_dir] + + app_bundles = [] + for filename in source_dir.glob('*'): + if not filename.is_dir(): + continue + if not filename.name.endswith('.app'): + continue + + app_bundles.append(filename) + + return app_bundles + + +def collect_and_log_app_bundles(source_dir: Path) -> List[Path]: + app_bundles = collect_app_bundles(source_dir) + + if not app_bundles: + print('No app bundles found for packing') + return + + print(f'Found {len(app_bundles)} to pack:') + for app_bundle in app_bundles: + print(f'- {app_bundle}') + + return app_bundles + + +def estimate_dmg_size(app_bundles: List[Path]) -> int: + """ + Estimate size of DMG to hold requested app bundles + + The size is based on actual size of all files in all bundles plus some + space to compensate for different size-on-disk plus some space to hold + codesign signatures. + + Is better to be on a high side since the empty space is compressed, but + lack of space might cause silent failures later on. + """ + + app_bundles_size = 0 + for app_bundle in app_bundles: + app_bundles_size += get_directory_size(app_bundle) + + return app_bundles_size + EXTRA_DMG_SIZE_IN_BYTES + + +def copy_app_bundles_to_directory(app_bundles: List[Path], + directory: Path) -> None: + """ + Copy all bundles to a given directory + + This directory is what the DMG will be created from. + """ + for app_bundle in app_bundles: + print(f'Copying {app_bundle.name}...') + shutil.copytree(app_bundle, directory / app_bundle.name) + + +def create_dmg_image(app_bundles: List[Path], + dmg_filepath: Path, + volume_name: str) -> None: + """ + Create DMG disk image and put app bundles in it + + No DMG configuration or codesigning is happening here. + """ + + if dmg_filepath.exists(): + print(f'Removing existing writable DMG {dmg_filepath}...') + dmg_filepath.unlink() + + print('Preparing directory with app bundles for the DMG...') + with TemporaryDirectory(prefix='blender-dmg-content-') as content_dir_str: + # Copy all bundles to a clean directory. + content_dir = Path(content_dir_str) + copy_app_bundles_to_directory(app_bundles, content_dir) + + # Estimate size of the DMG. + dmg_size = estimate_dmg_size(app_bundles) + print(f'Estimated DMG size: {dmg_size:,} bytes.') + + # Create the DMG. + print(f'Creating writable DMG {dmg_filepath}') + command = ('hdiutil', + 'create', + '-size', str(dmg_size), + '-fs', 'HFS+', + '-srcfolder', content_dir, + '-volname', volume_name, + '-format', 'UDRW', + dmg_filepath) + subprocess.run(command) + + +def get_writable_dmg_filepath(dmg_filepath: Path): + """ + Get file path for writable DMG image + """ + parent = dmg_filepath.parent + return parent / (dmg_filepath.stem + '-temp.dmg') + + +def mount_readwrite_dmg(dmg_filepath: Path) -> None: + """ + Mount writable DMG + + Mounting point would be /Volumes/<volume name> + """ + + print(f'Mounting read-write DMG ${dmg_filepath}') + command = ('hdiutil', + 'attach', '-readwrite', + '-noverify', + '-noautoopen', + dmg_filepath) + subprocess.run(command) + + +def get_mount_directory_for_volume_name(volume_name: str) -> Path: + """ + Get directory under which the volume will be mounted + """ + + return Path('/Volumes') / volume_name + + +def eject_volume(volume_name: str) -> None: + """ + Eject given volume, if mounted + """ + mount_directory = get_mount_directory_for_volume_name(volume_name) + if not mount_directory.exists(): + return + mount_directory_str = str(mount_ @@ Diff output truncated at 10240 characters. @@ _______________________________________________ Bf-blender-cvs mailing list Bf-blender-cvs@blender.org https://lists.blender.org/mailman/listinfo/bf-blender-cvs