Commit: 5b27e6f24bd4c369dcb62ad81077c12373e4168b Author: Ray Molenkamp Date: Wed Oct 5 12:05:26 2022 -0600 Branches: blender-v3.3-release https://developer.blender.org/rB5b27e6f24bd4c369dcb62ad81077c12373e4168b
deps_builder: harden the package download process During the 3.3 release some packages were missing in SVN during the release and it ended up building the release tarball without issues when re-running the `make source_archive_complete` command after it failed initially. The tarball however had 0 byte files for the missing packages.... not good. This diff hardens the download process by : 1) Validating all required variables are set. This catches the erroneously attempt at downloading the nanovdb package even though we have removed it from versions.cmake but neglected to remove it from download.cmake 2) When a download fails (due to either a missing package, or bad download URL) FILE Download will warn about a hash mismatch but will carry on happily, you then have to go into the file system go delete the 0 byte file to retry the download. We know for a fact the file is bad when it is 0 bytes, just delete it. 3) When we are using the blender repository (and likely building a source archive) explicitly validate the hash of all packages. Normally the build process does this, however when building a source archive the build does not actually run for a dep. So preform this check during the configuration stage. Reviewed By: brecht Differential Revision: https://developer.blender.org/D16124 =================================================================== M build_files/build_environment/cmake/download.cmake =================================================================== diff --git a/build_files/build_environment/cmake/download.cmake b/build_files/build_environment/cmake/download.cmake index 76744aa9b0c..20944bd2c19 100644 --- a/build_files/build_environment/cmake/download.cmake +++ b/build_files/build_environment/cmake/download.cmake @@ -14,6 +14,20 @@ function(download_source dep) else() set(TARGET_URI https://svn.blender.org/svnroot/bf-blender/trunk/lib/packages/${TARGET_FILE}) endif() + # Validate all required variables are set and give an explicit error message + # rather than CMake erroring out later on with a more ambigious error. + if (NOT DEFINED TARGET_FILE) + message(FATAL_ERROR "${dep}_FILE variable not set") + endif() + if (NOT DEFINED TARGET_HASH) + message(FATAL_ERROR "${dep}_HASH variable not set") + endif() + if (NOT DEFINED TARGET_HASH_TYPE) + message(FATAL_ERROR "${dep}_HASH_TYPE variable not set") + endif() + if (NOT DEFINED TARGET_URI) + message(FATAL_ERROR "${dep}_URI variable not set") + endif() set(TARGET_FILE ${PACKAGE_DIR}/${TARGET_FILE}) message("Checking source : ${dep} (${TARGET_FILE})") if(NOT EXISTS ${TARGET_FILE}) @@ -25,6 +39,36 @@ function(download_source dep) SHOW_PROGRESS ) endif() + if(EXISTS ${TARGET_FILE}) + # Sometimes the download fails, but that is not a + # fail condition for "file(DOWNLOAD" it will warn about + # a crc mismatch and just carry on, we need to explicitly + # catch this and remove the bogus 0 byte file so we can + # retry without having to go find the file and manually + # delete it. + file (SIZE ${TARGET_FILE} TARGET_SIZE) + if(${TARGET_SIZE} EQUAL 0) + file(REMOVE ${TARGET_FILE}) + message(FATAL_ERROR "for ${TARGET_FILE} file size 0, download likely failed, deleted...") + endif() + + # If we are using sources from the blender repo also + # validate that the hashes match, this takes a + # little more time, but protects us when we are + # building a release package and one of the packages + # is missing or incorrect. + # + # For regular platform maintenaince this is not needed + # since the actual build of the dep will notify the + # platform maintainer if there is a problem with the + # source package and refuse to build. + if(NOT PACKAGE_USE_UPSTREAM_SOURCES) + file(${TARGET_HASH_TYPE} ${TARGET_FILE} LOCAL_HASH) + if(NOT ${TARGET_HASH} STREQUAL ${LOCAL_HASH}) + message(FATAL_ERROR "${TARGET_FILE} ${TARGET_HASH_TYPE} mismatch\nExpected\t: ${TARGET_HASH}\nActual\t: ${LOCAL_HASH}") + endif() + endif() + endif() endfunction(download_source) download_source(ZLIB) @@ -51,7 +95,6 @@ download_source(OSL) download_source(PYTHON) download_source(TBB) download_source(OPENVDB) -download_source(NANOVDB) download_source(NUMPY) download_source(LAME) download_source(OGG) _______________________________________________ Bf-blender-cvs mailing list [email protected] List details, subscription details or unsubscribe: https://lists.blender.org/mailman/listinfo/bf-blender-cvs
