This patch introduces several options for the file(DOWNLOAD ...) command, which would be useful in case of dealing with big files or unstable network connections.
The added options are: RETRY_COUNT <int> -- sets maximal amount of download restarts, default value: 1 RETRY_DELAY <real> -- sets delay before restarting download (in seconds), default value: 0.0 RETRY_MAX_TIME <real> -- sets maximal time spent in downloading file (in seconds), default value: infinity RETRY_CONTINUE -- if set, makes cmake try to continue downloading of the existing chunk, instead of discarding it and starting all over. This option is not set by default Notes: The RETRY_CONTINUE option requires server-side support of http partial get (content-range header). Unfortunately, I haven't been able to properly test the RETRY_CONTINUE option, as I didn't have access to the appropriate server. Any help in this area is encouraged. --- Help/command/file.rst | 17 +++ Source/cmFileCommand.cxx | 271 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 205 insertions(+), 83 deletions(-) diff --git a/Help/command/file.rst b/Help/command/file.rst index 256d16d..f1095b7 100644 --- a/Help/command/file.rst +++ b/Help/command/file.rst @@ -240,6 +240,23 @@ Additional options to ``DOWNLOAD`` are: ``TLS_CAINFO <file>`` Specify a custom Certificate Authority file for ``https://`` URLs. +``RETRY_COUNT <count>`` + Set maximal amount of download restarts, default value: 1 + +``RETRY_DELAY <seconds>`` + Set delay before restarting download, default value: 0.0 + +``RETRY_MAX_TIME <seconds>`` + Set maximal time spent in downloading file, default value: infinity + +``RETRY_CONTINUE`` + If set, makes cmake try to continue downloading of the existing chunk, + instead of discarding it and starting all over. This option is not set + by default. + + Note that this option requires server-side support of http partial get + (content-range header). + For ``https://`` URLs CMake must be built with OpenSSL support. ``TLS/SSL`` certificates are not checked by default. Set ``TLS_VERIFY`` to ``ON`` to check certificates and/or use ``EXPECTED_HASH`` to verify downloaded content. diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx index 835b118..bbe8839 100644 --- a/Source/cmFileCommand.cxx +++ b/Source/cmFileCommand.cxx @@ -34,6 +34,9 @@ // include sys/stat.h after sys/types.h #include <sys/stat.h> +#include <float.h> +#include <time.h> + #include <cm_auto_ptr.hxx> #include <cmsys/Directory.hxx> #include <cmsys/Encoding.hxx> @@ -2481,6 +2484,11 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args) std::string hashMatchMSG; CM_AUTO_PTR<cmCryptoHash> hash; bool showProgress = false; + int retryMaxCount = 1; + double retryDelayS = 0.0; + double retryMaxTimeS = DBL_MAX; + bool retryContinue = false; + cmsys::ofstream fout; while (i != args.end()) { if (*i == "TIMEOUT") { @@ -2564,7 +2572,34 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args) return false; } hashMatchMSG = algo + " hash"; + } else if (*i == "RETRY_COUNT") { + ++i; + if (i != args.end()) { + retryMaxCount = atoi(i->c_str()); + } else { + this->SetError("DOWNLOAD missing count for RETRY_COUNT"); + return false; + } + } else if (*i == "RETRY_DELAY") { + ++i; + if (i != args.end()) { + retryDelayS = atof(i->c_str()); + } else { + this->SetError("DOWNLOAD missing time for RETRY_DELAY"); + return false; + } + } else if (*i == "RETRY_MAX_TIME") { + ++i; + if (i != args.end()) { + retryMaxTimeS = atof(i->c_str()); + } else { + this->SetError("DOWNLOAD missing time for RETRY_MAX_TIME"); + return false; + } + } else if (*i == "RETRY_CONTINUE") { + retryContinue = true; } + ++i; } // If file exists already, and caller specified an expected md5 or sha, @@ -2599,110 +2634,171 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args) return false; } - cmsys::ofstream fout(file.c_str(), std::ios::binary); - if (!fout) { - this->SetError("DOWNLOAD cannot open file for write."); - return false; - } - #if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8) url = fix_file_url_windows(url); #endif + cmFileCommandVectorOfChar chunkDebug; + ::CURL* curl; ::curl_global_init(CURL_GLOBAL_DEFAULT); - curl = ::curl_easy_init(); - if (!curl) { - this->SetError("DOWNLOAD error initializing curl."); - return false; - } - cURLEasyGuard g_curl(curl); - ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - check_curl_result(res, "DOWNLOAD cannot set url: "); + ::CURLcode res; + int tries = 0; + double elapsed = 0.0; + time_t start, end; + while (tries < retryMaxCount && elapsed <= retryMaxTimeS) { + ++tries; + time(&start); + + curl = ::curl_easy_init(); + if (!curl) { + this->SetError("DOWNLOAD error initializing curl."); + ::curl_global_cleanup(); + return false; + } - // enable HTTP ERROR parsing - res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); - check_curl_result(res, "DOWNLOAD cannot set http failure option: "); + if (cmSystemTools::FileExists(file.c_str())) { // Something was downloaded. + // Check hash. + if (hash.get()) { + std::string actualHash = hash->HashFile(file); + if (actualHash == expectedHash) { // File is complete, exit. + ::curl_easy_cleanup(curl); + ::curl_global_cleanup(); + return true; + } + } - res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION); - check_curl_result(res, "DOWNLOAD cannot set user agent option: "); + if (retryContinue == false) { // Discard downloaded chunk. + fout.open(file.c_str(), std::ios::binary | std::ios::trunc); + if (!fout.good()) { + this->SetError("Cannot open file for writing"); + ::curl_easy_cleanup(curl); + ::curl_global_cleanup(); + return false; + } + } else { // Try to continue. + fout.open(file.c_str(), std::ios::binary | std::ios::app); + if (!fout.good()) { + this->SetError("Cannot open file for writing"); + ::curl_easy_cleanup(curl); + ::curl_global_cleanup(); + return false; + } + curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, fout.tellp()); + } + } else { // Create new file. + fout.open(file.c_str(), std::ios::binary); + if (!fout.good()) { + this->SetError("Cannot open file for writing"); + ::curl_easy_cleanup(curl); + ::curl_global_cleanup(); + return false; + } + } - res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback); - check_curl_result(res, "DOWNLOAD cannot set write function: "); + cURLEasyGuard g_curl(curl); + res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + check_curl_result(res, "DOWNLOAD cannot set url: "); - res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, - cmFileCommandCurlDebugCallback); - check_curl_result(res, "DOWNLOAD cannot set debug function: "); + // enable HTTP ERROR parsing + res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1); + check_curl_result(res, "DOWNLOAD cannot set http failure option: "); - // check to see if TLS verification is requested - if (tls_verify) { - res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1); - check_curl_result(res, "Unable to set TLS/SSL Verify on: "); - } else { - res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); - check_curl_result(res, "Unable to set TLS/SSL Verify off: "); - } - // check to see if a CAINFO file has been specified - // command arg comes first - std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo); - if (!cainfo_err.empty()) { - this->SetError(cainfo_err); - return false; - } + res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION); + check_curl_result(res, "DOWNLOAD cannot set user agent option: "); - cmFileCommandVectorOfChar chunkDebug; + res = + ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback); + check_curl_result(res, "DOWNLOAD cannot set write function: "); - res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout); - check_curl_result(res, "DOWNLOAD cannot set write data: "); + res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, + cmFileCommandCurlDebugCallback); + check_curl_result(res, "DOWNLOAD cannot set debug function: "); - res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug); - check_curl_result(res, "DOWNLOAD cannot set debug data: "); + // check to see if TLS verification is requested + if (tls_verify) { + res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1); + check_curl_result(res, "Unable to set TLS/SSL Verify on: "); + } else { + res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0); + check_curl_result(res, "Unable to set TLS/SSL Verify off: "); + } + // check to see if a CAINFO file has been specified + // command arg comes first + std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo); + if (!cainfo_err.empty()) { + this->SetError(cainfo_err); + ::curl_easy_cleanup(curl); + ::curl_global_cleanup(); + return false; + } - res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: "); + res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout); + check_curl_result(res, "DOWNLOAD cannot set write data: "); - if (!logVar.empty()) { - res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); - check_curl_result(res, "DOWNLOAD cannot set verbose: "); - } + res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug); + check_curl_result(res, "DOWNLOAD cannot set debug data: "); - if (timeout > 0) { - res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); - check_curl_result(res, "DOWNLOAD cannot set timeout: "); - } + res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: "); - if (inactivity_timeout > 0) { - // Give up if there is no progress for a long time. - ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1); - ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout); - } + if (!logVar.empty()) { + res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + check_curl_result(res, "DOWNLOAD cannot set verbose: "); + } - // Need the progress helper's scope to last through the duration of - // the curl_easy_perform call... so this object is declared at function - // scope intentionally, rather than inside the "if(showProgress)" - // block... - // - cURLProgressHelper helper(this, "download"); + if (timeout > 0) { + res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); + check_curl_result(res, "DOWNLOAD cannot set timeout: "); + } - if (showProgress) { - res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); - check_curl_result(res, "DOWNLOAD cannot set noprogress value: "); + if (inactivity_timeout > 0) { + // Give up if there is no progress for a long time. + ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1); + ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout); + } - res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, - cmFileDownloadProgressCallback); - check_curl_result(res, "DOWNLOAD cannot set progress function: "); + // Need the progress helper's scope to last through the duration of + // the curl_easy_perform call... so this object is declared at loop + // scope intentionally, rather than inside the "if(showProgress)" + // block... + // + cURLProgressHelper helper(this, "download"); - res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, - reinterpret_cast<void*>(&helper)); - check_curl_result(res, "DOWNLOAD cannot set progress data: "); - } + if (showProgress) { + res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + check_curl_result(res, "DOWNLOAD cannot set noprogress value: "); - res = ::curl_easy_perform(curl); + res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, + cmFileDownloadProgressCallback); + check_curl_result(res, "DOWNLOAD cannot set progress function: "); - /* always cleanup */ - g_curl.release(); - ::curl_easy_cleanup(curl); + res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, + reinterpret_cast<void*>(&helper)); + check_curl_result(res, "DOWNLOAD cannot set progress data: "); + } + + res = ::curl_easy_perform(curl); + + /* always cleanup */ + g_curl.release(); + ::curl_easy_cleanup(curl); + fout.flush(); + fout.close(); + + // Download finished successfuly, exit the loop. + if (res == ::CURLE_OK) { + break; + // Server doesn't support content ranges... + } else if (retryContinue == true && res == ::CURLE_RANGE_ERROR) { + retryContinue = false; // Disable download continuation. + } + + cmUtils::Delay(retryDelayS * 1000); + time(&end); + elapsed += difftime(end, start); + } if (!statusVar.empty()) { std::ostringstream result; @@ -2712,10 +2808,19 @@ bool cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args) ::curl_global_cleanup(); - // Explicitly flush/close so we can measure the md5 accurately. - // - fout.flush(); - fout.close(); + if (res != ::CURLE_OK) { + std::ostringstream oss; + // Failed by exhausting attempts + if (retryMaxCount != 1 && tries == retryMaxCount) { + oss << "Download failed after " << tries << " attempts. "; + } + // Failed by exhausting maximal time. + if (retryMaxTimeS < DBL_MAX && elapsed >= retryMaxTimeS) { + oss << "Download failed: time exhausted, " << elapsed << "s. spent. "; + } + oss << "Last CURL error: " << curl_easy_strerror(res); + return false; + } // Verify MD5 sum if requested: // -- 2.9.2 -- Powered by www.kitware.com Please keep messages on-topic and check the CMake FAQ at: http://www.cmake.org/Wiki/CMake_FAQ Kitware offers various services to support the CMake community. For more information on each offering, please visit: CMake Support: http://cmake.org/cmake/help/support.html CMake Consulting: http://cmake.org/cmake/help/consulting.html CMake Training Courses: http://cmake.org/cmake/help/training.html Visit other Kitware open-source projects at http://www.kitware.com/opensource/opensource.html Follow this link to subscribe/unsubscribe: http://public.kitware.com/mailman/listinfo/cmake-developers
