This patch introduces several options for the file(DOWNLOAD ...) command, which
would be useful in case of dealing with big files or unstable network
connections.
The added options are:
RETRY_COUNT <int> -- sets maximal amount of download restarts, default value: 1
RETRY_DELAY <real> -- sets delay before restarting download (in seconds),
default value: 0.0
RETRY_MAX_TIME <real> -- sets maximal time spent in downloading file
(in seconds), default value: infinity
RETRY_CONTINUE -- if set, makes cmake try to continue downloading of the
existing chunk, instead of discarding it and starting all over. This option is
not set by default
Notes:
The RETRY_CONTINUE option requires server-side support of http partial get
(content-range header).
Unfortunately, I haven't been able to properly test the RETRY_CONTINUE option,
as I didn't have access to the appropriate server. Any help in this area is
encouraged.
---
Source/cmFileCommand.cxx | 287 +++++++++++++++++++++++++++++++++--------------
1 file changed, 204 insertions(+), 83 deletions(-)
diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx
index 835b118..aa7782e 100644
--- a/Source/cmFileCommand.cxx
+++ b/Source/cmFileCommand.cxx
@@ -34,6 +34,16 @@
// include sys/stat.h after sys/types.h
#include <sys/stat.h>
+#include <float.h>
+#include <time.h>
+
+// For crossplatform_sleep().
+#if defined(_WIN32) && !defined(__CYGWIN__)
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
+
#include <cm_auto_ptr.hxx>
#include <cmsys/Directory.hxx>
#include <cmsys/Encoding.hxx>
@@ -68,6 +78,15 @@ static mode_t mode_setuid = S_ISUID;
static mode_t mode_setgid = S_ISGID;
#endif
+void crossplatform_sleep(int delay_s)
+{
+#if defined(_WIN32) && !defined(__CYGWIN__)
+ Sleep(delay_s);
+#else
+ sleep(delay_s * 1000);
+#endif
+}
+
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
// libcurl doesn't support file:// urls for unicode filenames on Windows.
// Convert string from UTF-8 to ACP if this is a file:// URL.
@@ -2481,6 +2500,11 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
std::string hashMatchMSG;
CM_AUTO_PTR<cmCryptoHash> hash;
bool showProgress = false;
+ int retryMaxCount = 1;
+ double retryDelayS = 0.0;
+ double retryMaxTimeS = DBL_MAX;
+ bool retryContinue = false;
+ cmsys::ofstream fout;
while (i != args.end()) {
if (*i == "TIMEOUT") {
@@ -2564,7 +2588,34 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
return false;
}
hashMatchMSG = algo + " hash";
+ } else if (*i == "RETRY_COUNT") {
+ ++i;
+ if (i != args.end()) {
+ retryMaxCount = atoi(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing count for RETRY_COUNT");
+ return false;
+ }
+ } else if (*i == "RETRY_DELAY") {
+ ++i;
+ if (i != args.end()) {
+ retryDelayS = atof(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing time for RETRY_DELAY");
+ return false;
+ }
+ } else if (*i == "RETRY_MAX_TIME") {
+ ++i;
+ if (i != args.end()) {
+ retryMaxTimeS = atof(i->c_str());
+ } else {
+ this->SetError("DOWNLOAD missing time for RETRY_MAX_TIME");
+ return false;
+ }
+ } else if (*i == "RETRY_CONTINUE") {
+ retryContinue = true;
}
+
++i;
}
// If file exists already, and caller specified an expected md5 or sha,
@@ -2599,110 +2650,171 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
return false;
}
- cmsys::ofstream fout(file.c_str(), std::ios::binary);
- if (!fout) {
- this->SetError("DOWNLOAD cannot open file for write.");
- return false;
- }
-
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
url = fix_file_url_windows(url);
#endif
+ cmFileCommandVectorOfChar chunkDebug;
+
::CURL* curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
- curl = ::curl_easy_init();
- if (!curl) {
- this->SetError("DOWNLOAD error initializing curl.");
- return false;
- }
- cURLEasyGuard g_curl(curl);
- ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
- check_curl_result(res, "DOWNLOAD cannot set url: ");
+ ::CURLcode res;
+ int tries = 0;
+ double elapsed = 0.0;
+ time_t start, end;
+ while (tries < retryMaxCount && elapsed <= retryMaxTimeS) {
+ ++tries;
+ time(&start);
+
+ curl = ::curl_easy_init();
+ if (!curl) {
+ this->SetError("DOWNLOAD error initializing curl.");
+ ::curl_global_cleanup();
+ return false;
+ }
- // enable HTTP ERROR parsing
- res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
- check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
+ if (cmSystemTools::FileExists(file.c_str())) { // Something was downloaded.
+ // Check hash.
+ if (hash.get()) {
+ std::string actualHash = hash->HashFile(file);
+ if (actualHash == expectedHash) { // File is complete, exit.
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return true;
+ }
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION);
- check_curl_result(res, "DOWNLOAD cannot set user agent option: ");
+ if (retryContinue == false) { // Discard downloaded chunk.
+ fout.open(file.c_str(), std::ios::binary | std::ios::trunc);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ } else { // Try to continue.
+ fout.open(file.c_str(), std::ios::binary | std::ios::app);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, fout.tellp());
+ }
+ } else { // Create new file.
+ fout.open(file.c_str(), std::ios::binary);
+ if (!fout.good()) {
+ this->SetError("Cannot open file for writing");
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback);
- check_curl_result(res, "DOWNLOAD cannot set write function: ");
+ cURLEasyGuard g_curl(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+ check_curl_result(res, "DOWNLOAD cannot set url: ");
- res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
- cmFileCommandCurlDebugCallback);
- check_curl_result(res, "DOWNLOAD cannot set debug function: ");
+ // enable HTTP ERROR parsing
+ res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+ check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
- // check to see if TLS verification is requested
- if (tls_verify) {
- res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
- check_curl_result(res, "Unable to set TLS/SSL Verify on: ");
- } else {
- res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
- check_curl_result(res, "Unable to set TLS/SSL Verify off: ");
- }
- // check to see if a CAINFO file has been specified
- // command arg comes first
- std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);
- if (!cainfo_err.empty()) {
- this->SetError(cainfo_err);
- return false;
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_USERAGENT, "curl/" LIBCURL_VERSION);
+ check_curl_result(res, "DOWNLOAD cannot set user agent option: ");
- cmFileCommandVectorOfChar chunkDebug;
+ res =
+ ::curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cmWriteToFileCallback);
+ check_curl_result(res, "DOWNLOAD cannot set write function: ");
- res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout);
- check_curl_result(res, "DOWNLOAD cannot set write data: ");
+ res = ::curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION,
+ cmFileCommandCurlDebugCallback);
+ check_curl_result(res, "DOWNLOAD cannot set debug function: ");
- res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug);
- check_curl_result(res, "DOWNLOAD cannot set debug data: ");
+ // check to see if TLS verification is requested
+ if (tls_verify) {
+ res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
+ check_curl_result(res, "Unable to set TLS/SSL Verify on: ");
+ } else {
+ res = ::curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
+ check_curl_result(res, "Unable to set TLS/SSL Verify off: ");
+ }
+ // check to see if a CAINFO file has been specified
+ // command arg comes first
+ std::string const& cainfo_err = cmCurlSetCAInfo(curl, cainfo);
+ if (!cainfo_err.empty()) {
+ this->SetError(cainfo_err);
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return false;
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
- check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
+ res = ::curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&fout);
+ check_curl_result(res, "DOWNLOAD cannot set write data: ");
- if (!logVar.empty()) {
- res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
- check_curl_result(res, "DOWNLOAD cannot set verbose: ");
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void*)&chunkDebug);
+ check_curl_result(res, "DOWNLOAD cannot set debug data: ");
- if (timeout > 0) {
- res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
- check_curl_result(res, "DOWNLOAD cannot set timeout: ");
- }
+ res = ::curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+ check_curl_result(res, "DOWNLOAD cannot set follow-redirect option: ");
- if (inactivity_timeout > 0) {
- // Give up if there is no progress for a long time.
- ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1);
- ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout);
- }
+ if (!logVar.empty()) {
+ res = ::curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+ check_curl_result(res, "DOWNLOAD cannot set verbose: ");
+ }
- // Need the progress helper's scope to last through the duration of
- // the curl_easy_perform call... so this object is declared at function
- // scope intentionally, rather than inside the "if(showProgress)"
- // block...
- //
- cURLProgressHelper helper(this, "download");
+ if (timeout > 0) {
+ res = ::curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
+ check_curl_result(res, "DOWNLOAD cannot set timeout: ");
+ }
- if (showProgress) {
- res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
- check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
+ if (inactivity_timeout > 0) {
+ // Give up if there is no progress for a long time.
+ ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1);
+ ::curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, inactivity_timeout);
+ }
- res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,
- cmFileDownloadProgressCallback);
- check_curl_result(res, "DOWNLOAD cannot set progress function: ");
+ // Need the progress helper's scope to last through the duration of
+ // the curl_easy_perform call... so this object is declared at loop
+ // scope intentionally, rather than inside the "if(showProgress)"
+ // block...
+ //
+ cURLProgressHelper helper(this, "download");
- res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA,
- reinterpret_cast<void*>(&helper));
- check_curl_result(res, "DOWNLOAD cannot set progress data: ");
- }
+ if (showProgress) {
+ res = ::curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+ check_curl_result(res, "DOWNLOAD cannot set noprogress value: ");
- res = ::curl_easy_perform(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,
+ cmFileDownloadProgressCallback);
+ check_curl_result(res, "DOWNLOAD cannot set progress function: ");
- /* always cleanup */
- g_curl.release();
- ::curl_easy_cleanup(curl);
+ res = ::curl_easy_setopt(curl, CURLOPT_PROGRESSDATA,
+ reinterpret_cast<void*>(&helper));
+ check_curl_result(res, "DOWNLOAD cannot set progress data: ");
+ }
+
+ res = ::curl_easy_perform(curl);
+
+ /* always cleanup */
+ g_curl.release();
+ ::curl_easy_cleanup(curl);
+ fout.flush();
+ fout.close();
+
+ // Download finished successfuly, exit the loop.
+ if (res == ::CURLE_OK) {
+ break;
+ // Server doesn't support content ranges...
+ } else if (retryContinue == true && res == ::CURLE_RANGE_ERROR) {
+ retryContinue = false; // Disable download continuation.
+ }
+
+ crossplatform_sleep(retryDelayS);
+ time(&end);
+ elapsed += difftime(end, start);
+ }
if (!statusVar.empty()) {
std::ostringstream result;
@@ -2712,10 +2824,19 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector<std::string> const& args)
::curl_global_cleanup();
- // Explicitly flush/close so we can measure the md5 accurately.
- //
- fout.flush();
- fout.close();
+ if (res != ::CURLE_OK) {
+ std::ostringstream oss;
+ // Failed by exhausting attempts
+ if (retryMaxCount != 1 && tries == retryMaxCount) {
+ oss << "Download failed after " << tries << " attempts. ";
+ }
+ // Failed by exhausting maximal time.
+ if (retryMaxTimeS < DBL_MAX && elapsed >= retryMaxTimeS) {
+ oss << "Download failed: time exhausted, " << elapsed << "s. spent. ";
+ }
+ oss << "Last CURL error: " << curl_easy_strerror(res);
+ return false;
+ }
// Verify MD5 sum if requested:
//
--
2.9.2
--
Powered by www.kitware.com
Please keep messages on-topic and check the CMake FAQ at:
http://www.cmake.org/Wiki/CMake_FAQ
Kitware offers various services to support the CMake community. For more
information on each offering, please visit:
CMake Support: http://cmake.org/cmake/help/support.html
CMake Consulting: http://cmake.org/cmake/help/consulting.html
CMake Training Courses: http://cmake.org/cmake/help/training.html
Visit other Kitware open-source projects at
http://www.kitware.com/opensource/opensource.html
Follow this link to subscribe/unsubscribe:
http://public.kitware.com/mailman/listinfo/cmake-developers
