[cmake-developers] Subject: [PATCH v2] cmFileCommand: Download continuation support
This patch introduces several options for the file(DOWNLOAD ...) command, which
would be useful in case of dealing with big files or unstable network
connections.
The added options are:
RETRY_COUNT -- sets maximal amount of download restarts, default value: 1
RETRY_DELAY -- sets delay before restarting download (in seconds),
default value: 0.0
RETRY_MAX_TIME -- sets maximal time spent in downloading file
(in seconds), default value: infinity
RETRY_CONTINUE -- if set, makes cmake try to continue downloading of the
existing chunk, instead of discarding it and starting all over. This option is
not set by default
Notes:
The RETRY_CONTINUE option requires server-side support of http partial get
(content-range header).
Unfortunately, I haven't been able to properly test the RETRY_CONTINUE option,
as I didn't have access to the appropriate server. Any help in this area is
encouraged.
---
Help/command/file.rst| 17 +++
Source/cmFileCommand.cxx | 271 ---
2 files changed, 205 insertions(+), 83 deletions(-)
diff --git a/Help/command/file.rst b/Help/command/file.rst
index 256d16d..f1095b7 100644
--- a/Help/command/file.rst
+++ b/Help/command/file.rst
@@ -240,6 +240,23 @@ Additional options to ``DOWNLOAD`` are:
``TLS_CAINFO ``
Specify a custom Certificate Authority file for ``https://`` URLs.
+``RETRY_COUNT ``
+ Set maximal amount of download restarts, default value: 1
+
+``RETRY_DELAY ``
+ Set delay before restarting download, default value: 0.0
+
+``RETRY_MAX_TIME ``
+ Set maximal time spent in downloading file, default value: infinity
+
+``RETRY_CONTINUE``
+ If set, makes cmake try to continue downloading of the existing chunk,
+ instead of discarding it and starting all over. This option is not set
+ by default.
+
+ Note that this option requires server-side support of http partial get
+ (content-range header).
+
For ``https://`` URLs CMake must be built with OpenSSL support. ``TLS/SSL``
certificates are not checked by default. Set ``TLS_VERIFY`` to ``ON`` to
check certificates and/or use ``EXPECTED_HASH`` to verify downloaded content.
diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx
index 835b118..bbe8839 100644
--- a/Source/cmFileCommand.cxx
+++ b/Source/cmFileCommand.cxx
@@ -34,6 +34,9 @@
// include sys/stat.h after sys/types.h
#include
+#include
+#include
+
#include
#include
#include
@@ -2481,6 +2484,11 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector const& args)
std::string hashMatchMSG;
CM_AUTO_PTR hash;
bool showProgress = false;
+ int retryMaxCount = 1;
+ double retryDelayS = 0.0;
+ double retryMaxTimeS = DBL_MAX;
+ bool retryContinue = false;
+ cmsys::ofstream fout;
while (i != args.end()) {
if (*i == "TIMEOUT") {
@@ -2564,7 +2572,34 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector const& args)
return false;
}
hashMatchMSG = algo + " hash";
+} else if (*i == "RETRY_COUNT") {
+ ++i;
+ if (i != args.end()) {
+retryMaxCount = atoi(i->c_str());
+ } else {
+this->SetError("DOWNLOAD missing count for RETRY_COUNT");
+return false;
+ }
+} else if (*i == "RETRY_DELAY") {
+ ++i;
+ if (i != args.end()) {
+retryDelayS = atof(i->c_str());
+ } else {
+this->SetError("DOWNLOAD missing time for RETRY_DELAY");
+return false;
+ }
+} else if (*i == "RETRY_MAX_TIME") {
+ ++i;
+ if (i != args.end()) {
+retryMaxTimeS = atof(i->c_str());
+ } else {
+this->SetError("DOWNLOAD missing time for RETRY_MAX_TIME");
+return false;
+ }
+} else if (*i == "RETRY_CONTINUE") {
+ retryContinue = true;
}
+
++i;
}
// If file exists already, and caller specified an expected md5 or sha,
@@ -2599,110 +2634,171 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector const& args)
return false;
}
- cmsys::ofstream fout(file.c_str(), std::ios::binary);
- if (!fout) {
-this->SetError("DOWNLOAD cannot open file for write.");
-return false;
- }
-
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
url = fix_file_url_windows(url);
#endif
+ cmFileCommandVectorOfChar chunkDebug;
+
::CURL* curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
- curl = ::curl_easy_init();
- if (!curl) {
-this->SetError("DOWNLOAD error initializing curl.");
-return false;
- }
- cURLEasyGuard g_curl(curl);
- ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
- check_curl_result(res, "DOWNLOAD cannot set url: ");
+ ::CURLcode res;
+ int tries = 0;
+ double elapsed = 0.0;
+ time_t start, end;
+ while (tries < retryMaxCount && elapsed <= retryMaxTimeS) {
+++tries;
+time();
+
+curl = ::curl_easy_init();
+if (!curl) {
+ this->SetError("DOWNLOAD error initializing curl.");
+ ::curl_global_cleanup();
+ return false;
+}
- // enable HTTP
[cmake-developers] [PATCH] cmFileCommand: Download continuation support
This patch introduces several options for the file(DOWNLOAD ...) command, which
would be useful in case of dealing with big files or unstable network
connections.
The added options are:
RETRY_COUNT -- sets maximal amount of download restarts, default value: 1
RETRY_DELAY -- sets delay before restarting download (in seconds),
default value: 0.0
RETRY_MAX_TIME -- sets maximal time spent in downloading file
(in seconds), default value: infinity
RETRY_CONTINUE -- if set, makes cmake try to continue downloading of the
existing chunk, instead of discarding it and starting all over. This option is
not set by default
Notes:
The RETRY_CONTINUE option requires server-side support of http partial get
(content-range header).
Unfortunately, I haven't been able to properly test the RETRY_CONTINUE option,
as I didn't have access to the appropriate server. Any help in this area is
encouraged.
---
Source/cmFileCommand.cxx | 287 +--
1 file changed, 204 insertions(+), 83 deletions(-)
diff --git a/Source/cmFileCommand.cxx b/Source/cmFileCommand.cxx
index 835b118..aa7782e 100644
--- a/Source/cmFileCommand.cxx
+++ b/Source/cmFileCommand.cxx
@@ -34,6 +34,16 @@
// include sys/stat.h after sys/types.h
#include
+#include
+#include
+
+// For crossplatform_sleep().
+#if defined(_WIN32) && !defined(__CYGWIN__)
+#include
+#else
+#include
+#endif
+
#include
#include
#include
@@ -68,6 +78,15 @@ static mode_t mode_setuid = S_ISUID;
static mode_t mode_setgid = S_ISGID;
#endif
+void crossplatform_sleep(int delay_s)
+{
+#if defined(_WIN32) && !defined(__CYGWIN__)
+ Sleep(delay_s);
+#else
+ sleep(delay_s * 1000);
+#endif
+}
+
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
// libcurl doesn't support file:// urls for unicode filenames on Windows.
// Convert string from UTF-8 to ACP if this is a file:// URL.
@@ -2481,6 +2500,11 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector const& args)
std::string hashMatchMSG;
CM_AUTO_PTR hash;
bool showProgress = false;
+ int retryMaxCount = 1;
+ double retryDelayS = 0.0;
+ double retryMaxTimeS = DBL_MAX;
+ bool retryContinue = false;
+ cmsys::ofstream fout;
while (i != args.end()) {
if (*i == "TIMEOUT") {
@@ -2564,7 +2588,34 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector const& args)
return false;
}
hashMatchMSG = algo + " hash";
+} else if (*i == "RETRY_COUNT") {
+ ++i;
+ if (i != args.end()) {
+retryMaxCount = atoi(i->c_str());
+ } else {
+this->SetError("DOWNLOAD missing count for RETRY_COUNT");
+return false;
+ }
+} else if (*i == "RETRY_DELAY") {
+ ++i;
+ if (i != args.end()) {
+retryDelayS = atof(i->c_str());
+ } else {
+this->SetError("DOWNLOAD missing time for RETRY_DELAY");
+return false;
+ }
+} else if (*i == "RETRY_MAX_TIME") {
+ ++i;
+ if (i != args.end()) {
+retryMaxTimeS = atof(i->c_str());
+ } else {
+this->SetError("DOWNLOAD missing time for RETRY_MAX_TIME");
+return false;
+ }
+} else if (*i == "RETRY_CONTINUE") {
+ retryContinue = true;
}
+
++i;
}
// If file exists already, and caller specified an expected md5 or sha,
@@ -2599,110 +2650,171 @@ bool
cmFileCommand::HandleDownloadCommand(std::vector const& args)
return false;
}
- cmsys::ofstream fout(file.c_str(), std::ios::binary);
- if (!fout) {
-this->SetError("DOWNLOAD cannot open file for write.");
-return false;
- }
-
#if defined(_WIN32) && defined(CMAKE_ENCODING_UTF8)
url = fix_file_url_windows(url);
#endif
+ cmFileCommandVectorOfChar chunkDebug;
+
::CURL* curl;
::curl_global_init(CURL_GLOBAL_DEFAULT);
- curl = ::curl_easy_init();
- if (!curl) {
-this->SetError("DOWNLOAD error initializing curl.");
-return false;
- }
- cURLEasyGuard g_curl(curl);
- ::CURLcode res = ::curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
- check_curl_result(res, "DOWNLOAD cannot set url: ");
+ ::CURLcode res;
+ int tries = 0;
+ double elapsed = 0.0;
+ time_t start, end;
+ while (tries < retryMaxCount && elapsed <= retryMaxTimeS) {
+++tries;
+time();
+
+curl = ::curl_easy_init();
+if (!curl) {
+ this->SetError("DOWNLOAD error initializing curl.");
+ ::curl_global_cleanup();
+ return false;
+}
- // enable HTTP ERROR parsing
- res = ::curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
- check_curl_result(res, "DOWNLOAD cannot set http failure option: ");
+if (cmSystemTools::FileExists(file.c_str())) { // Something was downloaded.
+ // Check hash.
+ if (hash.get()) {
+std::string actualHash = hash->HashFile(file);
+if (actualHash == expectedHash) { // File is complete, exit.
+ ::curl_easy_cleanup(curl);
+ ::curl_global_cleanup();
+ return true;
+}
+ }
- res =
