This is an automated email from the git hooks/post-receive script. skitt pushed a commit to branch master in repository lgogdownloader.
commit 8a534553f63f60e3c50afac96cb26a2b60491401 Author: Stephen Kitt <[email protected]> Date: Mon Jan 16 09:10:04 2017 +0100 New upstream version 3.1 --- CMakeLists.txt | 2 +- include/api.h | 1 + include/downloader.h | 1 + include/gamedetails.h | 1 + include/gamefile.h | 1 + include/util.h | 10 + main.cpp | 69 ++++++- src/api.cpp | 23 ++- src/downloader.cpp | 501 ++++++++++++++++++-------------------------------- src/gamedetails.cpp | 22 +++ src/util.cpp | 13 +- src/website.cpp | 2 + 12 files changed, 307 insertions(+), 339 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e94267c..992cc90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR) -project (lgogdownloader LANGUAGES C CXX VERSION 3.0) +project (lgogdownloader LANGUAGES C CXX VERSION 3.1) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG=1") diff --git a/include/api.h b/include/api.h index 97abe45..93cfb47 100644 --- a/include/api.h +++ b/include/api.h @@ -55,6 +55,7 @@ class API API(const std::string& token,const std::string& secret); int init(); + bool isLoggedIn(); int login(const std::string& email, const std::string& password); int getAPIConfig(); std::string getResponse(const std::string& url); diff --git a/include/downloader.h b/include/downloader.h index 0aa1d45..cb0fa6b 100644 --- a/include/downloader.h +++ b/include/downloader.h @@ -63,6 +63,7 @@ class Downloader public: Downloader(Config &conf); virtual ~Downloader(); + bool isLoggedIn(); int init(); int login(); int listGames(); diff --git a/include/gamedetails.h b/include/gamedetails.h index abda193..7009364 100644 --- a/include/gamedetails.h +++ b/include/gamedetails.h @@ -35,6 +35,7 @@ class gameDetails std::string getSerialsFilepath(); std::string getChangelogFilepath(); Json::Value getDetailsAsJson(); + std::vector<gameFile> getGameFileVector(); virtual ~gameDetails(); protected: void filterListWithPriorities(std::vector<gameFile>& list, const gameSpecificConfig& config); diff --git a/include/gamefile.h b/include/gamefile.h index 80a37ef..90d6004 100644 --- a/include/gamefile.h +++ b/include/gamefile.h @@ -18,6 +18,7 @@ const unsigned int GFTYPE_INSTALLER = 1 << 0; const unsigned int GFTYPE_EXTRA = 1 << 1; const unsigned int GFTYPE_PATCH = 1 << 2; const unsigned int GFTYPE_LANGPACK = 1 << 3; +const unsigned int GFTYPE_DLC = 1 << 4; class gameFile { diff --git a/include/util.h b/include/util.h index f8e4176..b30f12c 100644 --- a/include/util.h +++ b/include/util.h @@ -15,6 +15,7 @@ #include <cerrno> #include <iostream> #include <sstream> +#include <memory> #include <rhash.h> #include <boost/filesystem.hpp> #include <boost/regex.hpp> @@ -49,6 +50,7 @@ struct gameItem std::string id; std::vector<std::string> dlcnames; Json::Value gamedetailsjson; + int updates = 0; }; struct wishlistItem @@ -90,6 +92,14 @@ namespace Util void parseOptionString(const std::string &option_string, std::vector<unsigned int> &priority, unsigned int &type, const std::vector<GlobalConstants::optionsStruct>& options); std::string getLocalFileHash(const std::string& xml_dir, const std::string& filepath, const std::string& gamename = std::string()); void shortenStringToTerminalWidth(std::string& str); + + template<typename ... Args> std::string formattedString(const std::string& format, Args ... args) + { + std::size_t sz = std::snprintf(nullptr, 0, format.c_str(), args ...) + 1; // +1 for null terminator + std::unique_ptr<char[]> buf(new char[sz]); + std::snprintf(buf.get(), sz, format.c_str(), args ...); + return std::string(buf.get(), buf.get() + sz - 1); // -1 because we don't want the null terminator + } } #endif // UTIL_H diff --git a/main.cpp b/main.cpp index 5a761fd..23b13c5 100644 --- a/main.cpp +++ b/main.cpp @@ -8,6 +8,7 @@ #include "config.h" #include "util.h" #include "globalconstants.h" +#include "ssl_thread_setup.h" #include <fstream> #include <boost/filesystem.hpp> @@ -475,17 +476,43 @@ int main(int argc, char *argv[]) std::cerr << std::endl; } + // Init curl globally + ssl_thread_setup(); + curl_global_init(CURL_GLOBAL_ALL); + + if (config.bLoginAPI) + { + config.sToken = ""; + config.sSecret = ""; + } + Downloader downloader(config); - int initResult = downloader.init(); - int iLoginResult = 0; - if (config.bLoginAPI || config.bLoginHTTP || initResult == 1) + int iLoginTries = 0; + bool bLoginOK = false; + + // Login because --login, --login-api or --login-website was used + if (config.bLoginAPI || config.bLoginHTTP) + bLoginOK = downloader.login(); + + bool bIsLoggedin = downloader.isLoggedIn(); + + // Login because we are not logged in + while (iLoginTries++ < config.iRetries && !bIsLoggedin) { - if (!config.bLoginAPI && !config.bLoginHTTP) - downloader.config.bLoginAPI = true; - iLoginResult = downloader.login(); - if (iLoginResult == 0) - return 1; + bLoginOK = downloader.login(); + if (bLoginOK) + { + bIsLoggedin = downloader.isLoggedIn(); + } + } + + // Login failed, cleanup + if (!bLoginOK && !bIsLoggedin) + { + curl_global_cleanup(); + ssl_thread_cleanup(); + return 1; } // Make sure that config file and cookie file are only readable/writable by owner @@ -495,9 +522,9 @@ int main(int argc, char *argv[]) Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write); } - if (config.bSaveConfig || iLoginResult == 1) + if (config.bSaveConfig || bLoginOK) { - if (iLoginResult == 1) + if (bLoginOK) { set_vm_value(vm, "token", downloader.config.sToken); set_vm_value(vm, "secret", downloader.config.sSecret); @@ -553,11 +580,17 @@ int main(int argc, char *argv[]) if (!config.bRespectUmask) Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write); if (config.bSaveConfig) + { + curl_global_cleanup(); + ssl_thread_cleanup(); return 0; + } } else { std::cerr << "Failed to create config: " << config.sConfigFilePath << std::endl; + curl_global_cleanup(); + ssl_thread_cleanup(); return 1; } } @@ -574,15 +607,28 @@ int main(int argc, char *argv[]) ofs.close(); if (!config.bRespectUmask) Util::setFilePermissions(config.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write); + + curl_global_cleanup(); + ssl_thread_cleanup(); return 0; } else { std::cerr << "Failed to create config: " << config.sConfigFilePath << std::endl; + curl_global_cleanup(); + ssl_thread_cleanup(); return 1; } } + bool bInitOK = downloader.init(); + if (!bInitOK) + { + curl_global_cleanup(); + ssl_thread_cleanup(); + return 1; + } + int res = 0; if (config.bShowWishlist) @@ -622,5 +668,8 @@ int main(int argc, char *argv[]) if (!config.sOrphanRegex.empty() && config.bDownload) downloader.checkOrphans(); + curl_global_cleanup(); + ssl_thread_cleanup(); + return res; } diff --git a/src/api.cpp b/src/api.cpp index 0ada213..f465ce1 100644 --- a/src/api.cpp +++ b/src/api.cpp @@ -34,17 +34,39 @@ API::API(const std::string& token, const std::string& secret) curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_PROGRESSDATA, this); curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1); this->error = false; this->config.oauth_token = token; this->config.oauth_secret = secret; } +/* Initialize the API + returns 0 if failed + returns 1 if successful +*/ int API::init() { int res = 0; + this->getAPIConfig(); + if (!this->getError()) + res = 1; + else + this->clearError(); + + return res; +} + +/* Login check + returns false if not logged in + returns true if logged in +*/ +bool API::isLoggedIn() +{ + int res = 0; + // Check if we already have token and secret if (!this->config.oauth_token.empty() && !this->config.oauth_secret.empty()) { @@ -55,7 +77,6 @@ int API::init() return res; } - int API::getAPIConfig() { std::string url = "https://api.gog.com/downloader2/status/stable/";; // Stable API diff --git a/src/downloader.cpp b/src/downloader.cpp index 9674cc2..d0019d0 100644 --- a/src/downloader.cpp +++ b/src/downloader.cpp @@ -7,7 +7,6 @@ #include "downloader.h" #include "util.h" #include "globalconstants.h" -#include "ssl_thread_setup.h" #include "downloadinfo.h" #include "message.h" @@ -28,6 +27,8 @@ #include <htmlcxx/html/Uri.h> #include <termios.h> #include <algorithm> +#include <thread> +#include <mutex> namespace bptime = boost::posix_time; @@ -41,45 +42,20 @@ std::mutex mtx_create_directories; // Mutex for creating directories in Download Downloader::Downloader(Config &conf) { - ssl_thread_setup(); this->config = conf; if (config.bLoginHTTP && boost::filesystem::exists(config.sCookiePath)) if (!boost::filesystem::remove(config.sCookiePath)) std::cerr << "Failed to delete " << config.sCookiePath << std::endl; -} - -Downloader::~Downloader() -{ - if (config.bReport) - if (this->report_ofs) - this->report_ofs.close(); - delete progressbar; - delete gogAPI; - delete gogWebsite; - curl_easy_cleanup(curlhandle); - curl_global_cleanup(); - ssl_thread_cleanup(); - // Make sure that cookie file is only readable/writable by owner - if (!config.bRespectUmask) - Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write); -} - -/* Initialize the downloader - returns 0 if successful - returns 1 if failed -*/ -int Downloader::init() -{ this->resume_position = 0; this->retries = 0; // Initialize curl and set curl options - curl_global_init(CURL_GLOBAL_ALL); curlhandle = curl_easy_init(); curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, config.sVersionString.c_str()); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, config.iTimeout); curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true); curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, config.bVerifyPeer); @@ -99,7 +75,6 @@ int Downloader::init() // Create new GOG website handle gogWebsite = new Website(config); - bool bWebsiteIsLoggedIn = gogWebsite->IsLoggedIn(); // Create new API handle and set curl options for the API gogAPI = new API(config.sToken, config.sSecret); @@ -109,12 +84,55 @@ int Downloader::init() if (!config.sCACertPath.empty()) gogAPI->curlSetOpt(CURLOPT_CAINFO, config.sCACertPath.c_str()); + gogAPI->init(); + progressbar = new ProgressBar(config.bUnicode, config.bColor); +} - bool bInitOK = gogAPI->init(); // Initialize the API - if (!bInitOK || !bWebsiteIsLoggedIn || config.bLoginHTTP || config.bLoginAPI) - return 1; +Downloader::~Downloader() +{ + if (config.bReport) + if (this->report_ofs) + this->report_ofs.close(); + delete progressbar; + delete gogAPI; + delete gogWebsite; + curl_easy_cleanup(curlhandle); + // Make sure that cookie file is only readable/writable by owner + if (!config.bRespectUmask) + Util::setFilePermissions(config.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write); +} + +/* Login check + returns false if not logged in + returns true if logged in +*/ +bool Downloader::isLoggedIn() +{ + bool bIsLoggedIn = false; + config.bLoginAPI = false; + config.bLoginHTTP = false; + + bool bWebsiteIsLoggedIn = gogWebsite->IsLoggedIn(); + if (!bWebsiteIsLoggedIn) + config.bLoginHTTP = true; + + bool bIsLoggedInAPI = gogAPI->isLoggedIn(); + if (!bIsLoggedInAPI) + config.bLoginAPI = true; + + if (bIsLoggedInAPI && bWebsiteIsLoggedIn) + bIsLoggedIn = true; + return bIsLoggedIn; +} + +/* Initialize the downloader + returns 0 if failed + returns 1 if successful +*/ +int Downloader::init() +{ if (!config.sGameHasDLCList.empty()) { if (config.gamehasdlc.empty()) @@ -126,25 +144,18 @@ int Downloader::init() } gogWebsite->setConfig(config); // Update config for website handle - if (config.bCover && config.bDownload && !config.bUpdateCheck) - coverXML = this->getResponse(config.sCoverList); - - // updateCheck() calls getGameList() if needed - // getGameList() is not needed when using cache unless we want to list games from account - if ( !config.bUpdateCheck && (!config.bUseCache || (config.bUseCache && config.bList)) && config.sFileIdString.empty() && !config.bShowWishlist ) - this->getGameList(); - if (config.bReport && (config.bDownload || config.bRepair)) { this->report_ofs.open(config.sReportFilePath); if (!this->report_ofs) { + config.bReport = false; std::cerr << "Failed to create " << config.sReportFilePath << std::endl; - return 1; + return 0; } } - return 0; + return 1; } /* Login @@ -182,6 +193,11 @@ int Downloader::login() // Login to website if (config.bLoginHTTP) { + // Delete old cookies + if (boost::filesystem::exists(config.sCookiePath)) + if (!boost::filesystem::remove(config.sCookiePath)) + std::cerr << "Failed to delete " << config.sCookiePath << std::endl; + if (!gogWebsite->Login(email, password)) { std::cerr << "HTTP: Login failed" << std::endl; @@ -307,6 +323,9 @@ int Downloader::getGameDetails() } } + if (gameItems.empty()) + this->getGameList(); + if (!gameItems.empty()) { for (unsigned int i = 0; i < gameItems.size(); ++i) @@ -315,11 +334,7 @@ int Downloader::getGameDetails() } // Create threads - unsigned int threads = config.iThreads; - if (gameItemQueue.size() < config.iThreads) - { - threads = gameItemQueue.size(); - } + unsigned int threads = std::min(config.iThreads, static_cast<unsigned int>(gameItemQueue.size())); std::vector<std::thread> vThreads; for (unsigned int i = 0; i < threads; ++i) { @@ -577,9 +592,19 @@ int Downloader::listGames() } else { // List game names + if (gameItems.empty()) + this->getGameList(); + for (unsigned int i = 0; i < gameItems.size(); ++i) { - std::cout << gameItems[i].name << std::endl; + std::string gamename = gameItems[i].name; + if (gameItems[i].updates > 0) + { + gamename += " [" + std::to_string(gameItems[i].updates) + "]"; + if (config.bColor) + gamename = "\033[32m" + gamename + "\033[0m"; + } + std::cout << gamename << std::endl; for (unsigned int j = 0; j < gameItems[i].dlcnames.size(); ++j) std::cout << "+> " << gameItems[i].dlcnames[j] << std::endl; } @@ -877,6 +902,9 @@ void Downloader::download() if (this->games.empty()) this->getGameDetails(); + if (config.bCover && !config.bUpdateCheck) + coverXML = this->getResponse(config.sCoverList); + for (unsigned int i = 0; i < games.size(); ++i) { if (config.bSaveSerials && !games[i].serials.empty()) @@ -981,24 +1009,30 @@ void Downloader::download() } } - // Create download threads - std::vector<std::thread> vThreads; - for (unsigned int i = 0; i < config.iThreads; ++i) + if (!dlQueue.empty()) { - DownloadInfo dlInfo; - dlInfo.setStatus(DLSTATUS_NOTSTARTED); - vDownloadInfo.push_back(dlInfo); - vThreads.push_back(std::thread(Downloader::processDownloadQueue, this->config, i)); - } + // Limit thread count to number of items in download queue + unsigned int iThreads = std::min(config.iThreads, static_cast<unsigned int>(dlQueue.size())); - this->printProgress(); + // Create download threads + std::vector<std::thread> vThreads; + for (unsigned int i = 0; i < iThreads; ++i) + { + DownloadInfo dlInfo; + dlInfo.setStatus(DLSTATUS_NOTSTARTED); + vDownloadInfo.push_back(dlInfo); + vThreads.push_back(std::thread(Downloader::processDownloadQueue, this->config, i)); + } + + this->printProgress(); - // Join threads - for (unsigned int i = 0; i < vThreads.size(); ++i) - vThreads[i].join(); + // Join threads + for (unsigned int i = 0; i < vThreads.size(); ++i) + vThreads[i].join(); - vThreads.clear(); - vDownloadInfo.clear(); + vThreads.clear(); + vDownloadInfo.clear(); + } // Create xml data for all files in the queue if (!createXMLQueue.empty()) @@ -1706,8 +1740,7 @@ int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t d // Create progressbar double fraction = starting ? 0.0 : static_cast<double>(dlnow) / static_cast<double>(dltotal); - // assuming that config is provided. - printf("\033[0K\r%3.0f%% ", fraction * 100); + std::cout << Util::formattedString("\033[0K\r%3.0f%% ", fraction * 100); // Download rate unit conversion std::string rate_unit; @@ -1721,9 +1754,8 @@ int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t d rate /= 1024; rate_unit = "kB/s"; } - char status_text[200]; // We're probably never going to go as high as 200 characters but it's better to use too big number here than too small - sprintf(status_text, " %0.2f/%0.2fMB @ %0.2f%s ETA: %s\r", static_cast<double>(dlnow)/1024/1024, static_cast<double>(dltotal)/1024/1024, rate, rate_unit.c_str(), eta_ss.str().c_str()); - int status_text_length = strlen(status_text) + 6; + std::string status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s\r", static_cast<double>(dlnow)/1024/1024, static_cast<double>(dltotal)/1024/1024, rate, rate_unit.c_str(), eta_ss.str().c_str()); + int status_text_length = status_text.length() + 6; if ((status_text_length + bar_length) > iTermWidth) bar_length -= (status_text_length + bar_length) - iTermWidth; @@ -2059,276 +2091,88 @@ void Downloader::checkStatus() if (this->games.empty()) this->getGameDetails(); + // Create a vector containing all game files + std::vector<gameFile> vGameFiles; for (unsigned int i = 0; i < games.size(); ++i) { - if (config.bInstallers) - { - for (unsigned int j = 0; j < games[i].installers.size(); ++j) - { - boost::filesystem::path filepath = games[i].installers[j].getFilepath(); - - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string remoteHash; - std::string localHash; - bool bHashOK = true; // assume hash OK - uintmax_t filesize; - - localHash = this->getLocalFileHash(filepath.string(), games[i].gamename); - remoteHash = this->getRemoteFileHash(games[i].gamename, games[i].installers[j].id); - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - - if (remoteHash != localHash) - bHashOK = false; - else - { - // Check for incomplete file by comparing the filesizes - // Remote hash was saved but download was incomplete and therefore getLocalFileHash returned the same as getRemoteFileHash - uintmax_t filesize_xml = 0; - boost::filesystem::path path = filepath; - boost::filesystem::path local_xml_file; - if (!games[i].gamename.empty()) - local_xml_file = config.sXMLDirectory + "/" + games[i].gamename + "/" + path.filename().string() + ".xml"; - else - local_xml_file = config.sXMLDirectory + "/" + path.filename().string() + ".xml"; - - if (boost::filesystem::exists(local_xml_file)) - { - tinyxml2::XMLDocument local_xml; - local_xml.LoadFile(local_xml_file.string().c_str()); - tinyxml2::XMLElement *fileElemLocal = local_xml.FirstChildElement("file"); - if (fileElemLocal) - { - std::string filesize_xml_str = fileElemLocal->Attribute("total_size"); - filesize_xml = std::stoull(filesize_xml_str); - } - } - - if (filesize_xml > 0 && filesize_xml != filesize) - { - localHash = Util::getFileHash(path.string(), RHASH_MD5); - std::cout << "FS " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - continue; - } - } + std::vector<gameFile> vec = games[i].getGameFileVector(); + vGameFiles.insert(std::end(vGameFiles), std::begin(vec), std::end(vec)); + } - std::cout << (bHashOK ? "OK " : "MD5 ") << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else - { - std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl; - } - } - } + for (unsigned int i = 0; i < vGameFiles.size(); ++i) + { + unsigned int type = vGameFiles[i].type; + if (!config.bDLC && (type & GFTYPE_DLC)) + continue; + if (!config.bInstallers && (type & GFTYPE_INSTALLER)) + continue; + if (!config.bExtras && (type & GFTYPE_EXTRA)) + continue; + if (!config.bPatches && (type & GFTYPE_PATCH)) + continue; + if (!config.bLanguagePacks && (type & GFTYPE_LANGPACK)) + continue; - if (config.bExtras) - { - for (unsigned int j = 0; j < games[i].extras.size(); ++j) - { - boost::filesystem::path filepath = games[i].extras[j].getFilepath(); + boost::filesystem::path filepath = vGameFiles[i].getFilepath(); - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename); - uintmax_t filesize; + if (config.blacklist.isBlacklisted(filepath.native())) + continue; - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else - { - std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl; - } - } - } + std::string gamename = vGameFiles[i].gamename; + std::string id = vGameFiles[i].id; - if (config.bPatches) + if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) { - for (unsigned int j = 0; j < games[i].patches.size(); ++j) - { - boost::filesystem::path filepath = games[i].patches[j].getFilepath(); + std::string remoteHash; + bool bHashOK = true; // assume hash OK + uintmax_t filesize = boost::filesystem::file_size(filepath); - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename); - uintmax_t filesize; - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else - { - std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl; - } - } - } + // GOG only provides xml data for installers, patches and language packs + if (type & (GFTYPE_INSTALLER | GFTYPE_PATCH | GFTYPE_LANGPACK)) + remoteHash = this->getRemoteFileHash(gamename, id); + std::string localHash = this->getLocalFileHash(filepath.string(), gamename); - if (config.bLanguagePacks) - { - for (unsigned int j = 0; j < games[i].languagepacks.size(); ++j) + if (!remoteHash.empty()) { - boost::filesystem::path filepath = games[i].languagepacks[j].getFilepath(); - - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename); - uintmax_t filesize; - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } + if (remoteHash != localHash) + bHashOK = false; else { - std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl; - } - } - } - - if (config.bDLC) - { - for (unsigned int j = 0; j < games[i].dlcs.size(); ++j) - { - if (config.bInstallers) - { - for (unsigned int k = 0; k < games[i].dlcs[j].installers.size(); ++k) - { - boost::filesystem::path filepath = games[i].dlcs[j].installers[k].getFilepath(); - - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string remoteHash; - std::string localHash; - bool bHashOK = true; // assume hash OK - uintmax_t filesize; - - localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename); - remoteHash = this->getRemoteFileHash(games[i].dlcs[j].gamename, games[i].dlcs[j].installers[k].id); - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - - if (remoteHash != localHash) - bHashOK = false; - else - { - // Check for incomplete file by comparing the filesizes - // Remote hash was saved but download was incomplete and therefore getLocalFileHash returned the same as getRemoteFileHash - uintmax_t filesize_xml = 0; - boost::filesystem::path path = filepath; - boost::filesystem::path local_xml_file; - if (!games[i].dlcs[j].gamename.empty()) - local_xml_file = config.sXMLDirectory + "/" + games[i].dlcs[j].gamename + "/" + path.filename().string() + ".xml"; - else - local_xml_file = config.sXMLDirectory + "/" + path.filename().string() + ".xml"; - - if (boost::filesystem::exists(local_xml_file)) - { - tinyxml2::XMLDocument local_xml; - local_xml.LoadFile(local_xml_file.string().c_str()); - tinyxml2::XMLElement *fileElemLocal = local_xml.FirstChildElement("file"); - if (fileElemLocal) - { - std::string filesize_xml_str = fileElemLocal->Attribute("total_size"); - filesize_xml = std::stoull(filesize_xml_str); - } - } - - if (filesize_xml > 0 && filesize_xml != filesize) - { - localHash = Util::getFileHash(path.string(), RHASH_MD5); - std::cout << "FS " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - continue; - } - } - - std::cout << (bHashOK ? "OK " : "MD5 ") << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else - { - std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl; - } - } - } - - if (config.bPatches) - { - for (unsigned int k = 0; k < games[i].dlcs[j].patches.size(); ++k) - { - boost::filesystem::path filepath = games[i].dlcs[j].patches[k].getFilepath(); - - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename); - uintmax_t filesize; - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - std::cout << "OK " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else - { - std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl; - } - } - } + // Check for incomplete file by comparing the filesizes + // Remote hash was saved but download was incomplete and therefore getLocalFileHash returned the same as getRemoteFileHash + uintmax_t filesize_xml = 0; + boost::filesystem::path path = filepath; + boost::filesystem::path local_xml_file; + if (!gamename.empty()) + local_xml_file = config.sXMLDirectory + "/" + gamename + "/" + path.filename().string() + ".xml"; + else + local_xml_file = config.sXMLDirectory + "/" + path.filename().string() + ".xml"; - if (config.bExtras) - { - for (unsigned int k = 0; k < games[i].dlcs[j].extras.size(); ++k) + if (boost::filesystem::exists(local_xml_file)) { - boost::filesystem::path filepath = games[i].dlcs[j].extras[k].getFilepath(); - - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename); - uintmax_t filesize; - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - std::cout << "OK " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else + tinyxml2::XMLDocument local_xml; + local_xml.LoadFile(local_xml_file.string().c_str()); + tinyxml2::XMLElement *fileElemLocal = local_xml.FirstChildElement("file"); + if (fileElemLocal) { - std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl; + std::string filesize_xml_str = fileElemLocal->Attribute("total_size"); + filesize_xml = std::stoull(filesize_xml_str); } } - } - if (config.bLanguagePacks) - { - for (unsigned int k = 0; k < games[i].dlcs[j].languagepacks.size(); ++k) + if (filesize_xml > 0 && filesize_xml != filesize) { - boost::filesystem::path filepath = games[i].dlcs[j].languagepacks[k].getFilepath(); - - if (config.blacklist.isBlacklisted(filepath.native())) - continue; - std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename); - uintmax_t filesize; - - if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) - { - filesize = boost::filesystem::file_size(filepath); - std::cout << "OK " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; - } - else - { - std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl; - } + localHash = Util::getFileHash(path.string(), RHASH_MD5); + std::cout << "FS " << gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; + continue; } } } + std::cout << (bHashOK ? "OK " : "MD5 ") << gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl; + } + else + { + std::cout << "ND " << gamename << " " << filepath.filename().string() << std::endl; } } @@ -2805,6 +2649,7 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) curl_easy_setopt(dlhandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(dlhandle, CURLOPT_USERAGENT, conf.sVersionString.c_str()); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(dlhandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(dlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout); curl_easy_setopt(dlhandle, CURLOPT_FAILONERROR, true); @@ -2979,13 +2824,13 @@ void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) // Get download url std::string url; - if (gf.type == GFTYPE_INSTALLER) + if (gf.type & GFTYPE_INSTALLER) url = api->getInstallerLink(gf.gamename, gf.id); - else if (gf.type == GFTYPE_PATCH) + else if (gf.type & GFTYPE_PATCH) url = api->getPatchLink(gf.gamename, gf.id); - else if (gf.type == GFTYPE_LANGPACK) + else if (gf.type & GFTYPE_LANGPACK) url = api->getLanguagePackLink(gf.gamename, gf.id); - else if (gf.type == GFTYPE_EXTRA) + else if (gf.type & GFTYPE_EXTRA) url = api->getExtraLink(gf.gamename, gf.id); else url = api->getExtraLink(gf.gamename, gf.id); // assume extra if type didn't match any of the others @@ -3211,9 +3056,8 @@ void Downloader::printProgress() bool starting = ((0 == progress_info.dlnow) && (0 == progress_info.dltotal)); double fraction = starting ? 0.0 : static_cast<double>(progress_info.dlnow) / static_cast<double>(progress_info.dltotal); - char progress_percentage_text[200]; - sprintf(progress_percentage_text, "%3.0f%% ", fraction * 100); - int progress_percentage_text_length = strlen(progress_percentage_text) + 1; + std::string progress_percentage_text = Util::formattedString("%3.0f%% ", fraction * 100); + int progress_percentage_text_length = progress_percentage_text.length() + 1; bptime::time_duration eta(bptime::seconds((long)((progress_info.dltotal - progress_info.dlnow) / progress_info.rate))); std::stringstream eta_ss; @@ -3252,9 +3096,8 @@ void Downloader::printProgress() rate_unit = "kB/s"; } - char progress_status_text[200]; // We're probably never going to go as high as 200 characters but it's better to use too big number here than too small - sprintf(progress_status_text, " %0.2f/%0.2fMB @ %0.2f%s ETA: %s", static_cast<double>(progress_info.dlnow)/1024/1024, static_cast<double>(progress_info.dltotal)/1024/1024, progress_info.rate, rate_unit.c_str(), eta_ss.str().c_str()); - int status_text_length = strlen(progress_status_text) + 1; + std::string progress_status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s", static_cast<double>(progress_info.dlnow)/1024/1024, static_cast<double>(progress_info.dltotal)/1024/1024, progress_info.rate, rate_unit.c_str(), eta_ss.str().c_str()); + int status_text_length = progress_status_text.length() + 1; if ((status_text_length + progress_percentage_text_length + bar_length) > iTermWidth) bar_length -= (status_text_length + progress_percentage_text_length + bar_length) - iTermWidth; @@ -3264,7 +3107,7 @@ void Downloader::printProgress() if (bar_length >= min_bar_length) progress_bar_text = bar.createBarString(bar_length, fraction); - progress_text = std::string(progress_percentage_text) + progress_bar_text + std::string(progress_status_text); + progress_text = progress_percentage_text + progress_bar_text + progress_status_text; std::string filename_text = "#" + std::to_string(i) + " " + filename; Util::shortenStringToTerminalWidth(filename_text); @@ -3518,6 +3361,16 @@ void Downloader::getGameDetailsThread(Config config, const unsigned int& tid) } } + // Add DLC type to all DLC files + for (unsigned int a = 0; a < dlc.installers.size(); ++a) + dlc.installers[a].type |= GFTYPE_DLC; + for (unsigned int a = 0; a < dlc.extras.size(); ++a) + dlc.extras[a].type |= GFTYPE_DLC; + for (unsigned int a = 0; a < dlc.patches.size(); ++a) + dlc.patches[a].type |= GFTYPE_DLC; + for (unsigned int a = 0; a < dlc.languagepacks.size(); ++a) + dlc.languagepacks[a].type |= GFTYPE_DLC; + game.dlcs.push_back(dlc); } } diff --git a/src/gamedetails.cpp b/src/gamedetails.cpp index d3c7530..794ca70 100644 --- a/src/gamedetails.cpp +++ b/src/gamedetails.cpp @@ -181,3 +181,25 @@ std::string gameDetails::getChangelogFilepath() { return this->changelogFilepath; } + +// Return vector containing all game files +std::vector<gameFile> gameDetails::getGameFileVector() +{ + std::vector<gameFile> vGameFiles; + + vGameFiles.insert(std::end(vGameFiles), std::begin(installers), std::end(installers)); + vGameFiles.insert(std::end(vGameFiles), std::begin(patches), std::end(patches)); + vGameFiles.insert(std::end(vGameFiles), std::begin(extras), std::end(extras)); + vGameFiles.insert(std::end(vGameFiles), std::begin(languagepacks), std::end(languagepacks)); + + if (!dlcs.empty()) + { + for (unsigned int i = 0; i < dlcs.size(); ++i) + { + std::vector<gameFile> vGameFilesDLC = dlcs[i].getGameFileVector(); + vGameFiles.insert(std::end(vGameFiles), std::begin(vGameFilesDLC), std::end(vGameFilesDLC)); + } + } + + return vGameFiles; +} diff --git a/src/util.cpp b/src/util.cpp index f88f26f..e90392d 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -379,9 +379,16 @@ void Util::setFilePermissions(const boost::filesystem::path& path, const boost:: int Util::getTerminalWidth() { - struct winsize w; - ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); - return static_cast<int>(w.ws_col); + int width; + if(isatty(STDOUT_FILENO)) + { + struct winsize w; + ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); + width = static_cast<int>(w.ws_col); + } + else + width = 10000;//Something sufficiently big + return width; } diff --git a/src/website.cpp b/src/website.cpp index 0769968..0803432 100644 --- a/src/website.cpp +++ b/src/website.cpp @@ -19,6 +19,7 @@ Website::Website(Config &conf) curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, config.sVersionString.c_str()); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); + curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, config.iTimeout); curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true); curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, config.sCookiePath.c_str()); @@ -162,6 +163,7 @@ std::vector<gameItem> Website::getGames() gameItem game; game.name = product["slug"].asString(); game.id = product["id"].isInt() ? std::to_string(product["id"].asInt()) : product["id"].asString(); + game.updates = product["updates"].isInt() ? product["updates"].asInt() : std::stoi(product["updates"].asString()); unsigned int platform = 0; if (product["worksOn"]["Windows"].asBool()) -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/lgogdownloader.git _______________________________________________ Pkg-games-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-games-commits
