Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package libzypp for openSUSE:Factory checked in at 2022-10-15 16:35:01 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/libzypp (Old) and /work/SRC/openSUSE:Factory/.libzypp.new.2275 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "libzypp" Sat Oct 15 16:35:01 2022 rev:464 rq:1010957 version:17.31.3 Changes: -------- --- /work/SRC/openSUSE:Factory/libzypp/libzypp.changes 2022-10-10 18:44:50.530904768 +0200 +++ /work/SRC/openSUSE:Factory/.libzypp.new.2275/libzypp.changes 2022-10-15 16:36:19.785990821 +0200 @@ -1,0 +2,27 @@ +Fri Oct 14 14:09:51 CEST 2022 - m...@suse.de + +- Improve download of optional files (fixes #416) +- Do not use geoip rewrites if the repo has explicit country + settings. +- Implement geoIP feature for zypp. + This patch adds a feature to rewrite request URLs to the repo + servers by querying a geoIP file from download.opensuse.org. This + file can return a redirection target depending on the clients IP + adress, this way we can directly contact a local mirror of d.o.o + instead. The redir target stays valid for 24hrs. + This feature can be disabled in zypp.conf by setting + 'download.use_geoip_mirror = false'. +- Use a dynamic fallback for BLKSIZE in downloads. + When not receiving a blocklist via metalink file from the server + MediaMultiCurl used to fallback to a fixed, relatively small + BLKSIZE. This patch changes the fallback into a dynamic value + based on the filesize using a similar metric as the MirrorCache + implementation on the server side. +- Skip media.1/media download for http repo status calc. + This patch allows zypp to skip a extra media.1/media download to + calculate if a repository needs to be refreshed. This + optimisation only takes place if the repo does specify only + downloading base urls. +- version 17.31.3 (22) + +------------------------------------------------------------------- Old: ---- libzypp-17.31.2.tar.bz2 New: ---- libzypp-17.31.3.tar.bz2 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ libzypp.spec ++++++ --- /var/tmp/diff_new_pack.RvlBcU/_old 2022-10-15 16:36:20.597992772 +0200 +++ /var/tmp/diff_new_pack.RvlBcU/_new 2022-10-15 16:36:20.601992782 +0200 @@ -43,7 +43,7 @@ %bcond_with enable_preview_single_rpmtrans_as_default_for_zypper Name: libzypp -Version: 17.31.2 +Version: 17.31.3 Release: 0 License: GPL-2.0-or-later URL: https://github.com/openSUSE/libzypp ++++++ libzypp-17.31.2.tar.bz2 -> libzypp-17.31.3.tar.bz2 ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/VERSION.cmake new/libzypp-17.31.3/VERSION.cmake --- old/libzypp-17.31.2/VERSION.cmake 2022-10-05 18:39:29.000000000 +0200 +++ new/libzypp-17.31.3/VERSION.cmake 2022-10-14 15:34:44.000000000 +0200 @@ -61,8 +61,8 @@ SET(LIBZYPP_MAJOR "17") SET(LIBZYPP_COMPATMINOR "22") SET(LIBZYPP_MINOR "31") -SET(LIBZYPP_PATCH "2") +SET(LIBZYPP_PATCH "3") # -# LAST RELEASED: 17.31.2 (22) +# LAST RELEASED: 17.31.3 (22) # (The number in parenthesis is LIBZYPP_COMPATMINOR) #======= diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/package/libzypp.changes new/libzypp-17.31.3/package/libzypp.changes --- old/libzypp-17.31.2/package/libzypp.changes 2022-10-05 18:39:29.000000000 +0200 +++ new/libzypp-17.31.3/package/libzypp.changes 2022-10-14 15:34:44.000000000 +0200 @@ -1,4 +1,31 @@ ------------------------------------------------------------------- +Fri Oct 14 14:09:51 CEST 2022 - m...@suse.de + +- Improve download of optional files (fixes #416) +- Do not use geoip rewrites if the repo has explicit country + settings. +- Implement geoIP feature for zypp. + This patch adds a feature to rewrite request URLs to the repo + servers by querying a geoIP file from download.opensuse.org. This + file can return a redirection target depending on the clients IP + adress, this way we can directly contact a local mirror of d.o.o + instead. The redir target stays valid for 24hrs. + This feature can be disabled in zypp.conf by setting + 'download.use_geoip_mirror = false'. +- Use a dynamic fallback for BLKSIZE in downloads. + When not receiving a blocklist via metalink file from the server + MediaMultiCurl used to fallback to a fixed, relatively small + BLKSIZE. This patch changes the fallback into a dynamic value + based on the filesize using a similar metric as the MirrorCache + implementation on the server side. +- Skip media.1/media download for http repo status calc. + This patch allows zypp to skip a extra media.1/media download to + calculate if a repository needs to be refreshed. This + optimisation only takes place if the repo does specify only + downloading base urls. +- version 17.31.3 (22) + +------------------------------------------------------------------- Wed Oct 5 18:02:15 CEST 2022 - m...@suse.de - Resolver: Fix missing --[no]-recommends initialization in diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/po/it.po new/libzypp-17.31.3/po/it.po --- old/libzypp-17.31.2/po/it.po 2022-07-07 14:19:44.000000000 +0200 +++ new/libzypp-17.31.3/po/it.po 2022-10-14 15:34:44.000000000 +0200 @@ -15,8 +15,8 @@ "Project-Id-Version: zypp\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2022-03-14 15:21+0100\n" -"PO-Revision-Date: 2022-03-04 18:12+0000\n" -"Last-Translator: Davide Aiello <davide.aie...@novilingulists.com>\n" +"PO-Revision-Date: 2022-10-13 23:13+0000\n" +"Last-Translator: Paolo Za <zapa...@email.it>\n" "Language-Team: Italian <https://l10n.opensuse.org/projects/libzypp/master/it/"; ">\n" "Language: it\n" @@ -4608,11 +4608,11 @@ #: zypp/target/rpm/RpmDb.cc:1271 msgid "Package header is not signed!" -msgstr "" +msgstr "L'intestazione del pacchetto non ?? firmata!" #: zypp/target/rpm/RpmDb.cc:1273 msgid "Package payload is not signed!" -msgstr "" +msgstr "Il contenuto del pacchetto non ?? firmato!" #. Translator: %s = name of an rpm package. A list of diffs follows #. this message. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/CMakeLists.txt new/libzypp-17.31.3/zypp/CMakeLists.txt --- old/libzypp-17.31.2/zypp/CMakeLists.txt 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/CMakeLists.txt 2022-10-14 15:34:44.000000000 +0200 @@ -248,7 +248,7 @@ media/MediaDIR.cc media/MediaDISK.cc media/MediaCIFS.cc - media/MediaNetworkCommonHandler.h + media/MediaNetworkCommonHandler.cc media/MediaCurl.cc media/MediaMultiCurl.cc media/MediaISO.cc @@ -265,6 +265,7 @@ media/MediaCIFS.h media/MediaCurl.h media/MediaMultiCurl.h + media/MediaNetworkCommonHandler.h media/MediaDIR.h media/MediaDISK.h media/MediaException.h diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/MediaSetAccess.cc new/libzypp-17.31.3/zypp/MediaSetAccess.cc --- old/libzypp-17.31.2/zypp/MediaSetAccess.cc 2022-07-29 11:57:20.000000000 +0200 +++ new/libzypp-17.31.3/zypp/MediaSetAccess.cc 2022-10-14 15:34:44.000000000 +0200 @@ -183,8 +183,7 @@ { try { - if ( doesFileExist( file, media_nr ) ) - return provideFile( OnMediaLocation( file, media_nr ), PROVIDE_NON_INTERACTIVE ); + return provideFile( OnMediaLocation( file, media_nr ).setOptional( true ), PROVIDE_NON_INTERACTIVE ); } catch ( const media::MediaFileNotFoundException & excpt_r ) { ZYPP_CAUGHT( excpt_r ); } @@ -205,7 +204,8 @@ ManagedFile tmpFile = filesystem::TmpFile::asManagedFile(); - Pathname file = access.provideFile( OnMediaLocation(path, 1), options ); + bool optional { options & PROVIDE_NON_INTERACTIVE }; + Pathname file = access.provideFile( OnMediaLocation(path, 1).setOptional( optional ), options ); //prevent the file from being deleted when MediaSetAccess gets out of scope if ( filesystem::hardlinkCopy(file, tmpFile) != 0 ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/RepoInfo.cc new/libzypp-17.31.3/zypp/RepoInfo.cc --- old/libzypp-17.31.2/zypp/RepoInfo.cc 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/RepoInfo.cc 2022-10-14 15:34:44.000000000 +0200 @@ -27,6 +27,7 @@ #include <zypp/Pathname.h> #include <zypp/ZConfig.h> #include <zypp/repo/RepoMirrorList.h> +#include <zypp/repo/SUSEMediaVerifier.h> #include <zypp/ExternalProgram.h> #include <zypp/base/IOStream.h> @@ -1047,6 +1048,24 @@ return str << "GpgCheck::UNKNOWN"; } + bool RepoInfo::requireStatusWithMediaFile () const + { + bool canSkipMediaCheck = std::all_of( baseUrlsBegin(), baseUrlsEnd(), []( const zypp::Url &url ) { return url.schemeIsDownloading(); }); + + const auto &mDataPath = metadataPath(); + if ( canSkipMediaCheck && !mDataPath.empty() ) { + zypp::Pathname mediafile = mDataPath/"media.1/media"; + + zypp::repo::SUSEMediaVerifier lverifier { mediafile }; + if ( lverifier ) { + canSkipMediaCheck = lverifier.totalMedia() == 1; + } + } + + DBG << "Can SKIP media.1/media check for status calc: " << canSkipMediaCheck << " for repo " << *this << std::endl; + return !canSkipMediaCheck; + } + ///////////////////////////////////////////////////////////////// } // namespace zypp /////////////////////////////////////////////////////////////////// diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/RepoInfo.h new/libzypp-17.31.3/zypp/RepoInfo.h --- old/libzypp-17.31.2/zypp/RepoInfo.h 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/RepoInfo.h 2022-10-14 15:34:44.000000000 +0200 @@ -521,6 +521,12 @@ LocaleSet getLicenseLocales( const std::string & name_r ) const; //@} + /** + * Returns true if this repository requires the media.1/media file to be included + * in the metadata status and repo status calculations. + */ + bool requireStatusWithMediaFile () const; + public: /** * Write a human-readable representation of this RepoInfo object diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/RepoManager.cc new/libzypp-17.31.3/zypp/RepoManager.cc --- old/libzypp-17.31.2/zypp/RepoManager.cc 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/RepoManager.cc 2022-10-14 15:34:44.000000000 +0200 @@ -17,6 +17,7 @@ #include <list> #include <map> #include <algorithm> +#include <chrono> #include <solv/solvversion.h> @@ -42,6 +43,7 @@ #include <zypp/parser/RepoFileReader.h> #include <zypp/parser/ServiceFileReader.h> +#include <zypp/parser/xml/Reader.h> #include <zypp/repo/ServiceRepos.h> #include <zypp/repo/yum/Downloader.h> #include <zypp/repo/susetags/Downloader.h> @@ -668,6 +670,8 @@ repo::ServiceType probeService( const Url & url ) const; + void refreshGeoIPData (); + private: void saveService( ServiceInfo & service ) const; @@ -928,7 +932,9 @@ switch ( repokind.toEnum() ) { case RepoType::RPMMD_e : - status = RepoStatus( productdatapath/"repodata/repomd.xml") && RepoStatus( mediarootpath/"media.1/media" ); + status = RepoStatus( productdatapath/"repodata/repomd.xml"); + if ( info.requireStatusWithMediaFile() ) + status = status && RepoStatus( mediarootpath/"media.1/media" ); break; case RepoType::YAST2_e : @@ -997,6 +1003,8 @@ { MIL << "Check if to refresh repo " << info.alias() << " at " << url << " (" << info.type() << ")" << endl; + refreshGeoIPData(); + // first check old (cached) metadata Pathname mediarootpath = rawcache_path_for_repoinfo( _options, info ); filesystem::assert_dir( mediarootpath ); @@ -1120,6 +1128,9 @@ assert_alias(info); assert_urls(info); + // make sure geoIP data is up 2 date + refreshGeoIPData(); + // we will throw this later if no URL checks out fine RepoException rexception( info, PL_("Valid metadata not found at specified URL", "Valid metadata not found at specified URLs", @@ -1265,8 +1276,8 @@ { ProgressData progress(100); progress.sendTo(progressfnc); - - filesystem::recursive_rmdir(rawcache_path_for_repoinfo(_options, info)); + filesystem::recursive_rmdir( ZConfig::instance().geoipCachePath() ); + filesystem::recursive_rmdir( rawcache_path_for_repoinfo(_options, info) ); progress.toMax(); } @@ -2538,6 +2549,134 @@ return repo::ServiceType::NONE; } + void RepoManager::Impl::refreshGeoIPData () + { + try { + + if ( !ZConfig::instance().geoipEnabled() ) { + MIL << "GeoIp disabled via ZConfig, not refreshing the GeoIP information." << std::endl; + return; + } + + // for applications like packageKit that are running very long we remember the last time when we checked + // for geoIP data. We don't want to query this over and over again. + static auto lastCheck = std::chrono::steady_clock::time_point::min(); + if ( lastCheck != std::chrono::steady_clock::time_point::min() + && (std::chrono::steady_clock::now() - lastCheck) < std::chrono::hours(24) ) + return; + + lastCheck = std::chrono::steady_clock::now(); + + const auto &geoIPCache = ZConfig::instance().geoipCachePath(); + + if ( filesystem::assert_dir( geoIPCache ) != 0 ) { + MIL << "Unable to create cache directory for GeoIP." << std::endl; + return; + } + + if ( !PathInfo(geoIPCache).userMayRWX() ) { + MIL << "No access rights for the GeoIP cache directory." << std::endl; + return; + } + + // remove all older cache entries + filesystem::dirForEachExt( geoIPCache, []( const Pathname &dir, const filesystem::DirEntry &entry ){ + if ( entry.type != filesystem::FT_FILE ) + return true; + + PathInfo pi( dir/entry.name ); + auto age = std::chrono::system_clock::now() - std::chrono::system_clock::from_time_t( pi.mtime() ); + if ( age < std::chrono::hours(24) ) + return true; + + MIL << "Removing GeoIP file for " << entry.name << " since it's older than 24hrs." << std::endl; + filesystem::unlink( dir/entry.name ); + return true; + }); + + // go over all configured hostnames + const auto &hosts = ZConfig::instance().geoipHostnames(); + std::for_each( hosts.begin(), hosts.end(), [ & ]( const std::string &hostname ) { + + // do not query files that are still there + if ( zypp::PathInfo( geoIPCache / hostname ).isExist() ) { + MIL << "Skipping GeoIP request for " << hostname << " since a valid cache entry exists." << std::endl; + return; + } + + MIL << "Query GeoIP for " << hostname << std::endl; + + zypp::Url url; + try + { + url.setHost(hostname); + url.setScheme("https"); + } + catch(const zypp::Exception &e ) + { + ZYPP_CAUGHT(e); + MIL << "Ignoring invalid GeoIP hostname: " << hostname << std::endl; + return; + } + + zypp::ManagedFile file; + try { + + // query the file from the server + MediaSetAccess acc( url ); + file = zypp::ManagedFile (acc.provideOptionalFile("/geoip"), filesystem::unlink ); + + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + MIL << "Failed to query GeoIP from hostname: " << hostname << std::endl; + return; + } + if ( !file->empty() ) { + + constexpr auto writeHostToFile = []( const Pathname &fName, const std::string &host ){ + std::ofstream out; + out.open( fName.asString(), std::ios_base::trunc ); + if ( out.is_open() ) { + out << host << std::endl; + } else { + MIL << "Failed to create/open GeoIP cache file " << fName << std::endl; + } + }; + + std::string geoipMirror; + try { + xml::Reader reader( *file ); + if ( reader.seekToNode( 1, "host" ) ) { + const auto &str = reader.nodeText().asString(); + + // make a dummy URL to ensure the hostname is valid + zypp::Url testUrl; + testUrl.setHost(str); + testUrl.setScheme("https"); + + if ( testUrl.isValid() ) { + MIL << "Storing geoIP redirection: " << hostname << " -> " << str << std::endl; + geoipMirror = str; + } + + } else { + MIL << "No host entry or empty file returned for GeoIP, remembering for 24hrs" << std::endl; + } + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + MIL << "Empty or invalid GeoIP file, not requesting again for 24hrs" << std::endl; + } + + writeHostToFile( geoIPCache / hostname, geoipMirror ); + } + }); + + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + MIL << "Failed to query GeoIP data." << std::endl; + } + } + /////////////////////////////////////////////////////////////////// // // CLASS NAME : RepoManager @@ -2697,6 +2836,9 @@ void RepoManager::modifyService( const std::string & oldAlias, const ServiceInfo & service ) { return _pimpl->modifyService( oldAlias, service ); } + void RepoManager::refreshGeoIp () + { return _pimpl->refreshGeoIPData(); } + //////////////////////////////////////////////////////////////////////////// std::ostream & operator<<( std::ostream & str, const RepoManager & obj ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/RepoManager.h new/libzypp-17.31.3/zypp/RepoManager.h --- old/libzypp-17.31.2/zypp/RepoManager.h 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/RepoManager.h 2022-10-14 15:34:44.000000000 +0200 @@ -634,6 +634,12 @@ void modifyService( const ServiceInfo & service ) { modifyService( service.alias(), service ); } + /*! + * Checks with configured geoIP servers ( usually download.opensuse.org ) if there is new geoip data available, caches the results + * in the metadata cache for 24hrs. + */ + void refreshGeoIp (); + private: /** * Functor thats filter RepoInfo by service which it belongs to. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/ZConfig.cc new/libzypp-17.31.3/zypp/ZConfig.cc --- old/libzypp-17.31.2/zypp/ZConfig.cc 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/ZConfig.cc 2022-10-14 15:34:44.000000000 +0200 @@ -415,6 +415,8 @@ , pkgGpgCheck ( indeterminate ) , apply_locks_file ( true ) , pluginsPath ( "/usr/lib/zypp/plugins" ) + , geoipEnabled ( true ) + , geoipHosts { "download.opensuse.org" } { MIL << "libzypp: " LIBZYPP_VERSION_STRING << endl; if ( PathInfo(_parsedZyppConf).isExist() ) @@ -518,6 +520,9 @@ { download_mediaMountdir.restoreToDefault( Pathname(value) ); } + else if ( entry == "download.use_geoip_mirror") { + geoipEnabled = str::strToBool( value, geoipEnabled ); + } else if ( entry == "commit.downloadMode" ) { commit_downloadMode.set( deserializeDownloadMode( value ) ); @@ -712,6 +717,10 @@ Option<Pathname> pluginsPath; + bool geoipEnabled; + + std::vector<std::string> geoipHosts; + /* Other config singleton instances */ MediaConfig &_mediaConf = MediaConfig::instance(); @@ -1024,6 +1033,18 @@ Pathname ZConfig::needrebootPath() const { return configPath()/"needreboot.d"; } + void ZConfig::setGeoipEnabled( bool enable ) + { _pimpl->geoipEnabled = enable; } + + bool ZConfig::geoipEnabled () const + { return _pimpl->geoipEnabled; } + + Pathname ZConfig::geoipCachePath() const + { return builtinRepoCachePath()/"geoip.d"; } + + const std::vector<std::string> ZConfig::geoipHostnames () const + { return _pimpl->geoipHosts; } + Pathname ZConfig::varsPath() const { return ( _pimpl->cfg_vars_path.empty() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/ZConfig.h new/libzypp-17.31.3/zypp/ZConfig.h --- old/libzypp-17.31.2/zypp/ZConfig.h 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/ZConfig.h 2022-10-14 15:34:44.000000000 +0200 @@ -224,6 +224,28 @@ Pathname needrebootPath() const; /** + * Enables or disables the use of the geoip feature of download.opensuse.org + */ + void setGeoipEnabled( bool enable = true ); + + /** + * Returns true if zypp should use the geoip feature of download.opensuse.org + */ + bool geoipEnabled () const; + + /** + * Path where the geoip caches are kept (/var/cache/zypp/geoip) + */ + Pathname geoipCachePath() const; + + /** + * All hostnames we want to rewrite using the geoip feature. The \ref RepoManager + * will try to query each hostname via: https://hostname/geoip to receive a redirection + * target for requests to the given hostname. The geoip targets are cached in \ref geoipCachePath. + */ + const std::vector<std::string> geoipHostnames () const; + + /** * Path containing custom repo variable definitions (configPath()/vars.d). * \see \ref zypp::repo::RepoVarExpand Repo variable expander * \ingroup g_ZC_CONFIGFILES diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/media/MediaCurl.cc new/libzypp-17.31.3/zypp/media/MediaCurl.cc --- old/libzypp-17.31.2/zypp/media/MediaCurl.cc 2022-07-08 12:42:06.000000000 +0200 +++ new/libzypp-17.31.3/zypp/media/MediaCurl.cc 2022-10-14 15:34:44.000000000 +0200 @@ -42,6 +42,83 @@ namespace internal { using namespace zypp; + /// Optional files will send no report until data are actually received (we know it exists). + struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport> + { + OptionalDownloadProgressReport( bool isOptional=false ) + : _oldRec { Distributor::instance().getReceiver() } + , _isOptional { isOptional } + { connect(); } + + ~OptionalDownloadProgressReport() + { if ( _oldRec ) Distributor::instance().setReceiver( *_oldRec ); else Distributor::instance().noReceiver(); } + + void reportbegin() override + { if ( _oldRec ) _oldRec->reportbegin(); } + + void reportend() override + { if ( _oldRec ) _oldRec->reportend(); } + + void report( const UserData & userData_r = UserData() ) override + { if ( _oldRec ) _oldRec->report( userData_r ); } + + + void start( const Url & file_r, Pathname localfile_r ) override + { + if ( not _oldRec ) return; + if ( _isOptional ) { + // delay start until first data are received. + _startFile = file_r; + _startLocalfile = std::move(localfile_r); + return; + } + _oldRec->start( file_r, localfile_r ); + } + + bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override + { + if ( not _oldRec ) return true; + if ( notStarted() ) { + if ( not ( value_r || dbps_avg_r || dbps_current_r ) ) + return true; + sendStart(); + } + return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r ); + } + + Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override + { + if ( not _oldRec || notStarted() ) return ABORT; + return _oldRec->problem( file_r, error_r, description_r ); + } + + void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override + { + if ( not _oldRec || notStarted() ) return; + _oldRec->finish( file_r, error_r, reason_r ); + } + + private: + // _isOptional also indicates the delayed start + bool notStarted() const + { return _isOptional; } + + void sendStart() + { + if ( _isOptional ) { + // we know _oldRec is valid... + _oldRec->start( std::move(_startFile), std::move(_startLocalfile) ); + _isOptional = false; + } + } + + private: + Receiver *const _oldRec; + bool _isOptional; + Url _startFile; + Pathname _startLocalfile; + }; + struct ProgressData { ProgressData( CURL *_curl, time_t _timeout = 0, const zypp::Url & _url = zypp::Url(), @@ -626,20 +703,6 @@ disconnect(); } -Url MediaCurl::getFileUrl( const Pathname & filename_r ) const -{ - // Simply extend the URLs pathname. An 'absolute' URL path - // is achieved by encoding the leading '/' in an URL path: - // URL: ftp://user@server -> ~user - // URL: ftp://user@server/ -> ~user - // URL: ftp://user@server// -> ~user - // URL: ftp://user@server/%2F -> / - // ^- this '/' is just a separator - Url newurl( _url ); - newurl.setPathName( ( Pathname("./"+_url.getPathName()) / filename_r ).asString().substr(1) ); - return newurl; -} - /////////////////////////////////////////////////////////////////// void MediaCurl::getFile( const OnMediaLocation &file ) const @@ -656,6 +719,8 @@ const auto &filename = srcFile.filename(); + // Optional files will send no report until data are actually received (we know it exists). + OptionalDownloadProgressReport reportfilter( srcFile.optional() ); callback::SendReport<DownloadProgressReport> report; Url fileurl(getFileUrl(filename)); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/media/MediaCurl.h new/libzypp-17.31.3/zypp/media/MediaCurl.h --- old/libzypp-17.31.2/zypp/media/MediaCurl.h 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/media/MediaCurl.h 2022-10-14 15:34:44.000000000 +0200 @@ -110,7 +110,7 @@ }; protected: -// /** Callback sending just an alive trigger to the UI, without stats (e.g. during metalink download). */ + /** Callback sending just an alive trigger to the UI, without stats (e.g. during metalink download). */ static int aliveCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow ); /** Callback reporting download progress. */ static int progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow ); @@ -126,11 +126,6 @@ * \throws MediaCurlSetOptException if there is a problem **/ virtual void setupEasy(); - /** - * concatenate the attach url and the filename to a complete - * download url - **/ - Url getFileUrl(const Pathname & filename) const; /** * Evaluates a curl return code and throws the right MediaException diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/media/MediaHandler.h new/libzypp-17.31.3/zypp/media/MediaHandler.h --- old/libzypp-17.31.2/zypp/media/MediaHandler.h 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/media/MediaHandler.h 2022-10-14 15:34:44.000000000 +0200 @@ -427,7 +427,7 @@ * \throws MediaException * **/ - virtual bool getDoesFileExist( const Pathname & filename ) const = 0; + virtual bool getDoesFileExist( const Pathname & filename ) const = 0; protected: @@ -717,5 +717,3 @@ #endif // ZYPP_MEDIA_MEDIAHANDLERL_H - - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/media/MediaMultiCurl.cc new/libzypp-17.31.3/zypp/media/MediaMultiCurl.cc --- old/libzypp-17.31.2/zypp/media/MediaMultiCurl.cc 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/media/MediaMultiCurl.cc 2022-10-14 15:34:44.000000000 +0200 @@ -121,6 +121,9 @@ void run(std::vector<Url> &urllist); protected: + + static size_t makeBlksize ( size_t filesize ); + friend class multifetchworker; const MediaMultiCurl *_context; @@ -139,6 +142,7 @@ bool _havenewjob; size_t _blkno; + size_t _defaultBlksize = 0; //< The blocksize to use if the metalink file does not specify one off_t _blkoff; size_t _activeworkers; size_t _lookupworkers; @@ -164,7 +168,6 @@ }; constexpr auto MIN_REQ_MIRRS = 4; -constexpr auto BLKSIZE = 131072; constexpr auto MAXURLS = 10; ////////////////////////////////////////////////////////////////////// @@ -707,18 +710,19 @@ MediaBlockList *blklist = _request->_blklist; if (!blklist) { - _blksize = BLKSIZE; + _blksize = _request->_defaultBlksize; if (_request->_filesize != off_t(-1)) - { - if (_request->_blkoff >= _request->_filesize) - { - stealjob(); - return; - } - _blksize = _request->_filesize - _request->_blkoff; - if (_blksize > BLKSIZE) - _blksize = BLKSIZE; - } + { + if (_request->_blkoff >= _request->_filesize) + { + stealjob(); + return; + } + _blksize = _request->_filesize - _request->_blkoff; + if (_blksize > _request->_defaultBlksize) + _blksize = _request->_defaultBlksize; + } + DBG << "No BLOCKLIST falling back to chunk size: " << _request->_defaultBlksize << std::endl; } else { @@ -734,8 +738,10 @@ _request->_blkoff = blk.off; } _blksize = blk.off + blk.size - _request->_blkoff; - if (_blksize > BLKSIZE && !blklist->haveChecksum(_request->_blkno)) - _blksize = BLKSIZE; + if (_blksize > _request->_defaultBlksize && !blklist->haveChecksum(_request->_blkno)) { + DBG << "Block: "<< _request->_blkno << " has no checksum falling back to default blocksize: " << _request->_defaultBlksize << std::endl; + _blksize = _request->_defaultBlksize; + } } _blkno = _request->_blkno; _blkstart = _request->_blkoff; @@ -791,6 +797,7 @@ _report = report; _blklist = blklist; _filesize = filesize; + _defaultBlksize = makeBlksize( filesize ); _multi = multi; _stealing = false; _havenewjob = false; @@ -1156,6 +1163,15 @@ } } +inline size_t multifetchrequest::makeBlksize ( size_t filesize ) +{ + // this case should never happen because we never start a multi download if we do not know the filesize beforehand + if ( filesize == 0 ) return 2 * 1024 * 1024; + else if ( filesize < 2*256*1024 ) return filesize; + else if ( filesize < 8*1024*1024 ) return 256*1024; + else if ( filesize < 256*1024*1024 ) return 1024*1024; + return 4*1024*1024; +} ////////////////////////////////////////////////////////////////////// @@ -1610,4 +1626,3 @@ } // namespace media } // namespace zypp - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/media/MediaNetworkCommonHandler.cc new/libzypp-17.31.3/zypp/media/MediaNetworkCommonHandler.cc --- old/libzypp-17.31.2/zypp/media/MediaNetworkCommonHandler.cc 1970-01-01 01:00:00.000000000 +0100 +++ new/libzypp-17.31.3/zypp/media/MediaNetworkCommonHandler.cc 2022-10-14 15:34:44.000000000 +0200 @@ -0,0 +1,96 @@ +/*---------------------------------------------------------------------\ +| ____ _ __ __ ___ | +| |__ / \ / / . \ . \ | +| / / \ V /| _/ _/ | +| / /__ | | | | | | | +| /_____||_| |_| |_| | +| | +\---------------------------------------------------------------------*/ +/** \file zypp/media/MediaNetworkCommonHandler.cc + * +*/ + +#include "MediaNetworkCommonHandler.h" + +#include <zypp/ZConfig.h> +#include <zypp-core/fs/PathInfo.h> +#include <zypp/base/Logger.h> +#include <zypp-core/base/Regex.h> + +#include <fstream> + +namespace zypp::media +{ + zypp::Url MediaNetworkCommonHandler::findGeoIPRedirect ( const zypp::Url &url ) + { + try { + const auto &conf = ZConfig::instance(); + if ( !conf.geoipEnabled() ) { + MIL << "GeoIp rewrites disabled via ZConfig." << std::endl; + return Url(); + } + + if ( !( url.getQueryParam("COUNTRY").empty() && url.getQueryParam("AVOID_COUNTRY").empty() )) { + MIL << "GeoIp rewrites disabled since the baseurl " << url << " uses an explicit country setting." << std::endl; + return Url(); + } + + const auto &hostname = url.getHost(); + auto geoipFile = conf.geoipCachePath() / hostname ; + if ( PathInfo( geoipFile ).isFile() ) { + + MIL << "Found GeoIP file for host: " << hostname << std::endl; + + std::ifstream in( geoipFile.asString() ); + if (!in.is_open()) { + MIL << "Failed to open GeoIP for host: " << hostname << std::endl; + return Url(); + } + + try { + std::string newHost; + in >> newHost; + + Url newUrl = url; + newUrl.setHost( newHost ); + + MIL << "Found GeoIP rewrite: " << hostname << " -> " << newHost << std::endl; + + return newUrl; + + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + MIL << "No valid GeoIP rewrite target found for " << url << std::endl; + } + } + } catch ( const zypp::Exception &e ) { + ZYPP_CAUGHT(e); + MIL << "Failed to query GeoIP data, url rewriting disabled." << std::endl; + } + + // no rewrite + return Url(); + } + + Url MediaNetworkCommonHandler::getFileUrl( const Pathname & filename_r ) const + { + static const zypp::str::regex invalidRewrites("^.*\\/repomd.xml(.asc|.key)?$|^\\/geoip$"); + + const bool canRedir = _redirTarget.isValid() && !invalidRewrites.matches(filename_r.asString()); + const auto &baseUrl = ( canRedir ) ? _redirTarget : _url; + + if ( canRedir ) + MIL << "Redirecting " << filename_r << " request to geoip location." << std::endl; + + // Simply extend the URLs pathname. An 'absolute' URL path + // is achieved by encoding the leading '/' in an URL path:2 + // URL: ftp://user@server -> ~user + // URL: ftp://user@server/ -> ~user + // URL: ftp://user@server// -> ~user + // URL: ftp://user@server/%2F -> / + // ^- this '/' is just a separator + Url newurl( baseUrl ); + newurl.setPathName( ( Pathname("./"+baseUrl.getPathName()) / filename_r ).asString().substr(1) ); + return newurl; + } +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/media/MediaNetworkCommonHandler.h new/libzypp-17.31.3/zypp/media/MediaNetworkCommonHandler.h --- old/libzypp-17.31.2/zypp/media/MediaNetworkCommonHandler.h 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/media/MediaNetworkCommonHandler.h 2022-10-14 15:34:44.000000000 +0200 @@ -35,6 +35,8 @@ const Pathname & urlpath_below_attachpoint_r, const bool does_download_r ) : MediaHandler( url_r, attach_point_r, urlpath_below_attachpoint_r, does_download_r ) + , _redirTarget( findGeoIPRedirect(url_r) ) + {} public: @@ -42,7 +44,23 @@ { return _settings; } protected: + + /** + * concatenate the attach url and the filename to a complete + * download url + **/ + Url getFileUrl(const Pathname & filename) const; + + + /** + * Rewrites the baseURL to the geoIP target if one is found in the metadata cache, + * otherwise simply returns the url again. + */ + static zypp::Url findGeoIPRedirect ( const zypp::Url &url ); + + protected: mutable TransferSettings _settings; + Url _redirTarget; }; } // namespace media diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/repo/Downloader.cc new/libzypp-17.31.3/zypp/repo/Downloader.cc --- old/libzypp-17.31.2/zypp/repo/Downloader.cc 2022-07-29 11:57:20.000000000 +0200 +++ new/libzypp-17.31.3/zypp/repo/Downloader.cc 2022-10-14 15:34:44.000000000 +0200 @@ -221,6 +221,3 @@ }// ns repo } // ns zypp - - - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/repo/SUSEMediaVerifier.cc new/libzypp-17.31.3/zypp/repo/SUSEMediaVerifier.cc --- old/libzypp-17.31.2/zypp/repo/SUSEMediaVerifier.cc 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/repo/SUSEMediaVerifier.cc 2022-10-14 15:34:44.000000000 +0200 @@ -163,6 +163,12 @@ SEC << smvData << endl; SEC << ref.protocol() << " " << Url::schemeIsVolatile( ref.protocol() ) << endl; + // if we have a downloading URL and can do relaxed verification we do not check the media file again + if ( relaxed && Url::schemeIsDownloading( ref.protocol() ) ) { + DBG << "Skipping verification due to downloading medium" << std::endl; + return ret; + } + Pathname mediaFile { _pimpl->mediaFilePath() }; try { ref.provideFile( OnMediaLocation(mediaFile) ); @@ -201,4 +207,3 @@ } // namespace repo } // namespace zypp - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp/repo/yum/Downloader.cc new/libzypp-17.31.3/zypp/repo/yum/Downloader.cc --- old/libzypp-17.31.2/zypp/repo/yum/Downloader.cc 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp/repo/yum/Downloader.cc 2022-10-14 15:34:44.000000000 +0200 @@ -16,6 +16,7 @@ #include "Downloader.h" #include <zypp/repo/MediaInfoDownloader.h> +#include <zypp/repo/SUSEMediaVerifier.h> #include <zypp-core/base/UserRequestException> #include <zypp/parser/xml/Reader.h> #include <zypp/parser/yum/RepomdFileReader.h> @@ -62,6 +63,9 @@ deltafile = fn; } } + if ( !deltafile.empty() ) + return dir/deltafile; + return deltafile; } } // namespace @@ -200,8 +204,9 @@ RepoStatus Downloader::status( MediaSetAccess & media_r ) { - RepoStatus ret { media_r.provideOptionalFile( repoInfo().path() / "/repodata/repomd.xml" ) }; - if ( !ret.empty() ) // else: mandatory master index is missing + const auto & ri = repoInfo(); + RepoStatus ret { media_r.provideOptionalFile( ri.path() / "/repodata/repomd.xml" ) }; + if ( !ret.empty() && ri.requireStatusWithMediaFile() ) // else: mandatory master index is missing ret = ret && RepoStatus( media_r.provideOptionalFile( "/media.1/media" ) ); // else: mandatory master index is missing -> stay empty return ret; @@ -209,6 +214,3 @@ } // namespace yum } // namespace repo } // namespace zypp - - - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/libzypp-17.31.2/zypp.conf new/libzypp-17.31.3/zypp.conf --- old/libzypp-17.31.2/zypp.conf 2022-07-07 14:19:45.000000000 +0200 +++ new/libzypp-17.31.3/zypp.conf 2022-10-14 15:34:44.000000000 +0200 @@ -258,6 +258,21 @@ ## download.media_mountdir = /var/adm/mount ## +## Whether to use the geoip feature of download.opensuse.org +## +## Valid values: boolean +## Default value: true +## +## The media backend can rewrite download requests to the geographically closest availble mirror. +## Which exact mirror is used will be determined by requesting a "geoip" file from download.opensuse.org +## via a HTTP GET request to https://download.opensuse.org/geoip , the server will use the clients IP to +## determine the closest mirror if available. +## Some specific files are however excluded from this redirection due to security reasons, especially the +## repo metdata index and it's key and checksum files: repomd.xml, repomd.xml.key and repomd.xml.asc +## +## download.use_geoip_mirror = true + +## ## Signature checking (repo metadata and downloaded rpm packages) ## ## boolean gpgcheck (default: on)
