commit: cc533641381eda01900675f3b265d939b44ba2ea Author: Robin H. Johnson <robbat2 <AT> gentoo <DOT> org> AuthorDate: Tue Apr 28 17:27:30 2020 +0000 Commit: Robin H. Johnson <robbat2 <AT> gentoo <DOT> org> CommitDate: Tue Apr 28 17:27:30 2020 +0000 URL: https://gitweb.gentoo.org/proj/gentoo-mirrorstats.git/commit/?id=cc533641
refactor mirror list generation Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org> distfiles_mirrors/get-mirror-list-distfiles.rb | 18 ---------------- .../get-mirror-list-experimental.rb | 18 ---------------- get-mirrors-from-distfiles-xml.rb | 24 ++++++++++++++++++++++ get-mirrors-from-rsync-xml.rb | 24 ++++++++++++++++++++++ mirmon-distfiles.sh | 2 +- mirmon-experimental.sh | 2 +- mirmon-releases.sh | 2 +- mirmon-rsync.sh | 2 +- releases_mirrors/get-mirror-list-releases.rb | 18 ---------------- rsync_mirrors/get-mirror-list-rsync.rb | 18 ---------------- 10 files changed, 52 insertions(+), 76 deletions(-) diff --git a/distfiles_mirrors/get-mirror-list-distfiles.rb b/distfiles_mirrors/get-mirror-list-distfiles.rb deleted file mode 100755 index 0794df3..0000000 --- a/distfiles_mirrors/get-mirror-list-distfiles.rb +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/ruby - -MIRROR_DATA="https://api.gentoo.org/mirrors/distfiles.xml"; - -%w[ rexml/document open-uri ].each {|lib| require lib } - -m = URI.parse(MIRROR_DATA).read -x = REXML::Document.new(m) - -REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| - country = el.attributes['country'] - - el.each_element('mirror/uri/') do |mirror| - puts "#{country.downcase} #{mirror[0].to_s}" - end - -} - diff --git a/experimental_mirrors/get-mirror-list-experimental.rb b/experimental_mirrors/get-mirror-list-experimental.rb deleted file mode 100755 index 0794df3..0000000 --- a/experimental_mirrors/get-mirror-list-experimental.rb +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/ruby - -MIRROR_DATA="https://api.gentoo.org/mirrors/distfiles.xml"; - -%w[ rexml/document open-uri ].each {|lib| require lib } - -m = URI.parse(MIRROR_DATA).read -x = REXML::Document.new(m) - -REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| - country = el.attributes['country'] - - el.each_element('mirror/uri/') do |mirror| - puts "#{country.downcase} #{mirror[0].to_s}" - end - -} - diff --git a/get-mirrors-from-distfiles-xml.rb b/get-mirrors-from-distfiles-xml.rb new file mode 100755 index 0000000..02b9418 --- /dev/null +++ b/get-mirrors-from-distfiles-xml.rb @@ -0,0 +1,24 @@ +#!/usr/bin/ruby +require 'rexml/document' +require 'open-uri' + +MIRROR_DATA="https://api.gentoo.org/mirrors/distfiles.xml"; + +m = URI.parse(MIRROR_DATA).read +x = REXML::Document.new(m) + +def normalize_mirror(xml_elem) + return xml_elem.texts().join(' ').sub(/\/+$/, '') + '/' +end + +def select_mirror(xml_elem) + 1 +end + +REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| + country = el.attributes['country'] + + el.each_element('mirror/uri/') do |uri_elem| + puts "#{country.downcase} #{normalize_mirror(uri_elem)}" if select_mirror(uri_elem) + end +} diff --git a/get-mirrors-from-rsync-xml.rb b/get-mirrors-from-rsync-xml.rb new file mode 100755 index 0000000..100dffc --- /dev/null +++ b/get-mirrors-from-rsync-xml.rb @@ -0,0 +1,24 @@ +#!/usr/bin/ruby +require 'rexml/document' +require 'open-uri' + +MIRROR_DATA="https://api.gentoo.org/mirrors/rsync.xml"; + +m = URI.parse(MIRROR_DATA).read +x = REXML::Document.new(m) + +def normalize_mirror(xml_elem) + return xml_elem.texts().join(' ').sub(/\/+$/, '') + '/' +end + +def select_mirror(xml_elem) + xml_elem.texts().join(' ') =~ /rsync\d+\./ +end + +REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| + country = el.attributes['country'] + + el.each_element('mirror/uri/') do |uri_elem| + puts "#{country.downcase} #{normalize_mirror(uri_elem)}" if select_mirror(uri_elem) + end +} diff --git a/mirmon-distfiles.sh b/mirmon-distfiles.sh index 3e14e87..fedefdc 100755 --- a/mirmon-distfiles.sh +++ b/mirmon-distfiles.sh @@ -3,7 +3,7 @@ cd /var/www/mirrorstats.gentoo.org/gentoo-mirrorstats/distfiles_mirrors # Grab mirrors from the web [[ -d ./var ]] || mkdir ./var -./get-mirror-list-distfiles.rb > ./var/g.mirrors +../get-mirrors-from-distfiles-xml.rb > ./var/g.mirrors # fatal if the state file is NOT present. [[ -e ./var/mirmon.state ]] || touch ./var/mirmon.state # run mirmon diff --git a/mirmon-experimental.sh b/mirmon-experimental.sh index f76a3d3..fae7df6 100755 --- a/mirmon-experimental.sh +++ b/mirmon-experimental.sh @@ -3,7 +3,7 @@ cd /var/www/mirrorstats.gentoo.org/gentoo-mirrorstats/experimental_mirrors # Grab mirrors from the web [[ -d ./var ]] || mkdir ./var -./get-mirror-list-experimental.rb > ./var/g.mirrors +../get-mirrors-from-distfiles-xml.rb > ./var/g.mirrors # fatal if the state file is NOT present. [[ -e ./var/mirmon.state ]] || touch ./var/mirmon.state # run mirmon diff --git a/mirmon-releases.sh b/mirmon-releases.sh index a6bf5fd..6790756 100755 --- a/mirmon-releases.sh +++ b/mirmon-releases.sh @@ -3,7 +3,7 @@ cd /var/www/mirrorstats.gentoo.org/gentoo-mirrorstats/releases_mirrors # Grab mirrors from the web [[ -d ./var ]] || mkdir ./var -./get-mirror-list-releases.rb > ./var/g.mirrors +../get-mirrors-from-distfiles-xml.rb > ./var/g.mirrors # fatal if the state file is NOT present. [[ -e ./var/mirmon.state ]] || touch ./var/mirmon.state # run mirmon diff --git a/mirmon-rsync.sh b/mirmon-rsync.sh index 900832d..93f76b2 100755 --- a/mirmon-rsync.sh +++ b/mirmon-rsync.sh @@ -12,7 +12,7 @@ swan cd /var/www/mirrorstats.gentoo.org/gentoo-mirrorstats/rsync_mirrors # Grab mirrors from the web [[ -d ./var ]] || mkdir ./var -./get-mirror-list-rsync.rb > ./var/g.mirrors +../get-mirrors-from-rsync-xml.rb > ./var/g.mirrors # infra mirrors, "manually added" to list to check for i in ${INFRA}; do echo "gentoo rsync://$i.gentoo.org" >> ./var/g.mirrors diff --git a/releases_mirrors/get-mirror-list-releases.rb b/releases_mirrors/get-mirror-list-releases.rb deleted file mode 100755 index 0794df3..0000000 --- a/releases_mirrors/get-mirror-list-releases.rb +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/ruby - -MIRROR_DATA="https://api.gentoo.org/mirrors/distfiles.xml"; - -%w[ rexml/document open-uri ].each {|lib| require lib } - -m = URI.parse(MIRROR_DATA).read -x = REXML::Document.new(m) - -REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| - country = el.attributes['country'] - - el.each_element('mirror/uri/') do |mirror| - puts "#{country.downcase} #{mirror[0].to_s}" - end - -} - diff --git a/rsync_mirrors/get-mirror-list-rsync.rb b/rsync_mirrors/get-mirror-list-rsync.rb deleted file mode 100755 index 252bfd2..0000000 --- a/rsync_mirrors/get-mirror-list-rsync.rb +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/ruby - -MIRROR_DATA="https://api.gentoo.org/mirrors/rsync.xml"; - -%w[ rexml/document open-uri ].each {|lib| require lib } - -m = URI.parse(MIRROR_DATA).read -x = REXML::Document.new(m) - -REXML::XPath.each(x, '//*/mirrorgroup[@country]') {|el| - country = el.attributes['country'] - - el.each_element('mirror/uri/') do |mirror| - puts "#{country.downcase} #{mirror[0].to_s}" if mirror[0].to_s =~ /rsync\d+/ - end - -} -
