This is an automated email from the ASF dual-hosted git repository.
thelabdude pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new cd997e4 SOLR-15665: Move polling logic under main (#318)
cd997e4 is described below
commit cd997e48b157f8f0568e4df41aa4a97b7acf1986
Author: Timothy Potter <[email protected]>
AuthorDate: Wed Sep 29 09:59:37 2021 -0600
SOLR-15665: Move polling logic under main (#318)
---
dev-tools/scripts/poll-mirrors.py | 140 +++++++++++++++++++-------------------
1 file changed, 70 insertions(+), 70 deletions(-)
diff --git a/dev-tools/scripts/poll-mirrors.py
b/dev-tools/scripts/poll-mirrors.py
index 7a94a29..86fb7ec 100755
--- a/dev-tools/scripts/poll-mirrors.py
+++ b/dev-tools/scripts/poll-mirrors.py
@@ -101,75 +101,75 @@ def check_mirror(url):
p('\nFAIL: ' + url + '\n' if args.details else 'X')
return url
+if __name__ == '__main__':
+ desc = 'Periodically checks that all Lucene/Solr mirrors contain either a
copy of a release or a specified path'
+ parser = argparse.ArgumentParser(description=desc)
+ parser.add_argument('-version', '-v', help='Lucene/Solr version to check')
+ parser.add_argument('-path', '-p', help='instead of a versioned release,
check for some/explicit/path')
+ parser.add_argument('-interval', '-i', help='seconds to wait before
re-querying mirrors', type=int, default=300)
+ parser.add_argument('-details', '-d', help='print missing mirror URLs',
action='store_true', default=False)
+ parser.add_argument('-once', '-o', help='run only once',
action='store_true', default=False)
+ args = parser.parse_args()
+
+ if (args.version is None and args.path is None) \
+ or (args.version is not None and args.path is not None):
+ p('You must specify either -version or -path but not both!\n')
+ sys.exit(1)
-desc = 'Periodically checks that all Lucene/Solr mirrors contain either a copy
of a release or a specified path'
-parser = argparse.ArgumentParser(description=desc)
-parser.add_argument('-version', '-v', help='Lucene/Solr version to check')
-parser.add_argument('-path', '-p', help='instead of a versioned release, check
for some/explicit/path')
-parser.add_argument('-interval', '-i', help='seconds to wait before
re-querying mirrors', type=int, default=300)
-parser.add_argument('-details', '-d', help='print missing mirror URLs',
action='store_true', default=False)
-parser.add_argument('-once', '-o', help='run only once', action='store_true',
default=False)
-args = parser.parse_args()
-
-if (args.version is None and args.path is None) \
- or (args.version is not None and args.path is not None):
- p('You must specify either -version or -path but not both!\n')
- sys.exit(1)
-
-try:
- conn = http.HTTPConnection('www.apache.org')
- conn.request('GET', '/mirrors/')
- response = conn.getresponse()
- html = response.read()
-except Exception as e:
- p('Unable to fetch the Apache mirrors list!\n')
- sys.exit(1)
-
-mirror_path = args.path if args.path is not None else
'lucene/java/{}/changes/Changes.html'.format(args.version)
-maven_url = None if args.version is None else
'https://repo1.maven.org/maven2/' \
-
'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
-maven_available = False
-
-pending_mirrors = []
-for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE |
re.IGNORECASE | re.DOTALL):
- row = match.group(1)
- if not '<TD>ok</TD>' in row:
- # skip bad mirrors
- continue
-
- match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE |
re.IGNORECASE)
- if match:
- pending_mirrors.append(match.group(1) + mirror_path)
-
-total_mirrors = len(pending_mirrors)
-
-label = args.version if args.version is not None else args.path
-while True:
- p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
- p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
- if maven_url is not None and not maven_available:
- p(' and Maven Central')
- p('...\n')
-
- if maven_url is not None and not maven_available:
- maven_available = mirror_contains_file(maven_url)
-
- start = time.time()
- with Pool(processes=5) as pool:
- pending_mirrors = list(filter(lambda x: x is not None,
pool.map(check_mirror, pending_mirrors)))
- stop = time.time()
- remaining = args.interval - (stop - start)
-
- available_mirrors = total_mirrors - len(pending_mirrors)
-
- if maven_url is not None:
- p('\n\n{} is{}downloadable from Maven Central'.format(label, ' ' if
maven_available else ' not '))
- p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'
- .format(label, available_mirrors, total_mirrors, available_mirrors * 100 /
total_mirrors))
- if len(pending_mirrors) == 0 or args.once == True:
- break
-
- if remaining > 0:
- p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5)))
- time.sleep(remaining)
+ try:
+ conn = http.HTTPConnection('www.apache.org')
+ conn.request('GET', '/mirrors/')
+ response = conn.getresponse()
+ html = response.read()
+ except Exception as e:
+ p('Unable to fetch the Apache mirrors list!\n')
+ sys.exit(1)
+
+ mirror_path = args.path if args.path is not None else
'lucene/java/{}/changes/Changes.html'.format(args.version)
+ maven_url = None if args.version is None else
'https://repo1.maven.org/maven2/' \
+
'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
+ maven_available = False
+
+ pending_mirrors = []
+ for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE |
re.IGNORECASE | re.DOTALL):
+ row = match.group(1)
+ if not '<TD>ok</TD>' in row:
+ # skip bad mirrors
+ continue
+
+ match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE |
re.IGNORECASE)
+ if match:
+ pending_mirrors.append(match.group(1) + mirror_path)
+
+ total_mirrors = len(pending_mirrors)
+
+ label = args.version if args.version is not None else args.path
+ while True:
+ p('\n{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()))
+ p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
+ if maven_url is not None and not maven_available:
+ p(' and Maven Central')
+ p('...\n')
+
+ if maven_url is not None and not maven_available:
+ maven_available = mirror_contains_file(maven_url)
+
+ start = time.time()
+ with Pool(processes=5) as pool:
+ pending_mirrors = list(filter(lambda x: x is not None,
pool.map(check_mirror, pending_mirrors)))
+ stop = time.time()
+ remaining = args.interval - (stop - start)
+
+ available_mirrors = total_mirrors - len(pending_mirrors)
+
+ if maven_url is not None:
+ p('\n\n{} is{}downloadable from Maven Central'.format(label, ' ' if
maven_available else ' not '))
+ p('\n{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'
+ .format(label, available_mirrors, total_mirrors, available_mirrors * 100
/ total_mirrors))
+ if len(pending_mirrors) == 0 or args.once == True:
+ break
+
+ if remaining > 0:
+ p('Sleeping for {:d} seconds...\n'.format(int(remaining + 0.5)))
+ time.sleep(remaining)
