Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/63502
Change subject: Maintenance scripts - optimize the VCL file's CIDR
......................................................................
Maintenance scripts - optimize the VCL file's CIDR
Change-Id: I920e1e75bb682618ce045ceabec946651d6e5d94
---
M maintenance/cidr.py
M maintenance/test.py
A maintenance/vclopt.py
3 files changed, 56 insertions(+), 16 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroRatedMobileAccess
refs/changes/02/63502/1
diff --git a/maintenance/cidr.py b/maintenance/cidr.py
index ca2f2dc..492dfb2 100644
--- a/maintenance/cidr.py
+++ b/maintenance/cidr.py
@@ -1,4 +1,3 @@
-from utils import *
from netaddr import *
import re
@@ -12,6 +11,18 @@
def optimizeCidr(ips):
return list([str(i).replace('/32','') for i in cidr_merge([IPNetwork(ip)
for ip in ips])])
+def vclToCidrStrs(vals):
+ return [s2 for s2 in [re.sub(r'[ \t;"\n]', '', s) for s in vals] if s2 !=
'']
+
+def cidrToVclStrs(vals):
+ res = []
+ for ip in vals:
+ s = str(ip)
+ s = s.replace('/32','') if ':' not in s else s.replace('/128','')
+ s = s.replace('/','"/')
+ if '"' not in s: s = s + '"'
+ res.append( '"' + s + ';' )
+ return res
def printOptimization(confs):
opt = optimize(confs)
@@ -41,20 +52,8 @@
def load(file):
with open(file,'r') as f:
- return list([s2 for s2 in [re.sub(r'[ \t;"\n]', '', s) for s in f] if
s2 != ''])
+ return list(vclToCidrStrs(f))
def printAcl(ips):
- for ip in ips:
- s = str(ip)
- s = s.replace('/32','') if ':' not in s else s.replace('/128','')
- s = s.replace('/','"/')
- if '"' not in s: s = s + '"'
- s = '"' + s + ';'
- print(s)
-
-def run(site = TEST):
- confs = getAllConfigs(site)
- printConflicts(confs)
- print()
- printOptimization(confs)
-
+ for ip in cidrToVclStrs(ips):
+ print(ip)
diff --git a/maintenance/test.py b/maintenance/test.py
index 2db4938..7194f98 100644
--- a/maintenance/test.py
+++ b/maintenance/test.py
@@ -1,5 +1,6 @@
from utils import *
from netaddr import *
+from cidr import *
def orderIps(conf):
for xcs, c in conf.items():
@@ -83,4 +84,11 @@
return dict([(v.parent.parent.th.text, str(v)) for v in
soup.find_all('div', class_='mw-mf-banner')])
+def checkIPs(site = TEST):
+ confs = getAllConfigs(site)
+ printConflicts(confs)
+ print()
+ printOptimization(confs)
+
+
# checkProdBanners()
diff --git a/maintenance/vclopt.py b/maintenance/vclopt.py
new file mode 100644
index 0000000..224ab95
--- /dev/null
+++ b/maintenance/vclopt.py
@@ -0,0 +1,33 @@
+import re
+import sys
+from cidr import *
+
+if len(sys.argv) != 2:
+ raise Exception('Expects VCL file as the first argument')
+
+with open(sys.argv[1], 'r') as src, open(sys.argv[1]+'.out', 'w') as dst:
+ pending = []
+ acl = None
+ for line in src:
+ if acl is None:
+ if re.match(r'^acl [_a-z0-9]+ \{\n$', line, re.I):
+ acl = line
+ elif line.startswith('acl '):
+ print('Skipping ' + line.strip())
+ dst.write(line)
+ else:
+ if re.match(r'^\t"([0-9.:]+)"(/[0-9]+)?;\n$', line):
+ pending.append(line)
+ continue
+ if line == '}\n':
+ print('Optimizing ' + acl.strip())
+ vals = cidrToVclStrs(optimizeCidr(vclToCidrStrs(pending)))
+ for ip in vals:
+ dst.write('\t' + ip + '\n')
+ else:
+ print('Cannot parse "' + acl.strip() + '": ' + line)
+ for v in pending:
+ dst.write(v)
+ dst.write(line)
+ acl = None
+ pending = []
--
To view, visit https://gerrit.wikimedia.org/r/63502
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I920e1e75bb682618ce045ceabec946651d6e5d94
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroRatedMobileAccess
Gerrit-Branch: master
Gerrit-Owner: Yurik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
