Yurik has uploaded a new change for review.
https://gerrit.wikimedia.org/r/67450
Change subject: rewrote vcl script to verify and fix varnish file
......................................................................
rewrote vcl script to verify and fix varnish file
the script copies and optimizes ip ranges from zero page configs,
adds carrier name (comment field), and alerts of any mismatched blocks
To run:
python3 vclopt.py zero.inc.vcl.erb
Change-Id: I216afce5c63c2ad73d314b5d217e5566bcd006dd
---
M maintenance/vclopt.py
1 file changed, 110 insertions(+), 14 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ZeroRatedMobileAccess
refs/changes/50/67450/1
diff --git a/maintenance/vclopt.py b/maintenance/vclopt.py
index 224ab95..ba178ab 100644
--- a/maintenance/vclopt.py
+++ b/maintenance/vclopt.py
@@ -1,33 +1,129 @@
import re
import sys
-from cidr import *
+import netaddr
+import utils
+
+def write(dst, line):
+ dst.write(bytes(line, 'UTF-8'))
+
+def toVclStr(ip, comment):
+ s = str(ip)
+ s = s.replace('/32','') if ':' not in s else s.replace('/128','')
+ s = s.replace('/','"/')
+ if '"' not in s: s = s + '"'
+ return '\t"%s;%s\n' % (s,comment)
+
+def writeHdr(dst, id, conf, comment = ''):
+ names = {
+ '520-18':'dtac',
+ '630-86':'orange congo',
+ '652-02':'orange botswana',
+ '456-02':'hello cambodia',
+ '502-13':'celcom malaysia',
+ '413-02':'dialog sri lanka',
+ '405-0':'tata',
+ '502-16':'digi',
+ '612-03':'orange ivory coast',
+ '641-14':'orange uganda',
+ '420-01':'saudi telecom',
+ '639-07':'orange kenya',
+ '605-01':'orange tunisia',
+ '614-04':'orange niger',
+ '624-02':'orange cameroon',
+ '470-01':'grameenphone bangladesh',
+ '639-02':'safaricom kenya',
+ '297-01':'telenor montenegro',
+ '510-11':'xl indonesia',
+ '250-99':'vimpelcom beeline',
+ '410-01':'vimpelcom mobilink pakistan',
+ '604-00':'orange meditel morocco',
+ '623-01':'central african republic',
+ }
+ comment = comment.strip(' \t/')
+ if comment == '' and id in names:
+ comment = names[id].title()
+ if conf is not None:
+ comment = conf['comment'] if 'comment' in conf else comment
+ disabled = '(DISABLED)'
+ if comment.endswith(disabled):
+ comment = comment[:-len(disabled)].strip()
+ if 'enabled' in conf and not conf['enabled']:
+ comment += ' ' + disabled
+ if comment != '':
+ comment = ' // ' + comment.strip()
+ write(dst, 'acl carrier_%s {%s\n' % (id.replace('-','_'), comment))
if len(sys.argv) != 2:
raise Exception('Expects VCL file as the first argument')
-
-with open(sys.argv[1], 'r') as src, open(sys.argv[1]+'.out', 'w') as dst:
+
+confs = utils.getAllConfigs(utils.META)
+# Remove the '*' in any keys
+confs = dict([(v.replace('*',''),confs[v]) for v in confs])
+
+with open(sys.argv[1], 'r') as src, open(sys.argv[1]+'.out', 'wb') as dst:
pending = []
acl = None
for line in src:
if acl is None:
- if re.match(r'^acl [_a-z0-9]+ \{\n$', line, re.I):
- acl = line
+ r = re.match(r'^acl ([_a-z0-9]+) \{(.*)\n$', line, re.I)
+ if r:
+ acl = (r.group(1), '' if r.group(2) is None else r.group(2))
+ continue
elif line.startswith('acl '):
print('Skipping ' + line.strip())
- dst.write(line)
+ write(dst, line)
else:
- if re.match(r'^\t"([0-9.:]+)"(/[0-9]+)?;\n$', line):
- pending.append(line)
+ r = re.match(r'^\t"([0-9a-f.:]+)"(/[0-9]+)?;(.*)\n$', line)
+ if r:
+ bk = '' if r.group(2) is None else r.group(2)
+ pending.append((netaddr.IPNetwork(r.group(1) + bk),
r.group(3)))
continue
if line == '}\n':
- print('Optimizing ' + acl.strip())
- vals = cidrToVclStrs(optimizeCidr(vclToCidrStrs(pending)))
+ print('Optimizing ' + acl[0])
+ comments = dict([(v[0], v[1]) for v in pending if v[1]])
+ vals = netaddr.cidr_merge([v[0] for v in pending])
+ id = acl[0].replace('carrier_','').replace('_','-')
+ if id in confs:
+ conf = confs[id]
+ del confs[id]
+ ipset = netaddr.IPSet(vals)
+ vals2 = [netaddr.IPNetwork(v) for v in conf['ips']]
+ ipset2 = netaddr.IPSet(vals2)
+ extras = ipset - ipset2
+ new = ipset2 - ipset
+ if new != netaddr.IPSet():
+ print('Adding to %s: %s' % (acl[0], str(new)))
+ if extras != netaddr.IPSet():
+ print('Removing from %s: %s' % (acl[0], str(extras)))
+ vals = netaddr.cidr_merge(vals2)
+ writeHdr(dst, id, conf, acl[1])
+ else:
+ print(acl[0] + ' not found in configuration')
+ writeHdr(dst, id, None, acl[1])
for ip in vals:
- dst.write('\t' + ip + '\n')
+ if ip in comments:
+ c = comments[ip]
+ del comments[ip]
+ else:
+ c = ''
+ write(dst, toVclStr(ip, c))
+ for c in comments:
+ print('Deleted comment "%s" for ip %s' % (comments[c],
str(c)))
else:
- print('Cannot parse "' + acl.strip() + '": ' + line)
+ print('Cannot parse %s: %s' % (acl[0], line))
+ id = acl[0].replace('carrier_','').replace('_','-')
+ writeHdr(dst, id, conf, acl[1])
for v in pending:
- dst.write(v)
- dst.write(line)
+ write(dst, toVclStr(v[0], v[1]))
+ write(dst, line)
acl = None
pending = []
+ for id in confs:
+ print('ID %s was not found' % id)
+ conf = confs[id]
+ vals = [netaddr.IPNetwork(v) for v in conf['ips']]
+ vals = netaddr.cidr_merge(vals)
+ writeHdr(dst, id, conf)
+ for v in vals:
+ write(dst, toVclStr(v, ''))
+ write(dst, '}\n\n')
--
To view, visit https://gerrit.wikimedia.org/r/67450
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I216afce5c63c2ad73d314b5d217e5566bcd006dd
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ZeroRatedMobileAccess
Gerrit-Branch: master
Gerrit-Owner: Yurik <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
