commit ce065946118bda2901eeed03d7f842f9dfd415dd
Author: Tom Ritter <[email protected]>
Date: Fri Jul 1 17:49:06 2016 -0500
Track the number of known relays in addition to running relays. Store the
data in the SQLite DB after retrieving the data
---
.gitignore | 1 +
parseOldConsensuses.py | 9 ++++++---
write_website.py | 34 ++++++++++++++++++++++++++++++++++
3 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/.gitignore b/.gitignore
index ad4f990..0fa8066 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,5 +3,6 @@
*.swp
stem
out/index.html
+out/graphs.html
out/consensus-health*
out/download-stats.csv
\ No newline at end of file
diff --git a/parseOldConsensuses.py b/parseOldConsensuses.py
index a616014..e45fd27 100755
--- a/parseOldConsensuses.py
+++ b/parseOldConsensuses.py
@@ -70,8 +70,8 @@ def main(dir):
dirauth_columns = ""
dirauth_columns_questions = ""
for d in dirAuths:
- dirauth_columns += d + "_running integer, " + d + "_bwauth
integer, "
- dirauth_columns_questions += ", ?, ?"
+ dirauth_columns += d + "_known integer, " + d + "_running
integer, " + d + "_bwauth integer, "
+ dirauth_columns_questions += ",?,?,?"
votes = {}
for root, dirs, files in os.walk(dir):
@@ -97,8 +97,9 @@ def main(dir):
print "Found two votes for dirauth " + dirauth
+ " and time " + filepath
votes[voteTime][dirauth]['present'] = 1
- votes[voteTime][dirauth]['bwlines'] =
int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l',
shell=True))
+ votes[voteTime][dirauth]['known'] =
int(subprocess.check_output('egrep "^r " "' + filepath + '" | wc -l',
shell=True))
votes[voteTime][dirauth]['running'] =
int(subprocess.check_output('egrep "^s " "' + filepath + '" | grep " Running" |
wc -l', shell=True))
+ votes[voteTime][dirauth]['bwlines'] =
int(subprocess.check_output('grep Measured= "' + filepath + '" | wc -l',
shell=True))
dbc.execute("CREATE TABLE IF NOT EXISTS vote_data(date integer, " +
dirauth_columns + "PRIMARY KEY(date ASC))")
dbc.commit()
@@ -112,11 +113,13 @@ def main(dir):
insertValues = [t]
for d in dirAuths:
if d in votes[t]:
+ insertValues.append(votes[t][d]['known'])
insertValues.append(votes[t][d]['running'])
insertValues.append(votes[t][d]['bwlines'])
else:
insertValues.append(None)
insertValues.append(None)
+ insertValues.append(None)
dbc.execute("INSERT OR REPLACE INTO vote_data VALUES (?" +
dirauth_columns_questions + ")", insertValues)
dbc.commit()
diff --git a/write_website.py b/write_website.py
index 85cca00..e47d9a3 100755
--- a/write_website.py
+++ b/write_website.py
@@ -9,6 +9,7 @@ Performs a variety of checks against the present votes and
consensus.
import os
import sys
import time
+import sqlite3
import datetime
import operator
import traceback
@@ -44,6 +45,8 @@ downloader = stem.descriptor.remote.DescriptorDownloader(
def directory_authorities():
return dict((k, v) for (k, v) in DIRECTORY_AUTHORITIES.items() if k not
in CONFIG['ignored_authorities'])
+def unix_time(dt):
+ return (dt - datetime.datetime.utcfromtimestamp(0)).total_seconds() *
1000.0
def main():
# loads configuration data
@@ -59,6 +62,37 @@ def main():
f.write("%s,%i,%i\n" % (ds, time.time() * 1000,
int(consensus_fetching_runtimes[ds] * 1000)))
f.close()
+ dbc = sqlite3.connect(os.path.join('data', 'historical.db'))
+
+ #Calculate the number of known and measured relays for each dirauth and
insert it into the database
+ databaseDirAuths = "faravahar, gabelmoo, dizum, moria1, urras,
maatuska, longclaw, tor26, dannenberg, turtles".split(", ")
+ data = {}
+ for dirauth_nickname in votes:
+ vote = votes[dirauth_nickname]
+
+ runningRelays = 0
+ bandwidthWeights = 0
+ for r in vote.routers.values():
+ if r.measured >= 0L:
+ bandwidthWeights += 1
+ if u'Running' in r.flags:
+ runningRelays += 1
+ data[dirauth_nickname] = {'known' : len(vote.routers.values()),
'running' : runningRelays, 'bwlines' : bandwidthWeights}
+
+ insertValues = [unix_time(consensuses.values()[0].valid_after)]
+ for dirauth_nickname in databaseDirAuths:
+ if dirauth_nickname in votes:
+ insertValues.append(data[dirauth_nickname]['known'])
+ insertValues.append(data[dirauth_nickname]['running'])
+ insertValues.append(data[dirauth_nickname]['bwlines'])
+ else:
+ insertValues.append(None)
+ insertValues.append(None)
+ insertValues.append(None)
+
+ dbc.execute("INSERT OR REPLACE INTO vote_data VALUES
(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", insertValues)
+ dbc.commit()
+
# great for debugging
#import pickle
#pickle.dump(consensuses, open('consensus.p', 'wb'))
_______________________________________________
tor-commits mailing list
[email protected]
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
