Jdlrobson has uploaded a new change for review. https://gerrit.wikimedia.org/r/181428
Change subject: Update scripts in light of recent changes ...................................................................... Update scripts in light of recent changes Change-Id: If828c64cfe50b07213d19c8904b70ea670ab0893 --- M README.md D generate-graph.py M scripts/localurl M scripts/remoteurl M scripts/ssh 5 files changed, 9 insertions(+), 191 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/limn-mobile-data refs/changes/28/181428/1 diff --git a/README.md b/README.md index 7a7883f..ca5dc85 100644 --- a/README.md +++ b/README.md @@ -45,15 +45,11 @@ ## Adding your own Graphs -(Specific to Mobile right now, limitation will be removed *soon*) - - Write an SQL Query that returns data in the appropriate format, and place it in `mobile/<name>.sql` -- Add `<name>` to appropriate position in `dashboards/reportcard.json` +- Add `http://datasets.wikimedia.org/limn-public-data/mobile/datafiles/<name>.csv` to appropriate position in `dashboards/reportcard.json` - Add the graph to `mobile/config.yaml` -- Generate data for the graph `python generate.py -c scripts/config.yaml mobile -g <graphname>` -- Run `python generate-graph.py -g <graphname>` - +- Generate the datafile (csv) for the graph `python generate.py -c scripts/config.yaml mobile -g <graphname>` ## Testing using local data @@ -75,7 +71,7 @@ Now, you should be able to run `generate.py` with config overrides: - $ python generate.py -c scripts/config.yaml mobile + $ python generate.py -c scripts/config.yaml mobile -g my_graph_id When all the data is generated you still need to do one more thing to let Limn know that it should use the local data. The hacky solution is to replace diff --git a/generate-graph.py b/generate-graph.py deleted file mode 100644 index 22702bf..0000000 --- a/generate-graph.py +++ /dev/null @@ -1,170 +0,0 @@ -# Tool to generate appropriate files in the datasources and graphs directory -# TOUSE: -import argparse -import json -import copy -import yaml -import io - - -def make_datasource(name): - try: - f = open("datafiles/%s.csv" % name, "r") - headers = f.readlines()[0] - columns = headers.split(',') - f.close() - ds_columns = [] - for i in range(len(columns)): - column = {} - label = columns[i].strip() - column["id"] = label - if i == 0: - column["type"] = "date" - else: - column["type"] = "int" - column["label"] = label - ds_columns.append(column) - - ds = { - "url": "http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/%s.csv"; % name, - "format": "csv", - "type": "timeseries", - "id": name, - "name": name, - "columns": ds_columns - } - print "Writing datasource..." - f = open("datasources/%s.json" % name, "w") - dump = json.dumps(ds, indent=4) - f.writelines(dump) - f.close() - return True - except IOError: - print "First you must run:\npython generate.py -c scripts/config.yaml mobile -g %s" % name - return False - - -def get_graph_config(name): - config_path = 'mobile/config.yaml' - with io.open(config_path, encoding='utf-8') as config_file: - config = yaml.load(config_file) - return config["graphs"][name] - - -def get_datasource(name): - f = open("datasources/%s.json" % name, "r") - jsonds = '\n'.join(f.readlines()) - ds = json.loads(jsonds) - f.close() - return ds - - -def generate_graph(name): - config = get_graph_config(name) - ds = get_datasource(name) - ds_columns = ds["columns"] - - graph_columns = [] - for i in range(len(ds_columns)): - if i > 0: - ds_column = ds_columns[i] - column = { - "nodeType": "line", - "metric": { - "sourceId": name, - "sourceCol": ds_column["id"] - } - } - graph_columns.append(column) - - axis = {"disabled": False, "nodeType": "axis"} - axis_x = copy.deepcopy(axis) - axis_y = copy.deepcopy(axis) - axis_x["options"] = { - "dimension": "x", - } - axis_y["options"] = { - "dimension": "y", - } - grid = { - "nodeType": "grid", - "options": { - "ticks": 10, - "dimension": "x" - } - } - grid_x = grid - grid_y = copy.deepcopy(grid) - grid_y["options"]["dimension"] = "y" - - callout = { - "nodeType": "callout", - "target": "latest", - "steps": [ - "1y", - "1M" - ], - "metricRef": 0, - "options": { - "deltaPercent": True, - "dateFormat": "MMM YYYY", - "colorDelta": True - } - } - legend = { - "nodeType": "legend", - "options": { - "shortLabels": True, - "dateFormat": "DD MMM YYYY" - } - } - zoom_brush = { - "nodeType": "zoom-brush" - } - graph_json = { - "graph_version": "0.6.0", - "name": config["title"], - "root": { - "scaling": "linear", - "nodeType": "canvas", - "minWidth": 750, - "minHeight": 500, - "height": 500, - "disabled": False, - "width": "auto", - "children": [ - axis_x, axis_y, grid_x, grid_y, callout, legend, zoom_brush, - { - "disabled": False, - "nodeType": "line-group", - "options": { - "palette": "wmf_projects", - "scale": "log", - "stroke": { - "opacity": 1, - "width": 2 - } - }, - "children": graph_columns - } - ], - "id": name - } - } - print "Writing graph" - f = open("graphs/%s.json" % name, "w") - dump = json.dumps(graph_json, indent=4) - f.writelines(dump) - f.close() - - -def generate(name): - success = make_datasource(name) - if success: - generate_graph(name) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate data for the mobile dashboard.') - parser.add_argument('-g', '--graph', help='the name of a single graph you want to generate for') - args = parser.parse_args() - generate(args.graph) diff --git a/scripts/localurl b/scripts/localurl index 2f7811c..433bac4 100755 --- a/scripts/localurl +++ b/scripts/localurl @@ -1,9 +1,6 @@ #!/usr/bin/env bash # make all data URLs local (for testing) -for f in `find datasources -name "*.json" -print`; do - # don't use sed's -i option, it's not cross-platform - # (http://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-and-linux) - sed 's/http:\/\/stat1001\.wikimedia\.org\/limn-public-data\/mobile\/datafiles/\/data\/datafiles\/mobile/g' "$f" > tmp.json - mv -f tmp.json "$f" -done - +if [ ! -f dashboards/reportcard.json.tmp ]; then + mv dashboards/reportcard.json dashboards/reportcard.json.tmp + sed 's/http:\/\/datasets.wikimedia.org\/limn-public-data\/mobile\/datafiles/http:\/\/localhost:5000\/data\/datafiles\/mobile/g' dashboards/reportcard.json.tmp > dashboards/reportcard.json +fi diff --git a/scripts/remoteurl b/scripts/remoteurl index d222f2e..deb82ee 100755 --- a/scripts/remoteurl +++ b/scripts/remoteurl @@ -1,9 +1,4 @@ #!/usr/bin/env bash # make all data URLs remote (use before pushing to repo if you used localurl) -for f in `find datasources -name "*.json" -print`; do - # don't use sed's -i option, it's not cross-platform - # (http://stackoverflow.com/questions/5694228/sed-in-place-flag-that-works-both-on-mac-and-linux) - sed 's/\/data\/datafiles\/mobile/http:\/\/stat1001\.wikimedia\.org\/limn-public-data\/mobile\/datafiles/g' "$f" > tmp.json - mv -f tmp.json "$f" -done +mv dashboards/reportcard.json.tmp dashboards/reportcard.json diff --git a/scripts/ssh b/scripts/ssh index e4556b2..2368757 100755 --- a/scripts/ssh +++ b/scripts/ssh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # create SSH tunnel to EventLogging databases through stat1 -ssh -L 3307:s1-analytics-slave.eqiad.wmnet:3306 -L 3308:s4-analytics-slave.eqiad.wmnet:3306 stat1003.wikimedia.org +ssh -L 3307:s1-analytics-slave.eqiad.wmnet:3306 -L 3308:s4-analytics-slave.eqiad.wmnet:3306 stat1003.eqiad.wmnet -- To view, visit https://gerrit.wikimedia.org/r/181428 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If828c64cfe50b07213d19c8904b70ea670ab0893 Gerrit-PatchSet: 1 Gerrit-Project: analytics/limn-mobile-data Gerrit-Branch: master Gerrit-Owner: Jdlrobson <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
