Hello Milimetric,
I'd like you to do a code review. Please visit
https://gerrit.wikimedia.org/r/91679
to review the following change.
Change subject: Split data directory into public and private part
......................................................................
Split data directory into public and private part
Card: analytics 1168
Change-Id: Ib915bbf8abeec37a409c315dea36e371f20834b8
---
M scripts/make_and_push_limn_files.sh
M scripts/make_limn_files.py
2 files changed, 60 insertions(+), 35 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/geowiki
refs/changes/79/91679/1
diff --git a/scripts/make_and_push_limn_files.sh
b/scripts/make_and_push_limn_files.sh
index 0634528..91acd0b 100755
--- a/scripts/make_and_push_limn_files.sh
+++ b/scripts/make_and_push_limn_files.sh
@@ -41,7 +41,8 @@
#---------------------------------------------------
# Filtering parameters
CRON_MODE=no
-BASE_DIR_REL=
+BASE_DIR_PUBLIC_DIR_REL=
+BASE_DIR_PRIVATE_DIR_REL=
for PARAM in "$@"
do
case "$PARAM" in
@@ -49,8 +50,11 @@
"$MAIN_SCRIPT_FILE_ABS" --help
exit 0
;;
- "--basedir="* | "-d="* )
- BASE_DIR_REL="${PARAM//*=/}"
+ "--basedir_public="* | "-d="* )
+ BASE_DIR_PUBLIC_DIR_REL="${PARAM//*=/}"
+ ;;
+ "--basedir_private="* )
+ BASE_DIR_PRIVATE_DIR_REL="${PARAM//*=/}"
;;
esac
done
@@ -72,28 +76,42 @@
exit "$EXIT_CODE"
fi
-if [ -z "$BASE_DIR_REL" ]
+if [ -z "$BASE_DIR_PUBLIC_DIR_REL" ]
then
- error "No --basedir provided."
+ error "No --basedir_public provided."
+fi
+
+if [ -z "$BASE_DIR_PRIVATE_DIR_REL" ]
+then
+ error "No --basedir_private provided."
fi
#---------------------------------------------------
# Actual making and pushing of limn files
-# Cleaning data repo
-cd "$BASE_DIR_REL"
-git reset --hard
-git pull
-cd -
+# Cleaning data repos
+for REPO_DIR_REL in "$BASE_DIR_PUBLIC_DIR_REL" "$BASE_DIR_PRIVATE_DIR_REL"
+do
+ pushd "$REPO_DIR_REL" >/dev/null
+ git reset --hard
+ git pull
+ popd >/dev/null
+done
# Recompute limn files
"$MAIN_SCRIPT_FILE_ABS" "$@"
-# Commit and push limn files
-cd "$BASE_DIR_REL"
-git add -A
-LAST_DATE="$(tail --lines=1 datafiles/en_all.csv | cut --field=1 --delimiter=,
)"
+# Get the most recent date. We assume en_all.csv always has at least one
+# editor on any given day :-)
+LAST_DATE="$(tail --lines=1 "$BASE_DIR_PRIVATE_DIR_REL/datafiles/en_all.csv" |
cut --field=1 --delimiter=, )"
LAST_DATE=${LAST_DATE////-}
-git commit -m "Automatic commit of data up to $LAST_DATE"
-git push
-cd -
+
+# Commit and push limn files
+for REPO_DIR_REL in "$BASE_DIR_PUBLIC_DIR_REL" "$BASE_DIR_PRIVATE_DIR_REL"
+do
+ pushd "$REPO_DIR_REL" >/dev/null
+ git add -A
+ git commit -m "Automatic commit of data up to $LAST_DATE"
+ git push
+ popd >/dev/null
+done
diff --git a/scripts/make_limn_files.py b/scripts/make_limn_files.py
index 9055bda..f133259 100755
--- a/scripts/make_limn_files.py
+++ b/scripts/make_limn_files.py
@@ -276,9 +276,13 @@
nargs='+',
help='any number of appropriately named json files')
parser.add_argument(
- '-d','--basedir',
- default='/home/erosen/src/dashboard/geowiki/data',
- help='directory in which to find or create the datafiles and
datasources directories for the *.csv and *.yaml files')
+ '-d', '--basedir_public',
+ default=os.path.join(os.path.expanduser('~'), 'geowiki',
'data-public'),
+ help='directory in which to find or create the public datafiles and
public datasources directories for the *.csv and *.json files')
+ parser.add_argument(
+ '--basedir_private',
+ default=os.path.join(os.path.expanduser('~'), 'geowiki',
'data-private'),
+ help='directory in which to find or create the private datafiles and
private datasources directories for the *.csv and *.json files')
parser.add_argument(
'-b', '--basename',
default='geo_editors',
@@ -371,21 +375,24 @@
for widget in g.graph["root"]["children"] \
if "nodeType" not in widget or widget["nodeType"] != "callout" ]
-def plot_active_editor_totals(basedir):
+def plot_active_editor_totals(basedir_source, basedir_destination):
"""Write out files for 'Active Editors Total' graph
Keyword arguments:
- basedir -- string. Path to the data repository
+ basedir_source -- string. Path to the data repository to read
+ global_south.csv from
+ basedir_destination -- string. Path to the data repository to store
+ the comupted data in
This function computes the total number of active editors and
writes out the necessary datafile, datasource, and graph files to
show them in Limn. Those files get written into the corresponding
- subdirectories of basedir.
+ subdirectories of basedir_destination.
For the computation of the data, this function relies solely on
the global_south.csv file.
"""
- df = pd.read_csv(basedir + '/datafiles/global_south.csv',
index_col='date', parse_dates=['date'])
+ df = pd.read_csv(basedir_source + '/datafiles/global_south.csv',
index_col='date', parse_dates=['date'])
df['Active Editors Total'] = (df['Global South (5+)'] + df['Global
North (5+)'] + df['Unkown (5+)'] ).apply(float)
df_total = df[['Active Editors Total']]
@@ -393,7 +400,7 @@
limn_name='Active Editors Total',
limn_group=LIMN_GROUP,
data = df_total)
- ds_total.write(basedir)
+ ds_total.write(basedir_destination)
g = ds_total.get_graph(metric_ids=['Active Editors Total'],
title='Active Editors Total (Tentative)',
graph_id='active_editors_total')
@@ -408,7 +415,7 @@
"""
drop_callout_widget(g)
- g.write(basedir)
+ g.write(basedir_destination)
if __name__ == '__main__':
args = parse_args()
@@ -419,7 +426,7 @@
cursor = db.cursor()
- write_project_mysql('en', cursor, args.basedir, country_graphs=True)
+ write_project_mysql('en', cursor, args.basedir_private,
country_graphs=True)
# # use metadata from Google Drive doc which lets us group by country
#country_data = gcat.get_file(META_DATA_TITLE, sheet=META_DATA_SHEET,
fmt='dict', usecache=False)
@@ -430,22 +437,22 @@
# logger.debug('typ(country_data): %s', type(country_data))
# logger.info('country_data[0].keys: %s', country_data[0].keys())
- write_group_mysql(META_DATA_GLOBAL_SOUTH_FIELD, country_data, cursor,
args.basedir)
- write_group_mysql(META_DATA_REGION_FIELD, country_data, cursor,
args.basedir)
+ write_group_mysql(META_DATA_GLOBAL_SOUTH_FIELD, country_data, cursor,
args.basedir_private)
+ write_group_mysql(META_DATA_REGION_FIELD, country_data, cursor,
args.basedir_private)
- write_group_mysql(META_DATA_COUNTRY_FIELD, country_data, cursor,
args.basedir)
+ write_group_mysql(META_DATA_COUNTRY_FIELD, country_data, cursor,
args.basedir_private)
projects = get_projects()
if not args.parallel or sql.threadsafety < 2:
for i, project in enumerate(projects):
logger.info('processing project: %s (%d/%d)', project, i,
len(projects))
- process_project(project, cursor, args.basedir)
+ process_project(project, cursor, args.basedir_private)
else:
pool = multiprocessing.Pool(20)
- pool.map_async(process_project_par, itertools.izip(projects,
itertools.repeat(args.basedir))).get(99999)
+ pool.map_async(process_project_par, itertools.izip(projects,
itertools.repeat(args.basedir_private))).get(99999)
- write_overall_mysql(projects, cursor, args.basedir)
- plot_gs_editor_fraction(args.basedir)
- plot_active_editor_totals(args.basedir)
+ write_overall_mysql(projects, cursor, args.basedir_private)
+ plot_gs_editor_fraction(args.basedir_private)
+ plot_active_editor_totals(args.basedir_private, args.basedir_public)
--
To view, visit https://gerrit.wikimedia.org/r/91679
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib915bbf8abeec37a409c315dea36e371f20834b8
Gerrit-PatchSet: 1
Gerrit-Project: analytics/geowiki
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
