JGonera has submitted this change and it was merged.
Change subject: Generate JSON files in datasources using templates
......................................................................
Generate JSON files in datasources using templates
That allows setting custom URL for data sources (output.url in
config.yaml).
Change-Id: I267961555de8608d7bb70aa74903e62d6025a7ac
---
M .gitignore
M generate.py
M mobile/config.yaml
R mobile/datagenerators/30-day-uploads.sql
R mobile/datagenerators/alltime-numbers.py
R mobile/datagenerators/cancelled-uploads.sql
R mobile/datagenerators/deleted-uploads.py
R mobile/datagenerators/error-correction.py
R mobile/datagenerators/error-uploads.sql
R mobile/datagenerators/no-cats.sql
R mobile/datagenerators/performance-rendering.sql
R mobile/datagenerators/share-attempts.sql
R mobile/datagenerators/successful-logins.sql
R mobile/datagenerators/successful-uploads.sql
R mobile/datagenerators/unique-uploaders.sql
R mobile/datagenerators/uploaders-experience.py
R mobile/datasources/30-day-uploads.json
R mobile/datasources/cancelled-uploads.json
R mobile/datasources/deleted-uploads.json
R mobile/datasources/error-uploads.json
R mobile/datasources/no-cats.json
R mobile/datasources/performance-rendering.json
R mobile/datasources/share-attempts.json
R mobile/datasources/successful-logins.json
R mobile/datasources/successful-uploads.json
R mobile/datasources/unique-uploaders.json
26 files changed, 51 insertions(+), 22 deletions(-)
Approvals:
JGonera: Verified; Looks good to me, approved
diff --git a/.gitignore b/.gitignore
index c07d99c..51eab90 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
-env/
-datafiles/
+/env/
+/datafiles/
+/datasources/
.output
*.swp
*.pyc
diff --git a/generate.py b/generate.py
index 95dfe9f..7ad1600 100644
--- a/generate.py
+++ b/generate.py
@@ -82,30 +82,57 @@
module = imp.load_source(name, file_path)
return module.execute(self)
- def execute(self):
+ def generate_datasources(self):
+ """Generates JSON files in datasources."""
+ output_path = 'datasources'
+ if not os.path.exists(output_path):
+ os.makedirs(output_path)
+
+ for key in self.config['graphs'].iterkeys():
+ filename = key + '.json'
+ print "Generating %s" % (filename)
+ template_filename = os.path.join(self.folder, 'datasources',
filename)
+
+ if os.path.exists(template_filename):
+ output_filename = os.path.join(output_path, filename)
+
+ with io.open(template_filename, encoding='utf-8') as
template_file:
+ content = self.render(template_file.read())
+
+ with open(output_filename, 'wb') as output_file:
+ output_file.write(content)
+
+ def generate_datafiles(self):
"""Generates a CSV report by executing Python code and SQL queries."""
+ output_path = self.config['output']['path']
+ if not os.path.exists(output_path):
+ os.makedirs(output_path)
+
for key, value in self.config['graphs'].iteritems():
- # Look for the sql first, then python
db_name = value.get('db', self.config['defaults']['db'])
- if os.path.exists(os.path.join(self.folder, key + '.sql')):
- file_path = os.path.join(self.folder, key + '.sql')
+ print "Generating %s (%s)" % (value['title'], key)
+
+ # Look for the sql first, then python
+ if os.path.exists(os.path.join(self.folder, 'datagenerators', key
+ '.sql')):
+ file_path = os.path.join(self.folder, 'datagenerators', key +
'.sql')
headers, rows = self.execute_sql(file_path, db_name)
- elif os.path.exists(os.path.join(self.folder, key + '.py')):
- file_path = os.path.join(self.folder, key + '.py')
+ elif os.path.exists(os.path.join(self.folder, 'datagenerators',
key + '.py')):
+ file_path = os.path.join(self.folder, 'datagenerators', key +
'.py')
headers, rows = self.execute_python(key, file_path)
else:
raise ValueError("Can not find SQL or Python for %s" % key)
- print "Generating %s (%s)" % (value['title'], file_path)
-
- output_path = self.config['output']['path']
csv_filename = os.path.join(output_path, key + '.csv')
with open(csv_filename, 'wb') as csv_file:
writer = csv.writer(csv_file)
writer.writerow(headers)
writer.writerows(rows)
+
+ def generate(self):
+ self.generate_datasources()
+ self.generate_datafiles()
if __name__ == "__main__":
@@ -115,4 +142,4 @@
args = parser.parse_args()
dg = DataGenerator(**vars(args))
- dg.execute()
+ dg.generate()
diff --git a/mobile/config.yaml b/mobile/config.yaml
index 3daa6b8..b31c8b7 100644
--- a/mobile/config.yaml
+++ b/mobile/config.yaml
@@ -20,6 +20,7 @@
running_average: 30
output:
path: /a/limn-public-data/mobile/datafiles
+ url: http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles
defaults:
db: el
graphs:
diff --git a/mobile/30-day-uploads.sql
b/mobile/datagenerators/30-day-uploads.sql
similarity index 100%
rename from mobile/30-day-uploads.sql
rename to mobile/datagenerators/30-day-uploads.sql
diff --git a/mobile/alltime-numbers.py
b/mobile/datagenerators/alltime-numbers.py
similarity index 100%
rename from mobile/alltime-numbers.py
rename to mobile/datagenerators/alltime-numbers.py
diff --git a/mobile/cancelled-uploads.sql
b/mobile/datagenerators/cancelled-uploads.sql
similarity index 100%
rename from mobile/cancelled-uploads.sql
rename to mobile/datagenerators/cancelled-uploads.sql
diff --git a/mobile/deleted-uploads.py
b/mobile/datagenerators/deleted-uploads.py
similarity index 100%
rename from mobile/deleted-uploads.py
rename to mobile/datagenerators/deleted-uploads.py
diff --git a/mobile/error-correction.py
b/mobile/datagenerators/error-correction.py
similarity index 100%
rename from mobile/error-correction.py
rename to mobile/datagenerators/error-correction.py
diff --git a/mobile/error-uploads.sql b/mobile/datagenerators/error-uploads.sql
similarity index 100%
rename from mobile/error-uploads.sql
rename to mobile/datagenerators/error-uploads.sql
diff --git a/mobile/no-cats.sql b/mobile/datagenerators/no-cats.sql
similarity index 100%
rename from mobile/no-cats.sql
rename to mobile/datagenerators/no-cats.sql
diff --git a/mobile/performance-rendering.sql
b/mobile/datagenerators/performance-rendering.sql
similarity index 100%
rename from mobile/performance-rendering.sql
rename to mobile/datagenerators/performance-rendering.sql
diff --git a/mobile/share-attempts.sql
b/mobile/datagenerators/share-attempts.sql
similarity index 100%
rename from mobile/share-attempts.sql
rename to mobile/datagenerators/share-attempts.sql
diff --git a/mobile/successful-logins.sql
b/mobile/datagenerators/successful-logins.sql
similarity index 100%
rename from mobile/successful-logins.sql
rename to mobile/datagenerators/successful-logins.sql
diff --git a/mobile/successful-uploads.sql
b/mobile/datagenerators/successful-uploads.sql
similarity index 100%
rename from mobile/successful-uploads.sql
rename to mobile/datagenerators/successful-uploads.sql
diff --git a/mobile/unique-uploaders.sql
b/mobile/datagenerators/unique-uploaders.sql
similarity index 100%
rename from mobile/unique-uploaders.sql
rename to mobile/datagenerators/unique-uploaders.sql
diff --git a/mobile/uploaders-experience.py
b/mobile/datagenerators/uploaders-experience.py
similarity index 100%
rename from mobile/uploaders-experience.py
rename to mobile/datagenerators/uploaders-experience.py
diff --git a/datasources/30-day-uploads.json
b/mobile/datasources/30-day-uploads.json
similarity index 87%
rename from datasources/30-day-uploads.json
rename to mobile/datasources/30-day-uploads.json
index cd3092a..22c226a 100644
--- a/datasources/30-day-uploads.json
+++ b/mobile/datasources/30-day-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/30-day-uploads.csv";,
+ "url": "{{ output.url }}/30-day-uploads.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/cancelled-uploads.json
b/mobile/datasources/cancelled-uploads.json
similarity index 86%
rename from datasources/cancelled-uploads.json
rename to mobile/datasources/cancelled-uploads.json
index 29664f1..646676f 100644
--- a/datasources/cancelled-uploads.json
+++ b/mobile/datasources/cancelled-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/cancelled-uploads.csv";,
+ "url": "{{ output.url }}/cancelled-uploads.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/deleted-uploads.json
b/mobile/datasources/deleted-uploads.json
similarity index 86%
rename from datasources/deleted-uploads.json
rename to mobile/datasources/deleted-uploads.json
index e1e5bfe..54e78ce 100644
--- a/datasources/deleted-uploads.json
+++ b/mobile/datasources/deleted-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/deleted-uploads.csv";,
+ "url": "{{ output.url }}/deleted-uploads.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/error-uploads.json
b/mobile/datasources/error-uploads.json
similarity index 88%
rename from datasources/error-uploads.json
rename to mobile/datasources/error-uploads.json
index 0fa843a..4233c46 100644
--- a/datasources/error-uploads.json
+++ b/mobile/datasources/error-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/error-uploads.csv";,
+ "url": "{{ output.url }}/error-uploads.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/no-cats.json b/mobile/datasources/no-cats.json
similarity index 84%
rename from datasources/no-cats.json
rename to mobile/datasources/no-cats.json
index 3e09cee..3d3b875 100644
--- a/datasources/no-cats.json
+++ b/mobile/datasources/no-cats.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/no-cats.csv";,
+ "url": "{{ output.url }}/no-cats.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/performance-rendering.json
b/mobile/datasources/performance-rendering.json
similarity index 85%
rename from datasources/performance-rendering.json
rename to mobile/datasources/performance-rendering.json
index 87f54a6..f624b33 100644
--- a/datasources/performance-rendering.json
+++ b/mobile/datasources/performance-rendering.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/performance-rendering.csv";,
+ "url": "{{ output.url }}/performance-rendering.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/share-attempts.json
b/mobile/datasources/share-attempts.json
similarity index 86%
rename from datasources/share-attempts.json
rename to mobile/datasources/share-attempts.json
index d79d425..d0e16d3 100644
--- a/datasources/share-attempts.json
+++ b/mobile/datasources/share-attempts.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/share-attempts.csv";,
+ "url": "{{ output.url }}/share-attempts.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/successful-logins.json
b/mobile/datasources/successful-logins.json
similarity index 86%
rename from datasources/successful-logins.json
rename to mobile/datasources/successful-logins.json
index d419fba..209e953 100644
--- a/datasources/successful-logins.json
+++ b/mobile/datasources/successful-logins.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/successful-logins.csv";,
+ "url": "{{ output.url }}/successful-logins.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/successful-uploads.json
b/mobile/datasources/successful-uploads.json
similarity index 87%
rename from datasources/successful-uploads.json
rename to mobile/datasources/successful-uploads.json
index ddd0a8a..8a145e1 100644
--- a/datasources/successful-uploads.json
+++ b/mobile/datasources/successful-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/successful-uploads.csv";,
+ "url": "{{ output.url }}/successful-uploads.csv",
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/unique-uploaders.json
b/mobile/datasources/unique-uploaders.json
similarity index 87%
rename from datasources/unique-uploaders.json
rename to mobile/datasources/unique-uploaders.json
index a4f4d10..ea67e7c 100644
--- a/datasources/unique-uploaders.json
+++ b/mobile/datasources/unique-uploaders.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/unique-uploaders.csv";,
+ "url": "{{ output.url }}/unique-uploaders.csv",
"notes": "",
"format": "csv",
"slug": null,
--
To view, visit https://gerrit.wikimedia.org/r/60608
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I267961555de8608d7bb70aa74903e62d6025a7ac
Gerrit-PatchSet: 5
Gerrit-Project: analytics/limn-mobile-data
Gerrit-Branch: master
Gerrit-Owner: JGonera <[email protected]>
Gerrit-Reviewer: JGonera <[email protected]>
Gerrit-Reviewer: Yuvipanda <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
