Yuvipanda has uploaded a new change for review.
https://gerrit.wikimedia.org/r/64885
Change subject: Revert prior 3 commit
......................................................................
Revert prior 3 commit
Limn doesn't quite likea autogenerated datasources. And since
mobile-dashboard.wmflabs.org has been broken for a bit, reverting
Change-Id: Iad2f118c508a6f752e5b60a01547069b73f53db3
---
M .gitignore
M datasources/30-day-uploads.json
M datasources/cancelled-uploads.json
M datasources/deleted-uploads.json
M datasources/error-uploads.json
M datasources/no-cats.json
M datasources/performance-rendering.json
M datasources/share-attempts.json
M datasources/successful-logins.json
M datasources/successful-uploads.json
M datasources/unique-uploaders.json
M generate.py
R mobile/30-day-uploads.sql
R mobile/alltime-numbers.py
R mobile/cancelled-uploads.sql
M mobile/config.yaml
R mobile/deleted-uploads.py
R mobile/error-correction.py
R mobile/error-uploads.sql
R mobile/no-cats.sql
R mobile/performance-rendering.sql
R mobile/share-attempts.sql
R mobile/successful-logins.sql
R mobile/successful-uploads.sql
R mobile/unique-uploaders.sql
R mobile/uploaders-experience.py
26 files changed, 22 insertions(+), 50 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/limn-mobile-data
refs/changes/85/64885/1
diff --git a/.gitignore b/.gitignore
index 1867364..c07d99c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
-/env/
-/datafiles/
+env/
+datafiles/
.output
*.swp
*.pyc
diff --git a/datasources/30-day-uploads.json b/datasources/30-day-uploads.json
index 22c226a..cd3092a 100644
--- a/datasources/30-day-uploads.json
+++ b/datasources/30-day-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/30-day-uploads.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/30-day-uploads.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/cancelled-uploads.json
b/datasources/cancelled-uploads.json
index 646676f..29664f1 100644
--- a/datasources/cancelled-uploads.json
+++ b/datasources/cancelled-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/cancelled-uploads.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/cancelled-uploads.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/deleted-uploads.json b/datasources/deleted-uploads.json
index 54e78ce..e1e5bfe 100644
--- a/datasources/deleted-uploads.json
+++ b/datasources/deleted-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/deleted-uploads.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/deleted-uploads.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/error-uploads.json b/datasources/error-uploads.json
index 4233c46..0fa843a 100644
--- a/datasources/error-uploads.json
+++ b/datasources/error-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/error-uploads.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/error-uploads.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/no-cats.json b/datasources/no-cats.json
index 3d3b875..3e09cee 100644
--- a/datasources/no-cats.json
+++ b/datasources/no-cats.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/no-cats.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/no-cats.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/performance-rendering.json
b/datasources/performance-rendering.json
index f624b33..87f54a6 100644
--- a/datasources/performance-rendering.json
+++ b/datasources/performance-rendering.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/performance-rendering.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/performance-rendering.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/share-attempts.json b/datasources/share-attempts.json
index d0e16d3..d79d425 100644
--- a/datasources/share-attempts.json
+++ b/datasources/share-attempts.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/share-attempts.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/share-attempts.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/successful-logins.json
b/datasources/successful-logins.json
index 209e953..d419fba 100644
--- a/datasources/successful-logins.json
+++ b/datasources/successful-logins.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/successful-logins.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/successful-logins.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/successful-uploads.json
b/datasources/successful-uploads.json
index 8a145e1..ddd0a8a 100644
--- a/datasources/successful-uploads.json
+++ b/datasources/successful-uploads.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/successful-uploads.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/successful-uploads.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/datasources/unique-uploaders.json
b/datasources/unique-uploaders.json
index ea67e7c..a4f4d10 100644
--- a/datasources/unique-uploaders.json
+++ b/datasources/unique-uploaders.json
@@ -5,7 +5,7 @@
"step": "1d",
"end": "2013/03/19"
},
- "url": "{{ output.url }}/unique-uploaders.csv",
+ "url":
"http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles/unique-uploaders.csv";,
"notes": "",
"format": "csv",
"slug": null,
diff --git a/generate.py b/generate.py
index 7ad1600..95dfe9f 100644
--- a/generate.py
+++ b/generate.py
@@ -82,57 +82,30 @@
module = imp.load_source(name, file_path)
return module.execute(self)
- def generate_datasources(self):
- """Generates JSON files in datasources."""
- output_path = 'datasources'
- if not os.path.exists(output_path):
- os.makedirs(output_path)
-
- for key in self.config['graphs'].iterkeys():
- filename = key + '.json'
- print "Generating %s" % (filename)
- template_filename = os.path.join(self.folder, 'datasources',
filename)
-
- if os.path.exists(template_filename):
- output_filename = os.path.join(output_path, filename)
-
- with io.open(template_filename, encoding='utf-8') as
template_file:
- content = self.render(template_file.read())
-
- with open(output_filename, 'wb') as output_file:
- output_file.write(content)
-
- def generate_datafiles(self):
+ def execute(self):
"""Generates a CSV report by executing Python code and SQL queries."""
- output_path = self.config['output']['path']
- if not os.path.exists(output_path):
- os.makedirs(output_path)
-
for key, value in self.config['graphs'].iteritems():
+ # Look for the sql first, then python
db_name = value.get('db', self.config['defaults']['db'])
- print "Generating %s (%s)" % (value['title'], key)
-
- # Look for the sql first, then python
- if os.path.exists(os.path.join(self.folder, 'datagenerators', key
+ '.sql')):
- file_path = os.path.join(self.folder, 'datagenerators', key +
'.sql')
+ if os.path.exists(os.path.join(self.folder, key + '.sql')):
+ file_path = os.path.join(self.folder, key + '.sql')
headers, rows = self.execute_sql(file_path, db_name)
- elif os.path.exists(os.path.join(self.folder, 'datagenerators',
key + '.py')):
- file_path = os.path.join(self.folder, 'datagenerators', key +
'.py')
+ elif os.path.exists(os.path.join(self.folder, key + '.py')):
+ file_path = os.path.join(self.folder, key + '.py')
headers, rows = self.execute_python(key, file_path)
else:
raise ValueError("Can not find SQL or Python for %s" % key)
+ print "Generating %s (%s)" % (value['title'], file_path)
+
+ output_path = self.config['output']['path']
csv_filename = os.path.join(output_path, key + '.csv')
with open(csv_filename, 'wb') as csv_file:
writer = csv.writer(csv_file)
writer.writerow(headers)
writer.writerows(rows)
-
- def generate(self):
- self.generate_datasources()
- self.generate_datafiles()
if __name__ == "__main__":
@@ -142,4 +115,4 @@
args = parser.parse_args()
dg = DataGenerator(**vars(args))
- dg.generate()
+ dg.execute()
diff --git a/mobile/datagenerators/30-day-uploads.sql
b/mobile/30-day-uploads.sql
similarity index 100%
rename from mobile/datagenerators/30-day-uploads.sql
rename to mobile/30-day-uploads.sql
diff --git a/mobile/datagenerators/alltime-numbers.py
b/mobile/alltime-numbers.py
similarity index 100%
rename from mobile/datagenerators/alltime-numbers.py
rename to mobile/alltime-numbers.py
diff --git a/mobile/datagenerators/cancelled-uploads.sql
b/mobile/cancelled-uploads.sql
similarity index 100%
rename from mobile/datagenerators/cancelled-uploads.sql
rename to mobile/cancelled-uploads.sql
diff --git a/mobile/config.yaml b/mobile/config.yaml
index b31c8b7..3daa6b8 100644
--- a/mobile/config.yaml
+++ b/mobile/config.yaml
@@ -20,7 +20,6 @@
running_average: 30
output:
path: /a/limn-public-data/mobile/datafiles
- url: http://stat1001.wikimedia.org/limn-public-data/mobile/datafiles
defaults:
db: el
graphs:
diff --git a/mobile/datagenerators/deleted-uploads.py
b/mobile/deleted-uploads.py
similarity index 100%
rename from mobile/datagenerators/deleted-uploads.py
rename to mobile/deleted-uploads.py
diff --git a/mobile/datagenerators/error-correction.py
b/mobile/error-correction.py
similarity index 100%
rename from mobile/datagenerators/error-correction.py
rename to mobile/error-correction.py
diff --git a/mobile/datagenerators/error-uploads.sql b/mobile/error-uploads.sql
similarity index 100%
rename from mobile/datagenerators/error-uploads.sql
rename to mobile/error-uploads.sql
diff --git a/mobile/datagenerators/no-cats.sql b/mobile/no-cats.sql
similarity index 100%
rename from mobile/datagenerators/no-cats.sql
rename to mobile/no-cats.sql
diff --git a/mobile/datagenerators/performance-rendering.sql
b/mobile/performance-rendering.sql
similarity index 100%
rename from mobile/datagenerators/performance-rendering.sql
rename to mobile/performance-rendering.sql
diff --git a/mobile/datagenerators/share-attempts.sql
b/mobile/share-attempts.sql
similarity index 100%
rename from mobile/datagenerators/share-attempts.sql
rename to mobile/share-attempts.sql
diff --git a/mobile/datagenerators/successful-logins.sql
b/mobile/successful-logins.sql
similarity index 100%
rename from mobile/datagenerators/successful-logins.sql
rename to mobile/successful-logins.sql
diff --git a/mobile/datagenerators/successful-uploads.sql
b/mobile/successful-uploads.sql
similarity index 100%
rename from mobile/datagenerators/successful-uploads.sql
rename to mobile/successful-uploads.sql
diff --git a/mobile/datagenerators/unique-uploaders.sql
b/mobile/unique-uploaders.sql
similarity index 100%
rename from mobile/datagenerators/unique-uploaders.sql
rename to mobile/unique-uploaders.sql
diff --git a/mobile/datagenerators/uploaders-experience.py
b/mobile/uploaders-experience.py
similarity index 100%
rename from mobile/datagenerators/uploaders-experience.py
rename to mobile/uploaders-experience.py
--
To view, visit https://gerrit.wikimedia.org/r/64885
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iad2f118c508a6f752e5b60a01547069b73f53db3
Gerrit-PatchSet: 1
Gerrit-Project: analytics/limn-mobile-data
Gerrit-Branch: master
Gerrit-Owner: Yuvipanda <yuvipa...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
