JGonera has uploaded a new change for review.
https://gerrit.wikimedia.org/r/58246
Change subject: Allow multiple folders as input
......................................................................
Allow multiple folders as input
Makes generate.py take more than one folder as its input. This can be
used either to separate *.sql/*.py files into several folders or to
provide additional config files to override the default config (values
defined in configs in subsequent folders override values from previous
configs).
Change-Id: I6ca8fd97c64df02c1336978f8c30346adb863e66
---
M .gitignore
M generate.py
2 files changed, 27 insertions(+), 20 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/limn-mobile-data
refs/changes/46/58246/1
diff --git a/.gitignore b/.gitignore
index 3418fd3..c07d99c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
env/
+datafiles/
.output
*.swp
+*.pyc
diff --git a/generate.py b/generate.py
index 7aeb743..dd244b9 100644
--- a/generate.py
+++ b/generate.py
@@ -14,13 +14,15 @@
class DataGenerator(object):
"""Executes queries and generates CSV reports based on YAML configs."""
- def __init__(self, folder_path):
- """Reads configuration 'config.yaml' in `folder_path`."""
- self.folder_path = folder_path
- config_file_path = os.path.join(folder_path, 'config.yaml')
- with io.open(config_file_path, encoding='utf-8') as config_file:
- self.config = yaml.load(config_file)
+ def __init__(self, *folders):
+ """Reads configuration 'config.yaml' in `folders`."""
+ self.config = {}
self.connections = {}
+ self.folders = folders
+ for folder in folders:
+ config_file_path = os.path.join(folder, 'config.yaml')
+ with io.open(config_file_path, encoding='utf-8') as config_file:
+ self.config.update(yaml.load(config_file))
def make_connection(self, name):
"""Opens a connection to a database using parameters specified in YAML
@@ -77,19 +79,22 @@
def execute(self):
"""Generates a CSV report by executing Python code and SQL queries."""
- global folder
-
for key, value in self.config['graphs'].iteritems():
- # Look for the sql first, then python
+ headers, rows = None, None
db_name = value.get('db', self.config['defaults']['db'])
- if os.path.exists(os.path.join(folder, key + '.sql')):
- file_path = os.path.join(folder, key + '.sql')
- headers, rows = self.execute_sql(file_path, db_name)
- elif os.path.exists(os.path.join(folder, key + '.py')):
- file_path = os.path.join(folder, key + '.py')
- headers, rows = self.execute_python(key, file_path)
- else:
+ for folder in self.folders:
+ # Look for the sql first, then python
+ if os.path.exists(os.path.join(folder, key + '.sql')):
+ file_path = os.path.join(folder, key + '.sql')
+ headers, rows = self.execute_sql(file_path, db_name)
+ break
+ elif os.path.exists(os.path.join(folder, key + '.py')):
+ file_path = os.path.join(folder, key + '.py')
+ headers, rows = self.execute_python(key, file_path)
+ break
+
+ if headers == None or rows == None:
raise ValueError("Can not find SQL or Python for %s" % key)
print "Generating %s (%s)" % (value['title'], file_path)
@@ -104,10 +109,10 @@
if __name__ == "__main__":
- if len(sys.argv) != 2: # FIXME: argparse please
- print "Usage: generate.py <folder with config.yaml and *.sql files>"
+ if len(sys.argv) < 2: # FIXME: argparse please
+ print "Usage: generate.py <folders with config.yaml and *.sql/*.py
files>"
sys.exit(1)
- folder = sys.argv[1]
- dg = DataGenerator(folder)
+ folders = sys.argv[1:]
+ dg = DataGenerator(*folders)
dg.execute()
--
To view, visit https://gerrit.wikimedia.org/r/58246
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6ca8fd97c64df02c1336978f8c30346adb863e66
Gerrit-PatchSet: 1
Gerrit-Project: analytics/limn-mobile-data
Gerrit-Branch: master
Gerrit-Owner: JGonera <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
