Milimetric has submitted this change and it was merged.
Change subject: Using tempfile script for Hive add partition queries.
......................................................................
Using tempfile script for Hive add partition queries.
Change-Id: Ib7de34fd718e7c01fb47f37a1427667b97323fdd
---
M kraken-etl/util.py
1 file changed, 23 insertions(+), 5 deletions(-)
Approvals:
Milimetric: Verified; Looks good to me, approved
diff --git a/kraken-etl/util.py b/kraken-etl/util.py
index edca75a..7995650 100644
--- a/kraken-etl/util.py
+++ b/kraken-etl/util.py
@@ -3,6 +3,7 @@
import os
import re
import subprocess
+import tempfile
logger = logging.getLogger('kraken-etl-util')
@@ -187,7 +188,9 @@
"""
if partition_datetimes:
q = self.add_partitions_ddl(table, partition_datetimes)
- return self.query(q, True)
+ # This query could be large if there are many partiitons to create.
+ # Use a tempfile when adding partitions.
+ return self.query(q, use_tempfile=True)
else:
logger.info("Not creating any partitions for table %s. No
partition datetimes were given." % table)
@@ -252,14 +255,29 @@
return self.tables[table]['interval']
+ def query(self, query, check_return_code=True, use_tempfile=False):
+ """
+ Runs the given Hive query and returns stdout.
- def query(self, query, check_return_code=True):
- """Runs the given hive query and returns stdout"""
- return self.command(['-e', query], check_return_code)
+ If use_tempfile is True, the query will be written to
+ a temporary file and run as a Hive script.
+ """
+
+ if use_tempfile:
+ f = tempfile.NamedTemporaryFile(prefix='tmp-hive-query-',
suffix='.hiveql')
+ logger.debug('Writing Hive query to tempfile %s.' % f.name)
+ f.write(query)
+ f.flush()
+ out = self.script(f.name)
+ # NamedTemporaryFile will be deleted on close().
+ f.close()
+ return out
+ else:
+ return self.command(['-e', query], check_return_code)
def script(self, script, check_return_code=True):
- """Runs the contents of the given script in hive and returns stdout"""
+ """Runs the contents of the given script in hive and returns stdout."""
if not os.path.isfile(script):
raise RuntimeError("Hive script: {0} does not
exist.".format(script))
return self.command( ['-f', script], check_return_code)
--
To view, visit https://gerrit.wikimedia.org/r/90553
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib7de34fd718e7c01fb47f37a1427667b97323fdd
Gerrit-PatchSet: 2
Gerrit-Project: analytics/kraken
Gerrit-Branch: master
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
