Repository: incubator-airflow Updated Branches: refs/heads/master 8d7297573 -> ee248559f
Updated HiveServer2Hook.to_csv() to add fetch_size Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/a5c00b3f Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/a5c00b3f Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/a5c00b3f Branch: refs/heads/master Commit: a5c00b3f1581580818b585b21abd3df3fa68af64 Parents: 5e40d98 Author: Michael Musson <[email protected]> Authored: Thu May 19 16:31:27 2016 -0700 Committer: Michael Musson <[email protected]> Committed: Thu May 19 16:31:27 2016 -0700 ---------------------------------------------------------------------- airflow/hooks/hive_hooks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a5c00b3f/airflow/hooks/hive_hooks.py ---------------------------------------------------------------------- diff --git a/airflow/hooks/hive_hooks.py b/airflow/hooks/hive_hooks.py index 6ea5ba7..d712e16 100644 --- a/airflow/hooks/hive_hooks.py +++ b/airflow/hooks/hive_hooks.py @@ -489,7 +489,8 @@ class HiveServer2Hook(BaseHook): schema='default', delimiter=',', lineterminator='\r\n', - output_header=True): + output_header=True, + fetch_size=1000): schema = schema or 'default' with self.get_conn() as conn: with conn.cursor() as cur: @@ -504,7 +505,7 @@ class HiveServer2Hook(BaseHook): for c in cur.description]) i = 0 while True: - rows = [row for row in cur.fetchmany() if row] + rows = [row for row in cur.fetchmany(fetch_size) if row] if not rows: break
