This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5e6997ed45 Update dbt.py (#24218)
5e6997ed45 is described below
commit 5e6997ed45be0972bf5ea7dc06e4e1cef73b735a
Author: Vincent Koc <[email protected]>
AuthorDate: Mon Jun 6 19:48:17 2022 +1000
Update dbt.py (#24218)
---
airflow/providers/dbt/cloud/hooks/dbt.py | 22 ++++++++--------------
1 file changed, 8 insertions(+), 14 deletions(-)
diff --git a/airflow/providers/dbt/cloud/hooks/dbt.py
b/airflow/providers/dbt/cloud/hooks/dbt.py
index 13d10dbc62..d88c0053d3 100644
--- a/airflow/providers/dbt/cloud/hooks/dbt.py
+++ b/airflow/providers/dbt/cloud/hooks/dbt.py
@@ -168,28 +168,22 @@ class DbtCloudHook(HttpHook):
return session
def _paginate(self, endpoint: str, payload: Optional[Dict[str, Any]] =
None) -> List[Response]:
- results = []
response = self.run(endpoint=endpoint, data=payload)
resp_json = response.json()
limit = resp_json["extra"]["filters"]["limit"]
num_total_results = resp_json["extra"]["pagination"]["total_count"]
num_current_results = resp_json["extra"]["pagination"]["count"]
- results.append(response)
-
- if not num_current_results == num_total_results:
+ results = [response]
+ if num_current_results != num_total_results:
_paginate_payload = payload.copy() if payload else {}
_paginate_payload["offset"] = limit
- while True:
- if num_current_results < num_total_results:
- response = self.run(endpoint=endpoint,
data=_paginate_payload)
- resp_json = response.json()
- results.append(response)
- num_current_results +=
resp_json["extra"]["pagination"]["count"]
- _paginate_payload["offset"] += limit
- else:
- break
-
+ while not num_current_results >= num_total_results:
+ response = self.run(endpoint=endpoint, data=_paginate_payload)
+ resp_json = response.json()
+ results.append(response)
+ num_current_results +=
resp_json["extra"]["pagination"]["count"]
+ _paginate_payload["offset"] += limit
return results
def _run_and_get_response(
