This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 55976af32e Replace sequence concatination by unpacking in Airflow
providers (#33933)
55976af32e is described below
commit 55976af32ea7d09831e2bcd21c0f3814d9b0eb3f
Author: Hussein Awala <[email protected]>
AuthorDate: Thu Aug 31 09:07:46 2023 +0200
Replace sequence concatination by unpacking in Airflow providers (#33933)
---
airflow/providers/amazon/aws/transfers/redshift_to_s3.py | 4 +---
airflow/providers/apache/beam/hooks/beam.py | 10 +++-------
airflow/providers/apache/hive/hooks/hive.py | 2 +-
airflow/providers/apache/pig/hooks/pig.py | 2 +-
airflow/providers/cncf/kubernetes/pod_generator.py | 7 ++++---
airflow/providers/databricks/hooks/databricks_sql.py | 2 +-
airflow/providers/docker/operators/docker.py | 2 +-
airflow/providers/elasticsearch/log/es_task_handler.py | 2 +-
airflow/providers/google/cloud/operators/datafusion.py | 2 +-
airflow/providers/google/cloud/transfers/bigquery_to_mssql.py | 3 ++-
airflow/providers/google/cloud/transfers/bigquery_to_mysql.py | 5 +----
.../providers/google/cloud/transfers/bigquery_to_postgres.py | 5 +----
.../google/cloud/utils/mlengine_prediction_summary.py | 2 +-
13 files changed, 19 insertions(+), 29 deletions(-)
diff --git a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py
b/airflow/providers/amazon/aws/transfers/redshift_to_s3.py
index 6b6b4890c0..47e997cd4f 100644
--- a/airflow/providers/amazon/aws/transfers/redshift_to_s3.py
+++ b/airflow/providers/amazon/aws/transfers/redshift_to_s3.py
@@ -131,9 +131,7 @@ class RedshiftToS3Operator(BaseOperator):
)
if self.include_header and "HEADER" not in [uo.upper().strip() for uo
in self.unload_options]:
- self.unload_options = list(self.unload_options) + [
- "HEADER",
- ]
+ self.unload_options = [*self.unload_options, "HEADER"]
if self.redshift_data_api_kwargs:
for arg in ["sql", "parameters"]:
diff --git a/airflow/providers/apache/beam/hooks/beam.py
b/airflow/providers/apache/beam/hooks/beam.py
index 27957b5ca7..c27cc41710 100644
--- a/airflow/providers/apache/beam/hooks/beam.py
+++ b/airflow/providers/apache/beam/hooks/beam.py
@@ -188,9 +188,7 @@ class BeamHook(BaseHook):
process_line_callback: Callable[[str], None] | None = None,
working_directory: str | None = None,
) -> None:
- cmd = command_prefix + [
- f"--runner={self.runner}",
- ]
+ cmd = [*command_prefix, f"--runner={self.runner}"]
if variables:
cmd.extend(beam_options_to_args(variables))
run_beam_command(
@@ -261,7 +259,7 @@ class BeamHook(BaseHook):
requirements=py_requirements,
)
- command_prefix = [py_interpreter] + py_options + [py_file]
+ command_prefix = [py_interpreter, *py_options, py_file]
beam_version = (
subprocess.check_output(
@@ -506,9 +504,7 @@ class BeamAsyncHook(BeamHook):
command_prefix: list[str],
working_directory: str | None = None,
) -> int:
- cmd = command_prefix + [
- f"--runner={self.runner}",
- ]
+ cmd = [*command_prefix, f"--runner={self.runner}"]
if variables:
cmd.extend(beam_options_to_args(variables))
return await self.run_beam_command_async(
diff --git a/airflow/providers/apache/hive/hooks/hive.py
b/airflow/providers/apache/hive/hooks/hive.py
index 773ea4af7d..e46530d35a 100644
--- a/airflow/providers/apache/hive/hooks/hive.py
+++ b/airflow/providers/apache/hive/hooks/hive.py
@@ -163,7 +163,7 @@ class HiveCliHook(BaseHook):
hive_params_list = self.hive_cli_params.split()
- return [hive_bin] + cmd_extra + hive_params_list
+ return [hive_bin, *cmd_extra, *hive_params_list]
def _validate_beeline_parameters(self, conn):
if ":" in conn.host or "/" in conn.host or ";" in conn.host:
diff --git a/airflow/providers/apache/pig/hooks/pig.py
b/airflow/providers/apache/pig/hooks/pig.py
index 31e6006de3..ca457e9b89 100644
--- a/airflow/providers/apache/pig/hooks/pig.py
+++ b/airflow/providers/apache/pig/hooks/pig.py
@@ -79,7 +79,7 @@ class PigCliHook(BaseHook):
pig_opts_list = pig_opts.split()
pig_cmd.extend(pig_opts_list)
- pig_cmd.extend(["-f", fname] + cmd_extra)
+ pig_cmd.extend(["-f", fname, *cmd_extra])
if verbose:
self.log.info("%s", " ".join(pig_cmd))
diff --git a/airflow/providers/cncf/kubernetes/pod_generator.py
b/airflow/providers/cncf/kubernetes/pod_generator.py
index a96236359b..c824fda7b4 100644
--- a/airflow/providers/cncf/kubernetes/pod_generator.py
+++ b/airflow/providers/cncf/kubernetes/pod_generator.py
@@ -345,9 +345,10 @@ class PodGenerator:
client_container = extend_object_field(base_container,
client_container, "volume_devices")
client_container = merge_objects(base_container, client_container)
- return [client_container] + PodGenerator.reconcile_containers(
- base_containers[1:], client_containers[1:]
- )
+ return [
+ client_container,
+ *PodGenerator.reconcile_containers(base_containers[1:],
client_containers[1:]),
+ ]
@classmethod
def construct_pod(
diff --git a/airflow/providers/databricks/hooks/databricks_sql.py
b/airflow/providers/databricks/hooks/databricks_sql.py
index 172ce394f1..c592142b73 100644
--- a/airflow/providers/databricks/hooks/databricks_sql.py
+++ b/airflow/providers/databricks/hooks/databricks_sql.py
@@ -83,7 +83,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
def _get_extra_config(self) -> dict[str, Any | None]:
extra_params = copy(self.databricks_conn.extra_dejson)
- for arg in ["http_path", "session_configuration"] +
self.extra_parameters:
+ for arg in ["http_path", "session_configuration",
*self.extra_parameters]:
if arg in extra_params:
del extra_params[arg]
diff --git a/airflow/providers/docker/operators/docker.py
b/airflow/providers/docker/operators/docker.py
index 522410d278..3908ae54d3 100644
--- a/airflow/providers/docker/operators/docker.py
+++ b/airflow/providers/docker/operators/docker.py
@@ -335,7 +335,7 @@ class DockerOperator(BaseOperator):
with TemporaryDirectory(prefix="airflowtmp",
dir=self.host_tmp_dir) as host_tmp_dir_generated:
tmp_mount = Mount(self.tmp_dir, host_tmp_dir_generated, "bind")
try:
- return self._run_image_with_mounts(self.mounts +
[tmp_mount], add_tmp_variable=True)
+ return self._run_image_with_mounts([*self.mounts,
tmp_mount], add_tmp_variable=True)
except APIError as e:
if host_tmp_dir_generated in str(e):
self.log.warning(
diff --git a/airflow/providers/elasticsearch/log/es_task_handler.py
b/airflow/providers/elasticsearch/log/es_task_handler.py
index bbfdb4c2b8..d0d723c7dc 100644
--- a/airflow/providers/elasticsearch/log/es_task_handler.py
+++ b/airflow/providers/elasticsearch/log/es_task_handler.py
@@ -364,7 +364,7 @@ class ElasticsearchTaskHandler(FileTaskHandler,
ExternalLoggingMixin, LoggingMix
if self.json_format:
self.formatter = ElasticsearchJSONFormatter(
fmt=self.formatter._fmt,
- json_fields=self.json_fields + [self.offset_field],
+ json_fields=[*self.json_fields, self.offset_field],
extras={
"dag_id": str(ti.dag_id),
"task_id": str(ti.task_id),
diff --git a/airflow/providers/google/cloud/operators/datafusion.py
b/airflow/providers/google/cloud/operators/datafusion.py
index 5dce5ba290..d82a270871 100644
--- a/airflow/providers/google/cloud/operators/datafusion.py
+++ b/airflow/providers/google/cloud/operators/datafusion.py
@@ -783,7 +783,7 @@ class
CloudDataFusionStartPipelineOperator(GoogleCloudBaseOperator):
if success_states:
self.success_states = success_states
else:
- self.success_states = SUCCESS_STATES + [PipelineStates.RUNNING]
+ self.success_states = [*SUCCESS_STATES, PipelineStates.RUNNING]
def execute(self, context: Context) -> str:
hook = DataFusionHook(
diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
index d46a001c28..c1e3e9f550 100644
--- a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
+++ b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
@@ -45,7 +45,8 @@ class BigQueryToMsSqlOperator(BigQueryToSqlBaseOperator):
:param mssql_conn_id: reference to a specific mssql hook
"""
- template_fields: Sequence[str] =
tuple(BigQueryToSqlBaseOperator.template_fields) + (
+ template_fields: Sequence[str] = (
+ *BigQueryToSqlBaseOperator.template_fields,
"source_project_dataset_table",
)
operator_extra_links = (BigQueryTableLink(),)
diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py
b/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py
index c8aec07b0e..dda5f9b742 100644
--- a/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py
+++ b/airflow/providers/google/cloud/transfers/bigquery_to_mysql.py
@@ -40,10 +40,7 @@ class BigQueryToMySqlOperator(BigQueryToSqlBaseOperator):
:param mysql_conn_id: Reference to :ref:`mysql connection id
<howto/connection:mysql>`.
"""
- template_fields: Sequence[str] =
tuple(BigQueryToSqlBaseOperator.template_fields) + (
- "dataset_id",
- "table_id",
- )
+ template_fields: Sequence[str] =
(*BigQueryToSqlBaseOperator.template_fields, "dataset_id", "table_id")
def __init__(
self,
diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py
b/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py
index 3d1415687c..0fc06a50d8 100644
--- a/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py
+++ b/airflow/providers/google/cloud/transfers/bigquery_to_postgres.py
@@ -36,10 +36,7 @@ class BigQueryToPostgresOperator(BigQueryToSqlBaseOperator):
:param postgres_conn_id: Reference to :ref:`postgres connection id
<howto/connection:postgres>`.
"""
- template_fields: Sequence[str] =
tuple(BigQueryToSqlBaseOperator.template_fields) + (
- "dataset_id",
- "table_id",
- )
+ template_fields: Sequence[str] =
(*BigQueryToSqlBaseOperator.template_fields, "dataset_id", "table_id")
def __init__(
self,
diff --git
a/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
b/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
index 0b1118f82d..fc53a333c5 100644
--- a/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
+++ b/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
@@ -139,7 +139,7 @@ def MakeSummary(pcoll, metric_fn, metric_keys):
return (
pcoll
| "ApplyMetricFnPerInstance" >> beam.Map(metric_fn)
- | "PairWith1" >> beam.Map(lambda tup: tup + (1,))
+ | "PairWith1" >> beam.Map(lambda tup: (*tup, 1))
| "SumTuple" >>
beam.CombineGlobally(beam.combiners.TupleCombineFn(*([sum] * (len(metric_keys)
+ 1))))
| "AverageAndMakeDict"
>> beam.Map(