Repository: incubator-airflow Updated Branches: refs/heads/master e83b01551 -> bb287ecf5
[AIRFLOW-2226] Rename google_cloud_storage_default to google_cloud_default The Google cloud operators uses both google_cloud_storage_default and google_cloud_default as a default conn_id. This is confusing and the google_cloud_storage_default conn_id isnt initialized by default in db.py Therefore we rename the google_cloud_storage_default to google_cloud_default for simplicity and convenience Closes #3141 from Fokko/airflow-2226 Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/bb287ecf Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/bb287ecf Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/bb287ecf Branch: refs/heads/master Commit: bb287ecf5bd1777124801c14750c6b685cc27c76 Parents: e83b015 Author: Fokko Driesprong <[email protected]> Authored: Mon Mar 19 22:02:10 2018 +0100 Committer: Fokko Driesprong <[email protected]> Committed: Mon Mar 19 22:02:10 2018 +0100 ---------------------------------------------------------------------- UPDATING.md | 4 ++++ airflow/contrib/hooks/gcs_hook.py | 2 +- airflow/contrib/operators/bigquery_operator.py | 4 ++-- airflow/contrib/operators/file_to_gcs.py | 4 ++-- airflow/contrib/operators/gcs_copy_operator.py | 2 +- airflow/contrib/operators/gcs_download_operator.py | 2 +- airflow/contrib/operators/gcs_list_operator.py | 2 +- airflow/contrib/operators/gcs_operator.py | 2 +- airflow/contrib/operators/gcs_to_bq.py | 2 +- airflow/contrib/operators/gcs_to_gcs.py | 2 +- airflow/contrib/operators/mysql_to_gcs.py | 2 +- airflow/contrib/operators/postgres_to_gcs_operator.py | 2 +- airflow/contrib/sensors/gcs_sensor.py | 6 +++--- 13 files changed, 20 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/UPDATING.md ---------------------------------------------------------------------- diff --git a/UPDATING.md b/UPDATING.md index e03243f..21581bf 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -24,6 +24,10 @@ Dataflow job labeling is now supported in Dataflow{Java,Python}Operator with a d "airflow-version" label, please upgrade your google-cloud-dataflow or apache-beam version to 2.2.0 or greater. +### Google cloud connection string + +With Airflow 1.9 or lower there where two connection strings for the Google Cloud operators, both `google_cloud_storage_default` and `google_cloud_default`. This can be confusing and therefore the `google_cloud_storage_default` connection id has been replaced with `google_cloud_default` to make the connection id consistent across Airflow. + ## Airflow 1.9 ### SSH Hook updates, along with new SSH Operator & SFTP Operator http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/hooks/gcs_hook.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/hooks/gcs_hook.py b/airflow/contrib/hooks/gcs_hook.py index 8c1e7bb..6a53b1d 100644 --- a/airflow/contrib/hooks/gcs_hook.py +++ b/airflow/contrib/hooks/gcs_hook.py @@ -27,7 +27,7 @@ class GoogleCloudStorageHook(GoogleCloudBaseHook): """ def __init__(self, - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None): super(GoogleCloudStorageHook, self).__init__(google_cloud_storage_conn_id, delegate_to) http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/bigquery_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py index ef1ada0..6f1cc6c 100644 --- a/airflow/contrib/operators/bigquery_operator.py +++ b/airflow/contrib/operators/bigquery_operator.py @@ -242,7 +242,7 @@ class BigQueryCreateEmptyTableOperator(BaseOperator): gcs_schema_object=None, time_partitioning={}, bigquery_conn_id='bigquery_default', - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): @@ -376,7 +376,7 @@ class BigQueryCreateExternalTableOperator(BaseOperator): allow_quoted_newlines=False, allow_jagged_rows=False, bigquery_conn_id='bigquery_default', - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, src_fmt_configs={}, *args, **kwargs): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/file_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py index b9f68f9..eaf8c9d 100644 --- a/airflow/contrib/operators/file_to_gcs.py +++ b/airflow/contrib/operators/file_to_gcs.py @@ -36,13 +36,13 @@ class FileToGoogleCloudStorageOperator(BaseOperator): :type delegate_to: string """ template_fields = ('src', 'dst', 'bucket') - + @apply_defaults def __init__(self, src, dst, bucket, - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', mime_type='application/octet-stream', delegate_to=None, *args, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_copy_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_copy_operator.py b/airflow/contrib/operators/gcs_copy_operator.py index 4922708..842d6dc 100644 --- a/airflow/contrib/operators/gcs_copy_operator.py +++ b/airflow/contrib/operators/gcs_copy_operator.py @@ -67,7 +67,7 @@ class GoogleCloudStorageCopyOperator(BaseOperator): source_files_delimiter=None, destination_bucket=None, destination_directory='', - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_download_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_download_operator.py b/airflow/contrib/operators/gcs_download_operator.py index 75de608..f6276b5 100644 --- a/airflow/contrib/operators/gcs_download_operator.py +++ b/airflow/contrib/operators/gcs_download_operator.py @@ -53,7 +53,7 @@ class GoogleCloudStorageDownloadOperator(BaseOperator): object, filename=None, store_to_xcom_key=None, - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_list_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_list_operator.py b/airflow/contrib/operators/gcs_list_operator.py index 3c49cc1..a708524 100644 --- a/airflow/contrib/operators/gcs_list_operator.py +++ b/airflow/contrib/operators/gcs_list_operator.py @@ -60,7 +60,7 @@ class GoogleCloudStorageListOperator(BaseOperator): bucket, prefix=None, delimiter=None, - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_operator.py b/airflow/contrib/operators/gcs_operator.py index d92d125..f03b5ec 100644 --- a/airflow/contrib/operators/gcs_operator.py +++ b/airflow/contrib/operators/gcs_operator.py @@ -85,7 +85,7 @@ class GoogleCloudStorageCreateBucketOperator(BaseOperator): location='US', project_id=None, labels=None, - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_to_bq.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py index be15e52..3126ae0 100644 --- a/airflow/contrib/operators/gcs_to_bq.py +++ b/airflow/contrib/operators/gcs_to_bq.py @@ -135,7 +135,7 @@ class GoogleCloudStorageToBigQueryOperator(BaseOperator): allow_jagged_rows=False, max_id_key=None, bigquery_conn_id='bigquery_default', - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, schema_update_options=(), src_fmt_configs={}, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/gcs_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/gcs_to_gcs.py b/airflow/contrib/operators/gcs_to_gcs.py index 8db3067..b6e49dc 100644 --- a/airflow/contrib/operators/gcs_to_gcs.py +++ b/airflow/contrib/operators/gcs_to_gcs.py @@ -65,7 +65,7 @@ class GoogleCloudStorageToGoogleCloudStorageOperator(BaseOperator): destination_bucket=None, destination_object=None, move_object=False, - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/mysql_to_gcs.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/mysql_to_gcs.py b/airflow/contrib/operators/mysql_to_gcs.py index 9ba84c7..2249b9c 100644 --- a/airflow/contrib/operators/mysql_to_gcs.py +++ b/airflow/contrib/operators/mysql_to_gcs.py @@ -45,7 +45,7 @@ class MySqlToGoogleCloudStorageOperator(BaseOperator): schema_filename=None, approx_max_file_size_bytes=1900000000, mysql_conn_id='mysql_default', - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', schema=None, delegate_to=None, *args, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/operators/postgres_to_gcs_operator.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/operators/postgres_to_gcs_operator.py b/airflow/contrib/operators/postgres_to_gcs_operator.py index ab6fdf4..8224b36 100644 --- a/airflow/contrib/operators/postgres_to_gcs_operator.py +++ b/airflow/contrib/operators/postgres_to_gcs_operator.py @@ -44,7 +44,7 @@ class PostgresToGoogleCloudStorageOperator(BaseOperator): schema_filename=None, approx_max_file_size_bytes=1900000000, postgres_conn_id='postgres_default', - google_cloud_storage_conn_id='google_cloud_storage_default', + google_cloud_storage_conn_id='google_cloud_default', delegate_to=None, parameters=None, *args, http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/bb287ecf/airflow/contrib/sensors/gcs_sensor.py ---------------------------------------------------------------------- diff --git a/airflow/contrib/sensors/gcs_sensor.py b/airflow/contrib/sensors/gcs_sensor.py index d91e2b1..aa7afd3 100644 --- a/airflow/contrib/sensors/gcs_sensor.py +++ b/airflow/contrib/sensors/gcs_sensor.py @@ -28,7 +28,7 @@ class GoogleCloudStorageObjectSensor(BaseSensorOperator): self, bucket, object, # pylint:disable=redefined-builtin - google_cloud_conn_id='google_cloud_storage_default', + google_cloud_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): @@ -84,7 +84,7 @@ class GoogleCloudStorageObjectUpdatedSensor(BaseSensorOperator): bucket, object, # pylint:disable=redefined-builtin ts_func=ts_function, - google_cloud_conn_id='google_cloud_storage_default', + google_cloud_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs): @@ -135,7 +135,7 @@ class GoogleCloudStoragePrefixSensor(BaseSensorOperator): self, bucket, prefix, - google_cloud_conn_id='google_cloud_storage_default', + google_cloud_conn_id='google_cloud_default', delegate_to=None, *args, **kwargs):
