This is an automated email from the ASF dual-hosted git repository. ash pushed a commit to branch revert-17201-support_alibabacloud in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e64e5bec2bb6fdbff64d3e943187b177df0ca50b Author: Ash Berlin-Taylor <[email protected]> AuthorDate: Wed Aug 11 16:12:08 2021 +0100 Revert "[AIRFLOW-17200] Add Alibaba Cloud OSS support (#17201)" This reverts commit 75ca6543da3754f0dfb53d059588ad66a2f8235a. --- CONTRIBUTING.rst | 2 +- INSTALL | 2 +- airflow/provider.yaml.schema.json | 1 - airflow/providers/alibaba/CHANGELOG.rst | 25 -- airflow/providers/alibaba/__init__.py | 17 -- airflow/providers/alibaba/cloud/__init__.py | 17 -- .../cloud/example_dags/example_oss_bucket.py | 39 --- .../cloud/example_dags/example_oss_object.py | 68 ----- airflow/providers/alibaba/cloud/hooks/__init__.py | 17 -- airflow/providers/alibaba/cloud/hooks/oss.py | 307 --------------------- .../providers/alibaba/cloud/operators/__init__.py | 16 -- airflow/providers/alibaba/cloud/operators/oss.py | 221 --------------- .../providers/alibaba/cloud/sensors/__init__.py | 16 -- airflow/providers/alibaba/cloud/sensors/oss_key.py | 93 ------- airflow/providers/alibaba/provider.yaml | 54 ---- airflow/utils/db.py | 12 - docs/apache-airflow-providers-alibaba/commits.rst | 27 -- .../connections/alibaba.rst | 55 ---- docs/apache-airflow-providers-alibaba/index.rst | 74 ----- .../operators/index.rst | 26 -- .../operators/oss.rst | 55 ---- .../apache-airflow-providers-alibaba/redirects.txt | 1 - docs/apache-airflow/extra-packages-ref.rst | 2 - docs/conf.py | 1 - .../alibaba/cloud/alibabacloud-oss.png | Bin 4968 -> 0 bytes docs/spelling_wordlist.txt | 4 - setup.py | 4 - tests/providers/alibaba/__init__.py | 16 -- tests/providers/alibaba/cloud/__init__.py | 16 -- tests/providers/alibaba/cloud/hooks/__init__.py | 16 -- tests/providers/alibaba/cloud/hooks/test_oss.py | 68 ----- .../providers/alibaba/cloud/operators/__init__.py | 16 -- .../providers/alibaba/cloud/operators/test_oss.py | 126 --------- tests/providers/alibaba/cloud/sensors/__init__.py | 16 -- .../alibaba/cloud/sensors/test_oss_key.py | 93 ------- tests/providers/alibaba/cloud/utils/__init__.py | 16 -- tests/providers/alibaba/cloud/utils/test_utils.py | 41 --- 37 files changed, 2 insertions(+), 1578 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d8e2c43..773e0b9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -585,7 +585,7 @@ This is the full list of those extras: .. START EXTRAS HERE -airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, +airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, diff --git a/INSTALL b/INSTALL index 47f48c3..c9af63f 100644 --- a/INSTALL +++ b/INSTALL @@ -89,7 +89,7 @@ The list of available extras: # START EXTRAS HERE -airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, +airbyte, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill, apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, diff --git a/airflow/provider.yaml.schema.json b/airflow/provider.yaml.schema.json index 840c902..6891206 100644 --- a/airflow/provider.yaml.schema.json +++ b/airflow/provider.yaml.schema.json @@ -66,7 +66,6 @@ "items": { "type": "string", "enum": [ - "alibaba", "apache", "aws", "azure", diff --git a/airflow/providers/alibaba/CHANGELOG.rst b/airflow/providers/alibaba/CHANGELOG.rst deleted file mode 100644 index cef7dda..0000000 --- a/airflow/providers/alibaba/CHANGELOG.rst +++ /dev/null @@ -1,25 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -Changelog ---------- - -1.0.0 -..... - -Initial version of the provider. diff --git a/airflow/providers/alibaba/__init__.py b/airflow/providers/alibaba/__init__.py deleted file mode 100644 index 217e5db..0000000 --- a/airflow/providers/alibaba/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/alibaba/cloud/__init__.py b/airflow/providers/alibaba/cloud/__init__.py deleted file mode 100644 index 217e5db..0000000 --- a/airflow/providers/alibaba/cloud/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py b/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py deleted file mode 100644 index 51a524e..0000000 --- a/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from airflow.models.dag import DAG -from airflow.providers.alibaba.cloud.operators.oss import OSSCreateBucketOperator, OSSDeleteBucketOperator -from airflow.utils.dates import days_ago - -with DAG( - dag_id='oss_bucket_dag', - start_date=days_ago(2), - max_active_runs=1, - tags=['example'], -) as dag: - - # [START howto_operator_oss_bucket] - create_bucket = OSSCreateBucketOperator( - oss_conn_id='oss_default', region='your region', task_id='task1', bucket_name='your bucket' - ) - - delete_bucket = OSSDeleteBucketOperator( - oss_conn_id='oss_default', region='your region', task_id='task2', bucket_name='your bucket' - ) - # [END howto_operator_oss_bucket] - - create_bucket >> delete_bucket diff --git a/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py b/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py deleted file mode 100644 index 4b69015..0000000 --- a/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from airflow.models.dag import DAG -from airflow.providers.alibaba.cloud.operators.oss import ( - OSSDeleteBatchObjectOperator, - OSSDeleteObjectOperator, - OSSDownloadObjectOperator, - OSSUploadObjectOperator, -) -from airflow.utils.dates import days_ago - -with DAG( - dag_id='oss_object_dag', - start_date=days_ago(2), - max_active_runs=1, - tags=['example'], -) as dag: - - create_object = OSSUploadObjectOperator( - file='your local file', - key='your oss key', - oss_conn_id='oss_default', - region='your region', - task_id='task1', - bucket_name='your bucket', - ) - - download_object = OSSDownloadObjectOperator( - file='your local file', - key='your oss key', - oss_conn_id='oss_default', - region='your region', - task_id='task2', - bucket_name='your bucket', - ) - - delete_object = OSSDeleteObjectOperator( - key='your oss key', - oss_conn_id='oss_default', - region='your region', - task_id='task3', - bucket_name='your bucket', - ) - - delete_batch_object = OSSDeleteBatchObjectOperator( - keys=['obj1', 'obj2', 'obj3'], - oss_conn_id='oss_default', - region='your region', - task_id='task4', - bucket_name='your bucket', - ) - - create_object >> download_object >> delete_object >> delete_batch_object diff --git a/airflow/providers/alibaba/cloud/hooks/__init__.py b/airflow/providers/alibaba/cloud/hooks/__init__.py deleted file mode 100644 index 217e5db..0000000 --- a/airflow/providers/alibaba/cloud/hooks/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/alibaba/cloud/hooks/oss.py b/airflow/providers/alibaba/cloud/hooks/oss.py deleted file mode 100644 index c318857..0000000 --- a/airflow/providers/alibaba/cloud/hooks/oss.py +++ /dev/null @@ -1,307 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from functools import wraps -from inspect import signature -from typing import TYPE_CHECKING, Callable, Optional, TypeVar, cast -from urllib.parse import urlparse - -import oss2 -from oss2.exceptions import ClientError - -from airflow.exceptions import AirflowException -from airflow.hooks.base import BaseHook - -if TYPE_CHECKING: - from airflow.models.connection import Connection - -T = TypeVar("T", bound=Callable) - - -def provide_bucket_name(func: T) -> T: - """ - Function decorator that unifies bucket name and key taken from the key - in case no bucket name and at least a key has been passed to the function. - """ - function_signature = signature(func) - - @wraps(func) - def wrapper(*args, **kwargs) -> T: - bound_args = function_signature.bind(*args, **kwargs) - self = args[0] - if 'bucket_name' not in bound_args.arguments or bound_args.arguments['bucket_name'] is None: - if self.oss_conn_id: - connection = self.get_connection(self.oss_conn_id) - if connection.schema: - bound_args.arguments['bucket_name'] = connection.schema - - return func(*bound_args.args, **bound_args.kwargs) - - return cast(T, wrapper) - - -def unify_bucket_name_and_key(func: T) -> T: - """ - Function decorator that unifies bucket name and key taken from the key - in case no bucket name and at least a key has been passed to the function. - """ - function_signature = signature(func) - - @wraps(func) - def wrapper(*args, **kwargs) -> T: - bound_args = function_signature.bind(*args, **kwargs) - - def get_key() -> Optional[str]: - if 'key' in bound_args.arguments: - return 'key' - raise ValueError('Missing key parameter!') - - key_name = get_key() - if 'bucket_name' not in bound_args.arguments or bound_args.arguments['bucket_name'] is None: - bound_args.arguments['bucket_name'], bound_args.arguments['key'] = OSSHook.parse_oss_url( - bound_args.arguments[key_name] - ) - - return func(*bound_args.args, **bound_args.kwargs) - - return cast(T, wrapper) - - -class OSSHook(BaseHook): - """Interact with Alibaba Cloud OSS, using the oss2 library.""" - - conn_name_attr = 'alibabacloud_conn_id' - default_conn_name = 'oss_default' - conn_type = 'oss' - hook_name = 'OSS' - - def __init__(self, region, oss_conn_id='oss_default', *args, **kwargs) -> None: - self.oss_conn_id = oss_conn_id - self.oss_conn = self.get_connection(oss_conn_id) - self.region = region - super().__init__(*args, **kwargs) - - def get_conn(self) -> "Connection": - """Returns connection for the hook.""" - return self.oss_conn - - @staticmethod - def parse_oss_url(ossurl: str) -> tuple: - """ - Parses the OSS Url into a bucket name and key. - - :param ossurl: The OSS Url to parse. - :rtype ossurl: str - :return: the parsed bucket name and key - :rtype: tuple of str - """ - parsed_url = urlparse(ossurl) - - if not parsed_url.netloc: - raise AirflowException(f'Please provide a bucket_name instead of "{ossurl}"') - - bucket_name = parsed_url.netloc - key = parsed_url.path.lstrip('/') - - return bucket_name, key - - @unify_bucket_name_and_key - def object_exists(self, key: str, bucket_name: Optional[str] = None) -> bool: - """ - Check if object exists. - - :param key: the path of the object - :type key: str - :param bucket_name: the name of the bucket - :type bucket_name: str - :return: True if it exists and False if not. - :rtype: bool - """ - try: - return self.get_bucket(bucket_name).object_exists(key) - except ClientError as e: - self.log.error(e.message) - return False - - @provide_bucket_name - def get_bucket(self, bucket_name: Optional[str] = None) -> oss2.api.Bucket: - """ - Returns a oss2.Bucket object - - :param bucket_name: the name of the bucket - :type bucket_name: str - :return: the bucket object to the bucket name. - :rtype: oss2.api.Bucket - """ - auth = self.get_credential() - return oss2.Bucket(auth, 'http://oss-' + self.region + '.aliyuncs.com', bucket_name) - - @unify_bucket_name_and_key - def load_string(self, key: str, content: str, bucket_name: Optional[str] = None) -> None: - """ - Loads a string to OSS - - :param key: the path of the object - :type key: str - :param content: str to set as content for the key. - :type content: str - :param bucket_name: the name of the bucket - :type bucket_name: str - """ - try: - self.get_bucket(bucket_name).put_object(key, content) - except Exception as e: - raise AirflowException(f"Errors: {e}") - - @unify_bucket_name_and_key - def upload_local_file( - self, - key: str, - file: str, - bucket_name: Optional[str] = None, - ) -> None: - """ - Upload a local file to OSS - - :param key: the OSS path of the object - :type key: str - :param file: local file to upload. - :type file: str - :param bucket_name: the name of the bucket - :type bucket_name: str - """ - try: - self.get_bucket(bucket_name).put_object_from_file(key, file) - except Exception as e: - raise AirflowException(f"Errors when upload file: {e}") - - @unify_bucket_name_and_key - def download_file( - self, - key: str, - local_file: str, - bucket_name: Optional[str] = None, - ) -> Optional[str]: - """ - Download file from OSS - - :param key: key of the file-like object to download. - :type key: str - :param local_file: local path + file name to save. - :type local_file: str - :param bucket_name: the name of the bucket - :type bucket_name: str - :return: the file name. - :rtype: str - """ - try: - self.get_bucket(bucket_name).get_object_to_file(key, local_file) - except Exception as e: - self.log.error(e) - return None - return local_file - - @unify_bucket_name_and_key - def delete_object( - self, - key: str, - bucket_name: Optional[str] = None, - ) -> None: - """ - Delete object from OSS - - :param key: key of the object to delete. - :type key: str - :param bucket_name: the name of the bucket - :type bucket_name: str - """ - try: - self.get_bucket(bucket_name).delete_object(key) - except Exception as e: - self.log.error(e) - raise AirflowException(f"Errors when deleting: {key}") - - @unify_bucket_name_and_key - def delete_objects( - self, - key: list, - bucket_name: Optional[str] = None, - ) -> None: - """ - Delete objects from OSS - - :param key: keys list of the objects to delete. - :type key: list of str - :param bucket_name: the name of the bucket - :type bucket_name: str - """ - try: - self.get_bucket(bucket_name).batch_delete_objects(key) - except Exception as e: - self.log.error(e) - raise AirflowException(f"Errors when deleting: {key}") - - @provide_bucket_name - def delete_bucket( - self, - bucket_name: Optional[str] = None, - ) -> None: - """ - Delete bucket from OSS - - :param bucket_name: the name of the bucket - :type bucket_name: str - """ - try: - self.get_bucket(bucket_name).delete_bucket() - except Exception as e: - self.log.error(e) - raise AirflowException(f"Errors when deleting: {bucket_name}") - - @provide_bucket_name - def create_bucket( - self, - bucket_name: Optional[str] = None, - ) -> None: - """ - Create bucket - - :param bucket_name: the name of the bucket - :type bucket_name: str - """ - try: - self.get_bucket(bucket_name).create_bucket() - except Exception as e: - self.log.error(e) - raise AirflowException(f"Errors when create bucket: {bucket_name}") - - def get_credential(self) -> oss2.auth.Auth: - extra_config = self.oss_conn.extra_dejson - auth_type = extra_config.get('auth_type', None) - if not auth_type: - raise Exception("No auth_type specified in extra_config. ") - - if auth_type == 'AK': - oss_access_key_id = extra_config.get('access_key_id', None) - oss_access_key_secret = extra_config.get('access_key_secret', None) - if not oss_access_key_id: - raise Exception("No access_key_id is specified for connection: " + self.oss_conn_id) - if not oss_access_key_secret: - raise Exception("No access_key_secret is specified for connection: " + self.oss_conn_id) - return oss2.Auth(oss_access_key_id, oss_access_key_secret) - else: - raise Exception("Unsupported auth_type: " + auth_type) diff --git a/airflow/providers/alibaba/cloud/operators/__init__.py b/airflow/providers/alibaba/cloud/operators/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/airflow/providers/alibaba/cloud/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/alibaba/cloud/operators/oss.py b/airflow/providers/alibaba/cloud/operators/oss.py deleted file mode 100644 index 2e5d318..0000000 --- a/airflow/providers/alibaba/cloud/operators/oss.py +++ /dev/null @@ -1,221 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""This module contains Alibaba Cloud OSS operators.""" -from typing import Optional - -from airflow.models import BaseOperator -from airflow.providers.alibaba.cloud.hooks.oss import OSSHook - - -class OSSCreateBucketOperator(BaseOperator): - """ - This operator creates an OSS bucket - - :param region: OSS region you want to create bucket - :type region: str - :param bucket_name: This is bucket name you want to create - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - def __init__( - self, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.oss_conn_id = oss_conn_id - self.region = region - self.bucket_name = bucket_name - - def execute(self, context): - oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - oss_hook.create_bucket(bucket_name=self.bucket_name) - - -class OSSDeleteBucketOperator(BaseOperator): - """ - This operator to delete an OSS bucket - - :param region: OSS region you want to create bucket - :type region: str - :param bucket_name: This is bucket name you want to delete - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - def __init__( - self, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.oss_conn_id = oss_conn_id - self.region = region - self.bucket_name = bucket_name - - def execute(self, context): - oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - oss_hook.delete_bucket(bucket_name=self.bucket_name) - - -class OSSUploadObjectOperator(BaseOperator): - """ - This operator to upload an file-like object - - :param key: the OSS path of the object - :type key: str - :param file: local file to upload. - :type file: str - :param region: OSS region you want to create bucket - :type region: str - :param bucket_name: This is bucket name you want to create - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - def __init__( - self, - key: str, - file: str, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.key = key - self.file = file - self.oss_conn_id = oss_conn_id - self.region = region - self.bucket_name = bucket_name - - def execute(self, context): - oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - oss_hook.upload_local_file(bucket_name=self.bucket_name, key=self.key, file=self.file) - - -class OSSDownloadObjectOperator(BaseOperator): - """ - This operator to Download an OSS object - - :param key: key of the object to download. - :type key: str - :param local_file: local path + file name to save. - :type local_file: str - :param region: OSS region - :type region: str - :param bucket_name: OSS bucket name - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - def __init__( - self, - key: str, - file: str, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.key = key - self.file = file - self.oss_conn_id = oss_conn_id - self.region = region - self.bucket_name = bucket_name - - def execute(self, context): - oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - oss_hook.download_file(bucket_name=self.bucket_name, key=self.key, local_file=self.file) - - -class OSSDeleteBatchObjectOperator(BaseOperator): - """ - This operator to delete OSS objects - - :param key: key list of the objects to delete. - :type key: str - :param region: OSS region - :type region: str - :param bucket_name: OSS bucket name - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - def __init__( - self, - keys: list, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.keys = keys - self.oss_conn_id = oss_conn_id - self.region = region - self.bucket_name = bucket_name - - def execute(self, context): - oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - oss_hook.delete_objects(bucket_name=self.bucket_name, key=self.keys) - - -class OSSDeleteObjectOperator(BaseOperator): - """ - This operator to delete an OSS object - - :param key: key of the object to delete. - :type key: str - :param region: OSS region - :type region: str - :param bucket_name: OSS bucket name - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - def __init__( - self, - key: str, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.key = key - self.oss_conn_id = oss_conn_id - self.region = region - self.bucket_name = bucket_name - - def execute(self, context): - oss_hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - oss_hook.delete_object(bucket_name=self.bucket_name, key=self.key) diff --git a/airflow/providers/alibaba/cloud/sensors/__init__.py b/airflow/providers/alibaba/cloud/sensors/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/airflow/providers/alibaba/cloud/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/airflow/providers/alibaba/cloud/sensors/oss_key.py b/airflow/providers/alibaba/cloud/sensors/oss_key.py deleted file mode 100644 index 22bd8e9..0000000 --- a/airflow/providers/alibaba/cloud/sensors/oss_key.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -try: - from functools import cached_property -except ImportError: - from cached_property import cached_property -from typing import Optional -from urllib.parse import urlparse - -from airflow.exceptions import AirflowException -from airflow.providers.alibaba.cloud.hooks.oss import OSSHook -from airflow.sensors.base import BaseSensorOperator - - -class OSSKeySensor(BaseSensorOperator): - """ - Waits for a key (a file-like instance on OSS) to be present in a OSS bucket. - OSS being a key/value it does not support folders. The path is just a key - a resource. - - :param bucket_key: The key being waited on. Supports full oss:// style url - or relative path from root level. When it's specified as a full oss:// - url, please leave bucket_name as `None`. - :type bucket_key: str - :param region: OSS region - :type region: str - :param bucket_name: OSS bucket name - :type bucket_name: str - :param oss_conn_id: The Airflow connection used for OSS credentials. - :type oss_conn_id: Optional[str] - """ - - template_fields = ('bucket_key', 'bucket_name') - - def __init__( - self, - bucket_key: str, - region: str, - bucket_name: Optional[str] = None, - oss_conn_id: Optional[str] = 'oss_default', - **kwargs, - ): - super().__init__(**kwargs) - - self.bucket_name = bucket_name - self.bucket_key = bucket_key - self.region = region - self.oss_conn_id = oss_conn_id - self.hook: Optional[OSSHook] = None - - def poke(self, context): - - if self.bucket_name is None: - parsed_url = urlparse(self.bucket_key) - if parsed_url.netloc == '': - raise AirflowException('If key is a relative path from root, please provide a bucket_name') - self.bucket_name = parsed_url.netloc - self.bucket_key = parsed_url.path.lstrip('/') - else: - parsed_url = urlparse(self.bucket_key) - if parsed_url.scheme != '' or parsed_url.netloc != '': - raise AirflowException( - 'If bucket_name is provided, bucket_key' - + ' should be relative path from root' - + ' level, rather than a full oss:// url' - ) - - self.log.info('Poking for key : oss://%s/%s', self.bucket_name, self.bucket_key) - return self.get_hook.object_exists(key=self.bucket_key, bucket_name=self.bucket_name) - - @cached_property - def get_hook(self) -> OSSHook: - """Create and return an OSSHook""" - if self.hook: - return self.hook - - self.hook = OSSHook(oss_conn_id=self.oss_conn_id, region=self.region) - return self.hook diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml deleted file mode 100644 index f93b545..0000000 --- a/airflow/providers/alibaba/provider.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -package-name: apache-airflow-providers-alibaba -name: Alibaba -description: | - Alibaba Cloud integration (including `Alibaba Cloud <https://www.alibabacloud.com//>`__). - -versions: - - 1.0.0 - -additional-dependencies: - - apache-airflow>=2.1.0 - -integrations: - - integration-name: Alibaba Cloud OSS - external-doc-url: https://www.alibabacloud.com/help/product/31815.htm - logo: /integration-logos/alibaba/cloud/alibabacloud-oss.png - how-to-guide: - - /docs/apache-airflow-providers-alibaba/operators/oss.rst - tags: [alibaba] - -operators: - - integration-name: Alibaba Cloud OSS - python-modules: - - airflow.providers.alibaba.cloud.operators.oss - -sensors: - - integration-name: Alibaba Cloud OSS - python-modules: - - airflow.providers.alibaba.cloud.sensors.oss_key - -hooks: - - integration-name: Alibaba Cloud OSS - python-modules: - - airflow.providers.alibaba.cloud.hooks.oss - -hook-class-names: - - airflow.providers.alibaba.cloud.hooks.oss.OSSHook diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 5777aa8..644fb00 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -399,18 +399,6 @@ def create_default_connections(session=None): ) merge_conn( Connection( - conn_id="oss_default", - conn_type="oss", - extra='''{ - "auth_type": "AK", - "access_key_id": "<ACCESS_KEY_ID>", - "access_key_secret": "<ACCESS_KEY_SECRET>"} - ''', - ), - session, - ) - merge_conn( - Connection( conn_id="pig_cli_default", conn_type="pig_cli", schema="default", diff --git a/docs/apache-airflow-providers-alibaba/commits.rst b/docs/apache-airflow-providers-alibaba/commits.rst deleted file mode 100644 index 7d5e85b..0000000 --- a/docs/apache-airflow-providers-alibaba/commits.rst +++ /dev/null @@ -1,27 +0,0 @@ - - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - - -Package apache-airflow-providers-alibaba ----------------------------------------- - -Alibaba integration (including `Alibabacloud <https://www.alibabacloud.com//>`__). - - -This is detailed commit list of changes for versions provider package: ``alibaba.cloud``. -For high-level changelog, see :doc:`package information including changelog <index>`. diff --git a/docs/apache-airflow-providers-alibaba/connections/alibaba.rst b/docs/apache-airflow-providers-alibaba/connections/alibaba.rst deleted file mode 100644 index d697b4d..0000000 --- a/docs/apache-airflow-providers-alibaba/connections/alibaba.rst +++ /dev/null @@ -1,55 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Alibaba Cloud Connection -======================== - -Authenticating to Alibaba Cloud -------------------------------- - -Authentication may be performed using `Security Token Service (STS) or a signed URL <https://www.alibabacloud.com/help/doc-detail/32033.htm>`_ . - -Default Connection IDs ----------------------- - -The default connection ID is ``oss_default``. - -Configuring the Connection --------------------------- - -Schema (optional) - Specify the default bucket name used for OSS hook. - -Extra (optional) - Specify the extra parameters (as json dictionary) that can be used in Alibaba Cloud - connection. The following parameters are all optional: - - * ``auth_type``: Auth type used to access Alibaba Cloud resource. Only support 'AK' now. - - * ``access_key_id``: Access key ID for Alibaba Cloud user. - * ``access_key_secret``: Access key secret for Alibaba Cloud user. - -Examples for the **Extra** field --------------------------------- - -.. code-block:: json - - { - "auth_type": "AK", - "access_key_id": "", - "access_key_secret": "" - } diff --git a/docs/apache-airflow-providers-alibaba/index.rst b/docs/apache-airflow-providers-alibaba/index.rst deleted file mode 100644 index 89380a6..0000000 --- a/docs/apache-airflow-providers-alibaba/index.rst +++ /dev/null @@ -1,74 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -``apache-airflow-providers-alibaba`` -==================================== - -Content -------- - -.. toctree:: - :maxdepth: 1 - :caption: Guides - - Connection types <connections/alibaba> - Operators <operators/index> - -.. toctree:: - :maxdepth: 1 - :caption: References - - Python API <_api/airflow/providers/alibaba/index> - -.. toctree:: - :maxdepth: 1 - :caption: Resources - - Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/alibaba/cloud/example_dags> - PyPI Repository <https://pypi.org/project/apache-airflow-providers-alibaba/> - -.. THE REMINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! - - -.. toctree:: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits <commits> - - -Provider package ----------------- - -This is a provider package for ``alibaba.cloud`` provider. All classes for this provider package -are in ``airflow.providers.alibaba.cloud`` python package. - -Installation ------------- - -You can install this package on top of an existing airflow 2.1+ installation via -``pip install apache-airflow-providers-alibaba`` - -PIP requirements ----------------- - -================== ==================== -PIP package Version required -================== ==================== -``apache-airflow`` ``>=2.1.0`` -``oss2`` ``>=2.14.0`` -================== ==================== diff --git a/docs/apache-airflow-providers-alibaba/operators/index.rst b/docs/apache-airflow-providers-alibaba/operators/index.rst deleted file mode 100644 index ccf9211..0000000 --- a/docs/apache-airflow-providers-alibaba/operators/index.rst +++ /dev/null @@ -1,26 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Alibaba Cloud Operators -======================= - - -.. toctree:: - :maxdepth: 1 - :glob: - - * diff --git a/docs/apache-airflow-providers-alibaba/operators/oss.rst b/docs/apache-airflow-providers-alibaba/operators/oss.rst deleted file mode 100644 index 661fc4b..0000000 --- a/docs/apache-airflow-providers-alibaba/operators/oss.rst +++ /dev/null @@ -1,55 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Alibaba Cloud OSS Operators -=========================== - -.. contents:: - :depth: 1 - :local: - -Overview --------- - -Airflow to Alibaba Cloud Object Storage Service (OSS) integration provides several operators to create and interact with OSS buckets. - - - :class:`~airflow.providers.alibaba.cloud.sensors.oss_key.OSSKeySensor` - - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSCreateBucketOperator` - - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDeleteBucketOperator` - - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSUploadObjectOperator` - - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDownloadObjectOperator` - - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDeleteBatchObjectOperator` - - :class:`~airflow.providers.alibaba.cloud.operators.oss.OSSDeleteObjectOperator` - -Create and Delete Alibaba Cloud OSS Buckets -------------------------------------------- - -Purpose -""""""" - -This example dag uses ``OSSCreateBucketOperator`` and ``OSSDeleteBucketOperator`` to create a -new OSS bucket with a given bucket name then delete it. - -Defining tasks -"""""""""""""" - -In the following code we create a new bucket and then delete the bucket. - -.. exampleinclude:: /../../airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py - :language: python - :start-after: [START howto_operator_oss_bucket] - :end-before: [END howto_operator_oss_bucket] diff --git a/docs/apache-airflow-providers-alibaba/redirects.txt b/docs/apache-airflow-providers-alibaba/redirects.txt deleted file mode 100644 index b6b18f5..0000000 --- a/docs/apache-airflow-providers-alibaba/redirects.txt +++ /dev/null @@ -1 +0,0 @@ -connections/index.rst connections/alibaba.rst diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index ae88728..29b6634 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -150,8 +150,6 @@ Those are extras that add dependencies needed for integration with external serv +=====================+=====================================================+=====================================================+ | airbyte | ``pip install 'apache-airflow[airbyte]'`` | Airbyte hooks and operators | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ -| alibaba | ``pip install 'apache-airflow[alibaba]'`` | Alibaba Cloud | -+---------------------+-----------------------------------------------------+-----------------------------------------------------+ | amazon | ``pip install 'apache-airflow[amazon]'`` | Amazon Web Services | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | asana | ``pip install 'apache-airflow[asana]'`` | Asana hooks and operators | diff --git a/docs/conf.py b/docs/conf.py index af642b8..dca3246 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -500,7 +500,6 @@ autodoc_mock_imports = [ 'jira', 'kubernetes', 'msrestazure', - 'oss2', 'pandas', 'pandas_gbq', 'paramiko', diff --git a/docs/integration-logos/alibaba/cloud/alibabacloud-oss.png b/docs/integration-logos/alibaba/cloud/alibabacloud-oss.png deleted file mode 100644 index abe40cd..0000000 Binary files a/docs/integration-logos/alibaba/cloud/alibabacloud-oss.png and /dev/null differ diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 47c82ba..d240ac9 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -4,7 +4,6 @@ AgentKey Airbnb Airbyte AirflowException -Alibaba Alphasort Analytics AnalyzeEntitiesResponse @@ -242,7 +241,6 @@ Nones NotFound Nullable OAuth -OSS Oauth Oauthlib Okta @@ -402,7 +400,6 @@ airbyte airflowignore ajax alertPolicies -alibaba allAuthenticatedUsers allUsers allowinsert @@ -1009,7 +1006,6 @@ orchestrator orgtbl orm os -oss ot overridable oversubscription diff --git a/setup.py b/setup.py index 2eaaa47..9d8597c 100644 --- a/setup.py +++ b/setup.py @@ -179,9 +179,6 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version # 'Start dependencies group' and 'Start dependencies group' are mark for ./scripts/ci/check_order_setup.py # If you change this mark you should also change ./scripts/ci/check_order_setup.py # Start dependencies group -alibaba = [ - 'oss2>=2.14.0', -] amazon = [ 'boto3>=1.15.0,<1.18.0', 'watchtower~=1.0.6', @@ -540,7 +537,6 @@ devel_hadoop = devel_minreq + hdfs + hive + kerberos + presto + webhdfs # Dict of all providers which are part of the Apache Airflow repository together with their requirements PROVIDERS_REQUIREMENTS: Dict[str, List[str]] = { 'airbyte': http_provider, - 'alibaba': alibaba, 'amazon': amazon, 'apache.beam': apache_beam, 'apache.cassandra': cassandra, diff --git a/tests/providers/alibaba/__init__.py b/tests/providers/alibaba/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/providers/alibaba/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/alibaba/cloud/__init__.py b/tests/providers/alibaba/cloud/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/providers/alibaba/cloud/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/alibaba/cloud/hooks/__init__.py b/tests/providers/alibaba/cloud/hooks/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/providers/alibaba/cloud/hooks/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/alibaba/cloud/hooks/test_oss.py b/tests/providers/alibaba/cloud/hooks/test_oss.py deleted file mode 100644 index b5e7d0b..0000000 --- a/tests/providers/alibaba/cloud/hooks/test_oss.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import os -import unittest - -import oss2 - -from airflow.exceptions import AirflowException -from airflow.providers.alibaba.cloud.hooks.oss import OSSHook -from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id - -TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default') -TEST_REGION = os.environ.get('TEST_OSS_REGION', 'cn-hangzhou') -TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket') - - -class TestOSSHook(unittest.TestCase): - def setUp(self): - try: - self.hook = OSSHook(region=TEST_REGION, oss_conn_id=TEST_CONN_ID) - self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET) - except AirflowException: - self.hook = None - except oss2.exceptions.ServerError as e: - if e.status == 403: - self.hook = None - - @skip_test_if_no_valid_conn_id - def test_init(self): - assert self.hook.oss_conn_id == TEST_CONN_ID - - @skip_test_if_no_valid_conn_id - def test_get_conn(self): - assert self.hook.get_conn() is not None - - @skip_test_if_no_valid_conn_id - def test_parse_oss_url(self): - parsed = self.hook.parse_oss_url(f"oss://{TEST_BUCKET}/this/is/not/a-real-key.txt") - print(parsed) - assert parsed == (TEST_BUCKET, "this/is/not/a-real-key.txt"), "Incorrect parsing of the oss url" - - @skip_test_if_no_valid_conn_id - def test_parse_oss_object_directory(self): - parsed = self.hook.parse_oss_url(f"oss://{TEST_BUCKET}/this/is/not/a-real-oss-directory/") - assert parsed == ( - TEST_BUCKET, - "this/is/not/a-real-oss-directory/", - ), "Incorrect parsing of the oss url" - - @skip_test_if_no_valid_conn_id - def test_get_bucket(self): - assert self.hook.get_bucket(TEST_BUCKET) is not None diff --git a/tests/providers/alibaba/cloud/operators/__init__.py b/tests/providers/alibaba/cloud/operators/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/providers/alibaba/cloud/operators/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/alibaba/cloud/operators/test_oss.py b/tests/providers/alibaba/cloud/operators/test_oss.py deleted file mode 100644 index c70aff3..0000000 --- a/tests/providers/alibaba/cloud/operators/test_oss.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import os -import unittest - -import oss2 - -from airflow.exceptions import AirflowException -from airflow.providers.alibaba.cloud.hooks.oss import OSSHook -from airflow.providers.alibaba.cloud.operators.oss import ( - OSSCreateBucketOperator, - OSSDeleteBatchObjectOperator, - OSSDeleteBucketOperator, - OSSDeleteObjectOperator, - OSSDownloadObjectOperator, - OSSUploadObjectOperator, -) -from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id - -TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default') -TEST_REGION = os.environ.get('TEST_OSS_REGION', 'cn-hangzhou') -TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket') -TEST_FILE_PATH = '/tmp/airflow-test' - - -class TestOSSOperator(unittest.TestCase): - def setUp(self): - self.create_bucket_operator = OSSCreateBucketOperator( - oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-1' - ) - self.delete_bucket_operator = OSSDeleteBucketOperator( - oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-2' - ) - try: - self.hook = OSSHook(region=TEST_REGION) - self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET) - except AirflowException: - self.hook = None - except oss2.exceptions.ServerError as e: - if e.status == 403: - self.hook = None - - @skip_test_if_no_valid_conn_id - def test_init(self): - assert self.create_bucket_operator.oss_conn_id == TEST_CONN_ID - - @skip_test_if_no_valid_conn_id - def test_create_delete_bucket(self): - self.create_bucket_operator.execute({}) - self.delete_bucket_operator.execute({}) - - @skip_test_if_no_valid_conn_id - def test_object(self): - self.create_bucket_operator.execute({}) - - upload_file = f'{TEST_FILE_PATH}_upload_1' - if not os.path.exists(upload_file): - with open(upload_file, 'w') as f: - f.write('test') - upload_object_operator = OSSUploadObjectOperator( - key='obj', - file=upload_file, - oss_conn_id=TEST_CONN_ID, - region=TEST_REGION, - bucket_name=TEST_BUCKET, - task_id='task-1', - ) - upload_object_operator.execute({}) - assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET) - - download_file = f'{TEST_FILE_PATH}_download_1' - download_object_operator = OSSDownloadObjectOperator( - key='obj', - file=download_file, - oss_conn_id=TEST_CONN_ID, - region=TEST_REGION, - bucket_name=TEST_BUCKET, - task_id='task-2', - ) - download_object_operator.execute({}) - assert os.path.exists(download_file) - - delete_object_operator = OSSDeleteObjectOperator( - key='obj', oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-3' - ) - delete_object_operator.execute({}) - assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET) is False - - upload_object_operator = OSSUploadObjectOperator( - key='obj', - file=upload_file, - oss_conn_id=TEST_CONN_ID, - region=TEST_REGION, - bucket_name=TEST_BUCKET, - task_id='task-4', - ) - upload_object_operator.execute({}) - assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET) - - delete_objects_operator = OSSDeleteBatchObjectOperator( - keys=['obj'], - oss_conn_id=TEST_CONN_ID, - region=TEST_REGION, - bucket_name=TEST_BUCKET, - task_id='task-5', - ) - delete_objects_operator.execute({}) - assert self.hook.object_exists(key='obj', bucket_name=TEST_BUCKET) is False - - self.delete_bucket_operator.execute({}) diff --git a/tests/providers/alibaba/cloud/sensors/__init__.py b/tests/providers/alibaba/cloud/sensors/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/providers/alibaba/cloud/sensors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/alibaba/cloud/sensors/test_oss_key.py b/tests/providers/alibaba/cloud/sensors/test_oss_key.py deleted file mode 100644 index 19ed30c..0000000 --- a/tests/providers/alibaba/cloud/sensors/test_oss_key.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -import os -import unittest - -import oss2 - -from airflow.exceptions import AirflowException -from airflow.providers.alibaba.cloud.hooks.oss import OSSHook -from airflow.providers.alibaba.cloud.operators.oss import ( - OSSCreateBucketOperator, - OSSDeleteBucketOperator, - OSSDeleteObjectOperator, - OSSUploadObjectOperator, -) -from airflow.providers.alibaba.cloud.sensors.oss_key import OSSKeySensor -from tests.providers.alibaba.cloud.utils.test_utils import skip_test_if_no_valid_conn_id - -TEST_CONN_ID = os.environ.get('TEST_OSS_CONN_ID', 'oss_default') -TEST_REGION = os.environ.get('TEST_OSS_REGION', 'cn-hangzhou') -TEST_BUCKET = os.environ.get('TEST_OSS_BUCKET', 'test-bucket') -TEST_FILE_PATH = '/tmp/airflow-test' - - -class TestOSSSensor(unittest.TestCase): - def setUp(self): - self.sensor = OSSKeySensor( - bucket_key='obj', - oss_conn_id=TEST_CONN_ID, - region=TEST_REGION, - bucket_name=TEST_BUCKET, - task_id='task-1', - ) - try: - self.hook = OSSHook(region=TEST_REGION, oss_conn_id=TEST_CONN_ID) - self.hook.object_exists(key='test-obj', bucket_name=TEST_BUCKET) - except AirflowException: - self.hook = None - except oss2.exceptions.ServerError as e: - if e.status == 403: - self.hook = None - - @skip_test_if_no_valid_conn_id - def test_init(self): - assert self.sensor.oss_conn_id == TEST_CONN_ID - - @skip_test_if_no_valid_conn_id - def test_poke(self): - create_bucket_operator = OSSCreateBucketOperator( - oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-2' - ) - create_bucket_operator.execute({}) - - upload_file = f'{TEST_FILE_PATH}_upload_1' - if not os.path.exists(upload_file): - with open(upload_file, 'w') as f: - f.write('test') - upload_object_operator = OSSUploadObjectOperator( - key='obj', - file=upload_file, - oss_conn_id=TEST_CONN_ID, - region=TEST_REGION, - bucket_name=TEST_BUCKET, - task_id='task-3', - ) - upload_object_operator.execute({}) - assert self.sensor.poke({}) - - delete_object_operator = OSSDeleteObjectOperator( - key='obj', oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-4' - ) - delete_object_operator.execute({}) - - delete_bucket_operator = OSSDeleteBucketOperator( - oss_conn_id=TEST_CONN_ID, region=TEST_REGION, bucket_name=TEST_BUCKET, task_id='task-5' - ) - delete_bucket_operator.execute({}) diff --git a/tests/providers/alibaba/cloud/utils/__init__.py b/tests/providers/alibaba/cloud/utils/__init__.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/providers/alibaba/cloud/utils/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/providers/alibaba/cloud/utils/test_utils.py b/tests/providers/alibaba/cloud/utils/test_utils.py deleted file mode 100644 index 82af3ce..0000000 --- a/tests/providers/alibaba/cloud/utils/test_utils.py +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from functools import wraps -from inspect import signature -from typing import Callable, TypeVar, cast - -T = TypeVar("T", bound=Callable) - - -def skip_test_if_no_valid_conn_id(func: T) -> T: - """ - Function decorator that skip this test function if no valid connection id is specified. - """ - function_signature = signature(func) - - @wraps(func) - def wrapper(*args, **kwargs) -> None: - bound_args = function_signature.bind(*args, **kwargs) - self = args[0] - - if self.hook is not None: - return func(*bound_args.args, **bound_args.kwargs) - else: - return None - - return cast(T, wrapper)
