turbaszek commented on a change in pull request #10990: URL: https://github.com/apache/airflow/pull/10990#discussion_r491921964
########## File path: tests/providers/google/cloud/transfers/test_mysql_to_gcs_system.py ########## @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from psycopg2 import ProgrammingError + +from airflow.providers.mysql.hooks.mysql import MySqlHook +from airflow.providers.google.cloud.example_dags.example_mysql_to_gcs import GCS_BUCKET +from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY +from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context + +CREATE_QUERY = """ +CREATE TABLE test_table +( + id int auto_increment primary key, + params json +); +""" + +LOAD_QUERY = """ +INSERT INTO test_table (id, params) +VALUES + ( + 1, '{ "customer": "Lily Bush", "items": {"product": "Diaper","qty": 24}}' + ), + ( + 2, '{ "customer": "Josh William", "items": {"product": "Toy Car","qty": 1}}' + ), + ( + 3, '{ "customer": "Mary Clark", "items": {"product": "Toy Train","qty": 2}}' + ); +""" +DELETE_QUERY = "DROP TABLE test_table;" + + [email protected]("mysql") [email protected]_file(GCP_GCS_KEY) +class MySQLToGCSSystemTest(GoogleSystemTest): + @staticmethod + def init_db(): + try: + hook = MySqlHook() + hook.run(CREATE_QUERY) + hook.run(LOAD_QUERY) + except ProgrammingError: Review comment: ```suggestion except (OperationalError, ProgrammingError): ``` ``` query = b'\nCREATE TABLE test_table\n(\n id int auto_increment primary key,\n params json\n);\n' def query(self, query): # Since _mysql releases GIL while querying, we need immutable buffer. if isinstance(query, bytearray): query = bytes(query) if self.waiter is not None: self.send_query(query) self.waiter(self.fileno()) self.read_query_result() else: > _mysql.connection.query(self, query) E _mysql_exceptions.OperationalError: (1050, "Table 'test_table' already exists") /usr/local/lib/python3.7/site-packages/MySQLdb/connections.py:280: OperationalError ``` ########## File path: tests/providers/google/cloud/transfers/test_mysql_to_gcs_system.py ########## @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from psycopg2 import ProgrammingError + +from airflow.providers.mysql.hooks.mysql import MySqlHook +from airflow.providers.google.cloud.example_dags.example_mysql_to_gcs import GCS_BUCKET +from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY +from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context + +CREATE_QUERY = """ +CREATE TABLE test_table +( + id int auto_increment primary key, + params json +); +""" + +LOAD_QUERY = """ +INSERT INTO test_table (id, params) +VALUES + ( + 1, '{ "customer": "Lily Bush", "items": {"product": "Diaper","qty": 24}}' + ), + ( + 2, '{ "customer": "Josh William", "items": {"product": "Toy Car","qty": 1}}' + ), + ( + 3, '{ "customer": "Mary Clark", "items": {"product": "Toy Train","qty": 2}}' + ); +""" +DELETE_QUERY = "DROP TABLE test_table;" + + [email protected]("mysql") [email protected]_file(GCP_GCS_KEY) +class MySQLToGCSSystemTest(GoogleSystemTest): + @staticmethod + def init_db(): + try: + hook = MySqlHook() + hook.run(CREATE_QUERY) + hook.run(LOAD_QUERY) + except ProgrammingError: + pass + + @staticmethod + def drop_db(): + hook = MySqlHook() + hook.run(DELETE_QUERY) + + @provide_gcp_context(GCP_GCS_KEY) + def setUp(self): + super().setUp() + self.create_gcs_bucket(GCS_BUCKET) + self.init_db() + + @provide_gcp_context(GCP_GCS_KEY) + def test_run_example_dag(self): + self.run_dag('example_mysql_to_gcs', CLOUD_DAG_FOLDER) Review comment: Looks good 🚀 ``` 1 passed, 28 warnings in 25.56s ``` ########## File path: docs/howto/operator/google/transfer/mysql_to_gcs.rst ########## @@ -0,0 +1,58 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +MySQL To Google Cloud Storage Operator +====================================== +The `Google Cloud Storage <https://cloud.google.com/storage/>`__ (GCS) service is +used to store large data from various applications. This page shows how to copy +data from MySQL to GCS. Review comment: ```suggestion data from MySQL database to GCS. ``` ########## File path: docs/howto/operator/google/transfer/mysql_to_gcs.rst ########## @@ -0,0 +1,58 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +MySQL To Google Cloud Storage Operator +====================================== +The `Google Cloud Storage <https://cloud.google.com/storage/>`__ (GCS) service is +used to store large data from various applications. This page shows how to copy +data from MySQL to GCS. + +.. contents:: + :depth: 1 + :local: + + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include::/howto/operator/google/_partials/prerequisite_tasks.rst + +.. _howto/operator:MySQLToGCSOperator: + +MySQLToGCSOperator +~~~~~~~~~~~~~~~~~~ + +:class:`~airflow.providers.google.cloud.transfers.mysql_to_gcs.MySQLToGCSOperator` allows you to upload +data from MySQL to GCS. + +When you use this operator, you can optionally compress the data being uploaded. Review comment: ```suggestion When you use this operator, you can optionally compress the data being uploaded to gzip format. ``` ########## File path: airflow/providers/google/cloud/example_dags/example_mysql_to_gcs.py ########## @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from airflow import models +from airflow.providers.google.cloud.transfers.mysql_to_gcs import MySQLToGCSOperator +from airflow.utils import dates + +GCS_BUCKET = os.environ.get("GCP_GCS_BUCKET", "example-bucket-name") Review comment: ```suggestion GCS_BUCKET = os.environ.get("GCP_GCS_BUCKET", "example-airflow-mysql-gcs") ``` More unique :) ########## File path: tests/providers/google/cloud/transfers/test_mysql_to_gcs_system.py ########## @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from psycopg2 import ProgrammingError + +from airflow.providers.mysql.hooks.mysql import MySqlHook +from airflow.providers.google.cloud.example_dags.example_mysql_to_gcs import GCS_BUCKET +from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY +from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context + +CREATE_QUERY = """ +CREATE TABLE test_table +( + id int auto_increment primary key, + params json +); +""" + +LOAD_QUERY = """ +INSERT INTO test_table (id, params) +VALUES + ( + 1, '{ "customer": "Lily Bush", "items": {"product": "Diaper","qty": 24}}' + ), + ( + 2, '{ "customer": "Josh William", "items": {"product": "Toy Car","qty": 1}}' + ), + ( + 3, '{ "customer": "Mary Clark", "items": {"product": "Toy Train","qty": 2}}' + ); +""" +DELETE_QUERY = "DROP TABLE test_table;" + + [email protected]("mysql") [email protected]_file(GCP_GCS_KEY) +class MySQLToGCSSystemTest(GoogleSystemTest): + @staticmethod + def init_db(): + try: + hook = MySqlHook() + hook.run(CREATE_QUERY) + hook.run(LOAD_QUERY) + except (OperationalError, ProgrammingError): Review comment: I'm afraid there will be an error as we are not importing `OperationalError` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
