This is an automated email from the ASF dual-hosted git repository.

mobuchowski pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 5b082c38a6 openlineage, snowflake: add OpenLineage support for 
Snowflake (#31696)
5b082c38a6 is described below

commit 5b082c38a66b1a0b6b496e0d3b15a6684339e1d1
Author: JDarDagran <[email protected]>
AuthorDate: Fri Jul 21 18:44:03 2023 +0200

    openlineage, snowflake: add OpenLineage support for Snowflake (#31696)
    
    * Add OpenLineage support for SnowflakeOperator.
    
    Signed-off-by: Jakub Dardzinski <[email protected]>
    
    * Change how default schema is retrieved.
    
    Signed-off-by: Jakub Dardzinski <[email protected]>
    
    ---------
    
    Signed-off-by: Jakub Dardzinski <[email protected]>
---
 airflow/providers/snowflake/hooks/snowflake.py     | 71 +++++++++++++++++++++-
 generated/provider_dependencies.json               |  1 +
 tests/providers/snowflake/hooks/test_snowflake.py  | 30 +++++++++
 .../snowflake/operators/test_snowflake_sql.py      | 68 +++++++++++++++++++++
 4 files changed, 169 insertions(+), 1 deletion(-)

diff --git a/airflow/providers/snowflake/hooks/snowflake.py 
b/airflow/providers/snowflake/hooks/snowflake.py
index 76b39be6ac..3cb0c2a704 100644
--- a/airflow/providers/snowflake/hooks/snowflake.py
+++ b/airflow/providers/snowflake/hooks/snowflake.py
@@ -22,7 +22,8 @@ from contextlib import closing, contextmanager
 from functools import wraps
 from io import StringIO
 from pathlib import Path
-from typing import Any, Callable, Iterable, Mapping, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, TypeVar, 
overload
+from urllib.parse import urlparse
 
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
@@ -36,6 +37,9 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook, 
return_single_quer
 from airflow.utils.strings import to_boolean
 
 T = TypeVar("T")
+if TYPE_CHECKING:
+    from airflow.providers.openlineage.extractors import OperatorLineage
+    from airflow.providers.openlineage.sqlparser import DatabaseInfo
 
 
 def _try_to_boolean(value: Any):
@@ -448,3 +452,68 @@ class SnowflakeHook(DbApiHook):
         finally:
             if cursor is not None:
                 cursor.close()
+
+    def get_openlineage_database_info(self, connection) -> DatabaseInfo:
+        from airflow.providers.openlineage.sqlparser import DatabaseInfo
+
+        database = self.database or self._get_field(connection.extra_dejson, 
"database")
+
+        return DatabaseInfo(
+            scheme=self.get_openlineage_database_dialect(connection),
+            authority=self._get_openlineage_authority(connection),
+            information_schema_columns=[
+                "table_schema",
+                "table_name",
+                "column_name",
+                "ordinal_position",
+                "data_type",
+            ],
+            database=database,
+            is_information_schema_cross_db=True,
+            is_uppercase_names=True,
+        )
+
+    def get_openlineage_database_dialect(self, _) -> str:
+        return "snowflake"
+
+    def get_openlineage_default_schema(self) -> str | None:
+        """
+        Attempts to get current schema.
+
+        Usually ``SELECT CURRENT_SCHEMA();`` should work.
+        However, apparently you may set ``database`` without ``schema``
+        and get results from ``SELECT CURRENT_SCHEMAS();`` but not
+        from ``SELECT CURRENT_SCHEMA();``.
+        It still may return nothing if no database is set in connection.
+        """
+        schema = self._get_conn_params()["schema"]
+        if not schema:
+            current_schemas = self.get_first("SELECT 
PARSE_JSON(CURRENT_SCHEMAS())[0]::string;")[0]
+            if current_schemas:
+                _, schema = current_schemas.split(".")
+        return schema
+
+    def _get_openlineage_authority(self, _) -> str:
+        from openlineage.common.provider.snowflake import 
fix_snowflake_sqlalchemy_uri
+
+        uri = fix_snowflake_sqlalchemy_uri(self.get_uri())
+        return urlparse(uri).hostname
+
+    def get_openlineage_database_specific_lineage(self, _) -> OperatorLineage 
| None:
+        from openlineage.client.facet import ExternalQueryRunFacet
+
+        from airflow.providers.openlineage.extractors import OperatorLineage
+        from airflow.providers.openlineage.sqlparser import SQLParser
+
+        connection = self.get_connection(getattr(self, self.conn_name_attr))
+        namespace = 
SQLParser.create_namespace(self.get_database_info(connection))
+
+        if self.query_ids:
+            return OperatorLineage(
+                run_facets={
+                    "externalQuery": ExternalQueryRunFacet(
+                        externalQueryId=self.query_ids[0], source=namespace
+                    )
+                }
+            )
+        return None
diff --git a/generated/provider_dependencies.json 
b/generated/provider_dependencies.json
index 7a160c96dc..03d821eff1 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -814,6 +814,7 @@
     ],
     "cross-providers-deps": [
       "common.sql",
+      "openlineage",
       "slack"
     ],
     "excluded-python-versions": []
diff --git a/tests/providers/snowflake/hooks/test_snowflake.py 
b/tests/providers/snowflake/hooks/test_snowflake.py
index 6a738952d9..fef57bc8fc 100644
--- a/tests/providers/snowflake/hooks/test_snowflake.py
+++ b/tests/providers/snowflake/hooks/test_snowflake.py
@@ -621,3 +621,33 @@ class TestPytestSnowflakeHook:
             "extra__snowflake__private_key_content",
             "extra__snowflake__insecure_mode",
         ]
+
+    @pytest.mark.parametrize(
+        "returned_schema,expected_schema",
+        [([None], ""), (["DATABASE.SCHEMA"], "SCHEMA")],
+    )
+    @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook.get_first")
+    def test_get_openlineage_default_schema_with_no_schema_set(
+        self, mock_get_first, returned_schema, expected_schema
+    ):
+        connection_kwargs = {
+            **BASE_CONNECTION_KWARGS,
+            "schema": None,
+        }
+        with mock.patch.dict("os.environ", 
AIRFLOW_CONN_TEST_CONN=Connection(**connection_kwargs).get_uri()):
+            hook = SnowflakeHook(snowflake_conn_id="test_conn")
+            mock_get_first.return_value = returned_schema
+            assert hook.get_openlineage_default_schema() == expected_schema
+
+    @mock.patch("airflow.providers.common.sql.hooks.sql.DbApiHook.get_first")
+    def test_get_openlineage_default_schema_with_schema_set(self, 
mock_get_first):
+        with mock.patch.dict(
+            "os.environ", 
AIRFLOW_CONN_TEST_CONN=Connection(**BASE_CONNECTION_KWARGS).get_uri()
+        ):
+            hook = SnowflakeHook(snowflake_conn_id="test_conn")
+            assert hook.get_openlineage_default_schema() == 
BASE_CONNECTION_KWARGS["schema"]
+            mock_get_first.assert_not_called()
+
+            hook_with_schema_param = 
SnowflakeHook(snowflake_conn_id="test_conn", schema="my_schema")
+            assert hook_with_schema_param.get_openlineage_default_schema() == 
"my_schema"
+            mock_get_first.assert_not_called()
diff --git a/tests/providers/snowflake/operators/test_snowflake_sql.py 
b/tests/providers/snowflake/operators/test_snowflake_sql.py
index 3794bfc550..75edeba405 100644
--- a/tests/providers/snowflake/operators/test_snowflake_sql.py
+++ b/tests/providers/snowflake/operators/test_snowflake_sql.py
@@ -21,8 +21,12 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 from databricks.sql.types import Row
+from openlineage.client.facet import SchemaDatasetFacet, SchemaField, 
SqlJobFacet
+from openlineage.client.run import Dataset
 
+from airflow.models.connection import Connection
 from airflow.providers.common.sql.hooks.sql import fetch_all_handler
+from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
 from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator
 
 DATE = "2017-04-20"
@@ -138,3 +142,67 @@ def test_exec_success(sql, return_last, split_statement, 
hook_results, hook_desc
             return_last=return_last,
             split_statements=split_statement,
         )
+
+
+def test_execute_openlineage_events():
+    DB_NAME = "DATABASE"
+    DB_SCHEMA_NAME = "PUBLIC"
+
+    class SnowflakeHookForTests(SnowflakeHook):
+        get_conn = MagicMock(name="conn")
+        get_connection = MagicMock()
+
+        def get_first(self, *_):
+            return [f"{DB_NAME}.{DB_SCHEMA_NAME}"]
+
+    dbapi_hook = SnowflakeHookForTests()
+
+    class SnowflakeOperatorForTest(SnowflakeOperator):
+        def get_db_hook(self):
+            return dbapi_hook
+
+    sql = """CREATE TABLE IF NOT EXISTS popular_orders_day_of_week (
+        order_day_of_week VARCHAR(64) NOT NULL,
+        order_placed_on   TIMESTAMP NOT NULL,
+        orders_placed     INTEGER NOT NULL
+    );
+FORGOT TO COMMENT"""
+    op = SnowflakeOperatorForTest(task_id="snowflake-operator", sql=sql)
+    rows = [
+        (DB_SCHEMA_NAME, "POPULAR_ORDERS_DAY_OF_WEEK", "ORDER_DAY_OF_WEEK", 1, 
"TEXT"),
+        (DB_SCHEMA_NAME, "POPULAR_ORDERS_DAY_OF_WEEK", "ORDER_PLACED_ON", 2, 
"TIMESTAMP_NTZ"),
+        (DB_SCHEMA_NAME, "POPULAR_ORDERS_DAY_OF_WEEK", "ORDERS_PLACED", 3, 
"NUMBER"),
+    ]
+    dbapi_hook.get_connection.return_value = Connection(
+        conn_id="snowflake_default",
+        conn_type="snowflake",
+        extra={
+            "account": "test_account",
+            "region": "us-east",
+            "warehouse": "snow-warehouse",
+            "database": DB_NAME,
+        },
+    )
+    dbapi_hook.get_conn.return_value.cursor.return_value.fetchall.side_effect 
= [rows, []]
+
+    lineage = op.get_openlineage_facets_on_start()
+    assert len(lineage.inputs) == 0
+    assert lineage.outputs == [
+        Dataset(
+            namespace="snowflake://test_account.us-east.aws",
+            name=f"{DB_NAME}.{DB_SCHEMA_NAME}.POPULAR_ORDERS_DAY_OF_WEEK",
+            facets={
+                "schema": SchemaDatasetFacet(
+                    fields=[
+                        SchemaField(name="ORDER_DAY_OF_WEEK", type="TEXT"),
+                        SchemaField(name="ORDER_PLACED_ON", 
type="TIMESTAMP_NTZ"),
+                        SchemaField(name="ORDERS_PLACED", type="NUMBER"),
+                    ]
+                )
+            },
+        )
+    ]
+
+    assert lineage.job_facets == {"sql": SqlJobFacet(query=sql)}
+
+    assert lineage.run_facets["extractionError"].failedTasks == 1

Reply via email to