giftig commented on code in PR #31141: URL: https://github.com/apache/superset/pull/31141#discussion_r1863354140
########## superset/db_engine_specs/ydb.py: ########## @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Any, TYPE_CHECKING + +from sqlalchemy import types + +from superset.constants import TimeGrain +from superset.db_engine_specs.base import BaseEngineSpec +from superset.utils import json + +if TYPE_CHECKING: + from superset.models.core import Database + + +logger = logging.getLogger(__name__) + + +class YDBEngineSpec(BaseEngineSpec): + engine = "yql" + engine_aliases = {"ydb", "yql+ydb"} + engine_name = "YDB" + + default_driver = "ydb" + + sqlalchemy_uri_placeholder = "ydb://{host}:{port}/{database_name}" + + # pylint: disable=invalid-name + encrypted_extra_sensitive_fields = {"$.connect_args.credentials", "$.credentials"} + + disable_ssh_tunneling = False + + supports_file_upload = False + + allows_alias_in_orderby = True + + _time_grain_expressions = { + None: "{col}", + TimeGrain.SECOND: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1S')))", + TimeGrain.THIRTY_SECONDS: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30S')))", + TimeGrain.MINUTE: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1M')))", + TimeGrain.FIVE_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT5M')))", + TimeGrain.TEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT10M')))", + TimeGrain.FIFTEEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT15M')))", + TimeGrain.THIRTY_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30M')))", + TimeGrain.HOUR: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1H')))", + TimeGrain.DAY: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('P1D')))", + TimeGrain.WEEK: "DateTime::MakeDatetime(DateTime::StartOfWeek({col}))", + TimeGrain.MONTH: "DateTime::MakeDatetime(DateTime::StartOfMonth({col}))", + TimeGrain.QUARTER: "DateTime::MakeDatetime(DateTime::StartOfQuarter({col}))", + TimeGrain.YEAR: "DateTime::MakeDatetime(DateTime::StartOfYear({col}))", + } + + @classmethod + def epoch_to_dttm(cls) -> str: + return "DateTime::MakeDatetime({col})" + + @classmethod + def convert_dttm( + cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None + ) -> str | None: + sqla_type = cls.get_sqla_column_type(target_type) + + if isinstance(sqla_type, types.Date): + return f"DateTime::MakeDate(DateTime::ParseIso8601('{dttm.date().isoformat()}'))" + if isinstance(sqla_type, types.DateTime): + return f"""DateTime::MakeDatetime(DateTime::ParseIso8601('{dttm.isoformat(sep="T", timespec="seconds")}'))""" + return None + + @staticmethod + def update_params_from_encrypted_extra( + database: Database, + params: dict[str, Any], + ) -> None: + if not database.encrypted_extra: + return + + try: + encrypted_extra = json.loads(database.encrypted_extra) + connect_args = params.setdefault("connect_args", {}) + + if "protocol" in encrypted_extra: + connect_args["protocol"] = encrypted_extra["protocol"] + + if "credentials" in encrypted_extra: + credentials_info = encrypted_extra["credentials"] Review Comment: This argument seems unnecessary, could just be one line like L100 ########## superset/db_engine_specs/ydb.py: ########## @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Any, TYPE_CHECKING + +from sqlalchemy import types + +from superset.constants import TimeGrain +from superset.db_engine_specs.base import BaseEngineSpec +from superset.utils import json + +if TYPE_CHECKING: + from superset.models.core import Database + + +logger = logging.getLogger(__name__) + + +class YDBEngineSpec(BaseEngineSpec): + engine = "yql" + engine_aliases = {"ydb", "yql+ydb"} + engine_name = "YDB" + + default_driver = "ydb" + + sqlalchemy_uri_placeholder = "ydb://{host}:{port}/{database_name}" + + # pylint: disable=invalid-name + encrypted_extra_sensitive_fields = {"$.connect_args.credentials", "$.credentials"} + + disable_ssh_tunneling = False + + supports_file_upload = False + + allows_alias_in_orderby = True + + _time_grain_expressions = { + None: "{col}", + TimeGrain.SECOND: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1S')))", + TimeGrain.THIRTY_SECONDS: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30S')))", + TimeGrain.MINUTE: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1M')))", + TimeGrain.FIVE_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT5M')))", + TimeGrain.TEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT10M')))", + TimeGrain.FIFTEEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT15M')))", + TimeGrain.THIRTY_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30M')))", + TimeGrain.HOUR: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1H')))", + TimeGrain.DAY: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('P1D')))", + TimeGrain.WEEK: "DateTime::MakeDatetime(DateTime::StartOfWeek({col}))", + TimeGrain.MONTH: "DateTime::MakeDatetime(DateTime::StartOfMonth({col}))", + TimeGrain.QUARTER: "DateTime::MakeDatetime(DateTime::StartOfQuarter({col}))", + TimeGrain.YEAR: "DateTime::MakeDatetime(DateTime::StartOfYear({col}))", + } + + @classmethod + def epoch_to_dttm(cls) -> str: + return "DateTime::MakeDatetime({col})" + + @classmethod + def convert_dttm( + cls, target_type: str, dttm: datetime, db_extra: dict[str, Any] | None = None + ) -> str | None: + sqla_type = cls.get_sqla_column_type(target_type) + + if isinstance(sqla_type, types.Date): + return f"DateTime::MakeDate(DateTime::ParseIso8601('{dttm.date().isoformat()}'))" + if isinstance(sqla_type, types.DateTime): + return f"""DateTime::MakeDatetime(DateTime::ParseIso8601('{dttm.isoformat(sep="T", timespec="seconds")}'))""" + return None + + @staticmethod + def update_params_from_encrypted_extra( + database: Database, + params: dict[str, Any], + ) -> None: + if not database.encrypted_extra: + return + + try: + encrypted_extra = json.loads(database.encrypted_extra) + connect_args = params.setdefault("connect_args", {}) + + if "protocol" in encrypted_extra: Review Comment: Could you not just `connect_args["protocol"] = encrypted_extra.get("protocol")`? Or is it important the key be absent rather than `None`? ########## tests/unit_tests/db_engine_specs/test_ydb.py: ########## @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=unused-argument, import-outside-toplevel, protected-access +from __future__ import annotations + +from datetime import datetime +from typing import Any, Optional +from unittest.mock import Mock + +import pytest + +from superset.utils import json +from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm +from tests.unit_tests.fixtures.common import dttm # noqa: F401 + + +def test_epoch_to_dttm() -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec + + assert YDBEngineSpec.epoch_to_dttm() == "DateTime::MakeDatetime({col})" + + +@pytest.mark.parametrize( + "target_type,expected_result", + [ + ("Date", "DateTime::MakeDate(DateTime::ParseIso8601('2019-01-02'))"), + ( + "DateTime", + "DateTime::MakeDatetime(DateTime::ParseIso8601('2019-01-02T03:04:05'))", + ), + ("UnknownType", None), + ], +) +def test_convert_dttm( + target_type: str, + expected_result: Optional[str], + dttm: datetime, # noqa: F811 +) -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec as spec Review Comment: Does this import need to be local? ########## superset/db_engine_specs/ydb.py: ########## @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Any, TYPE_CHECKING + +from sqlalchemy import types + +from superset.constants import TimeGrain +from superset.db_engine_specs.base import BaseEngineSpec +from superset.utils import json + +if TYPE_CHECKING: + from superset.models.core import Database + + +logger = logging.getLogger(__name__) + + +class YDBEngineSpec(BaseEngineSpec): + engine = "yql" + engine_aliases = {"ydb", "yql+ydb"} + engine_name = "YDB" + + default_driver = "ydb" + + sqlalchemy_uri_placeholder = "ydb://{host}:{port}/{database_name}" + + # pylint: disable=invalid-name + encrypted_extra_sensitive_fields = {"$.connect_args.credentials", "$.credentials"} + + disable_ssh_tunneling = False + + supports_file_upload = False + + allows_alias_in_orderby = True + + _time_grain_expressions = { + None: "{col}", + TimeGrain.SECOND: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1S')))", + TimeGrain.THIRTY_SECONDS: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30S')))", + TimeGrain.MINUTE: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1M')))", + TimeGrain.FIVE_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT5M')))", + TimeGrain.TEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT10M')))", + TimeGrain.FIFTEEN_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT15M')))", + TimeGrain.THIRTY_MINUTES: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT30M')))", + TimeGrain.HOUR: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('PT1H')))", + TimeGrain.DAY: "DateTime::MakeDatetime(DateTime::StartOf({col}, Interval('P1D')))", + TimeGrain.WEEK: "DateTime::MakeDatetime(DateTime::StartOfWeek({col}))", + TimeGrain.MONTH: "DateTime::MakeDatetime(DateTime::StartOfMonth({col}))", + TimeGrain.QUARTER: "DateTime::MakeDatetime(DateTime::StartOfQuarter({col}))", + TimeGrain.YEAR: "DateTime::MakeDatetime(DateTime::StartOfYear({col}))", + } Review Comment: The pedant in me hates the fact that YDB apparently capitalises datetime two different ways in that function :( ########## tests/unit_tests/db_engine_specs/test_ydb.py: ########## @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=unused-argument, import-outside-toplevel, protected-access +from __future__ import annotations + +from datetime import datetime +from typing import Any, Optional +from unittest.mock import Mock + +import pytest + +from superset.utils import json +from tests.unit_tests.db_engine_specs.utils import assert_convert_dttm +from tests.unit_tests.fixtures.common import dttm # noqa: F401 + + +def test_epoch_to_dttm() -> None: + from superset.db_engine_specs.ydb import YDBEngineSpec + + assert YDBEngineSpec.epoch_to_dttm() == "DateTime::MakeDatetime({col})" Review Comment: I think we could probably live without this one since it's effectively just a constant string. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@superset.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: notifications-unsubscr...@superset.apache.org For additional commands, e-mail: notifications-h...@superset.apache.org
