This is an automated email from the ASF dual-hosted git repository. Gerrrr pushed a commit to branch dependency-extras in repository https://gitbox.apache.org/repos/asf/otava.git
commit 46164f8d7a0d3c8fafc514fcaecf7b649734c62d Author: Alex Sorokoumov <[email protected]> AuthorDate: Fri Aug 28 20:39:37 2026 -0700 Make external service clients optional The default Otava installation currently pulls every database and notification client even when users only need the analysis library, bundled CLI, and built-in data sources. Besides the unnecessary installation size, eager imports make basic CLI use depend on integrations that are not configured. Move BigQuery, PostgreSQL, InfluxDB, Grafana, and Slack clients to additive extras. Load each client only when its integration is used and report the exact extra needed when it is missing. Keep the Docker image fully featured with the all extra, add a no-extras wheel smoke job, and document the install choices in the README, installation and getting-started guides, and integration-specific pages. Closes #175 and #55. --- .github/workflows/python-app.yml | 27 ++++++++ Dockerfile | 5 +- README.md | 20 ++++++ docs/BIG_QUERY.md | 6 ++ docs/GETTING_STARTED.md | 12 ++++ docs/GRAFANA.md | 8 +++ docs/INFLUXDB.md | 6 ++ docs/INSTALL.md | 22 ++++++ docs/POSTGRESQL.md | 6 ++ otava/_optional.py | 39 +++++++++++ otava/bigquery.py | 27 ++++++-- otava/grafana.py | 17 +++-- otava/importer.py | 8 +-- otava/influxdb.py | 17 ++++- otava/main.py | 22 +++--- otava/postgres.py | 17 ++++- otava/slack.py | 16 ++++- pyproject.toml | 27 ++++++-- tests/core_install_smoke.py | 130 ++++++++++++++++++++++++++++++++++++ tests/optional_dependencies_test.py | 116 ++++++++++++++++++++++++++++++++ uv.lock | 44 +++++++++--- 21 files changed, 545 insertions(+), 47 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f23a6e1..9843ecd 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -30,6 +30,33 @@ permissions: contents: read jobs: + core-install: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.14 + uses: actions/setup-python@v5 + with: + python-version: "3.14" + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + python-version: "3.14" + prune-cache: true + + - name: Build the wheel + run: uv build --wheel + + - name: Install without extras + run: | + uv venv --python 3.14 .core-venv + uv pip install --python .core-venv/bin/python dist/*.whl + + - name: Test the default installation + run: .core-venv/bin/python tests/core_install_smoke.py + build: runs-on: ubuntu-latest strategy: diff --git a/Dockerfile b/Dockerfile index 1f67460..6a50619 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,8 +59,9 @@ COPY --from=builder /build/dist/*.whl /tmp/ # Install the wheel and remove temporary artifacts. # With the slim runtime image this should resolve binary dependencies from # prebuilt wheels instead of compiling NumPy/SciPy from source. -RUN uv pip install --system --no-cache /tmp/apache_otava-*.whl \ - && rm /tmp/apache_otava-*.whl \ +RUN set -- /tmp/apache_otava-*.whl \ + && uv pip install --system --no-cache "${1}[all]" \ + && rm "${1}" \ && rm /usr/local/bin/uv # Switch to otava user diff --git a/README.md b/README.md index 7f68e94..19c7311 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,26 @@ integrated with a pull request. See the documentation in https://otava.apache.org/docs/overview/. +## Installation + +The default package includes the Otava library and CLI, with support for CSV, +JSON, HistoStat, and Graphite data: + +```bash +pip install apache-otava +``` + +Install extras for additional integrations. Extras can be combined, or install +`all` to enable every integration: + +```bash +pip install 'apache-otava[bigquery]' +pip install 'apache-otava[postgres,slack]' +pip install 'apache-otava[all]' +``` + +See the [installation guide](docs/INSTALL.md) for the complete list of extras. + ## Supported Python Versions Apache Otava is tested against Python 3.10, 3.11, 3.12, 3.13, and 3.14. diff --git a/docs/BIG_QUERY.md b/docs/BIG_QUERY.md index 5c0ade5..4b51ccf 100644 --- a/docs/BIG_QUERY.md +++ b/docs/BIG_QUERY.md @@ -19,6 +19,12 @@ # BigQuery +## Installation + +```bash +pip install 'apache-otava[bigquery]' +``` + ## Schema See [schema.sql](../examples/bigquery/schema.sql) for the example schema. diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 5a5b145..b771c71 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -27,12 +27,24 @@ Otava requires Python 3.10 or later. pip install apache-otava ``` +This installs the Otava library and CLI with CSV, JSON, HistoStat, and Graphite +support. Install the extra for any additional service you use, for example: + +```bash +pip install 'apache-otava[postgres]' +pip install 'apache-otava[bigquery,slack]' +``` + +See [Installation](INSTALL.md) for the complete list of extras. + or ```bash docker pull apache/otava ``` +The Docker image includes all optional integrations. + ## Setup diff --git a/docs/GRAFANA.md b/docs/GRAFANA.md index 4d5fb0d..9626e7c 100644 --- a/docs/GRAFANA.md +++ b/docs/GRAFANA.md @@ -22,6 +22,14 @@ Change points found by `analyze` can be exported as Grafana annotations using the `--update-grafana` flag: +## Installation + +```bash +pip install 'apache-otava[grafana]' +``` + +## Usage + ``` $ otava analyze <test or group> --update-grafana ``` diff --git a/docs/INFLUXDB.md b/docs/INFLUXDB.md index 024131f..acfd7f6 100644 --- a/docs/INFLUXDB.md +++ b/docs/INFLUXDB.md @@ -24,6 +24,12 @@ Otava imports query results from InfluxDB 3 Core or Enterprise through the client. SQL is the default query language; set `query_language: influxql` for InfluxQL queries. +## Installation + +```bash +pip install 'apache-otava[influxdb]' +``` + ## Connection ```yaml diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 1844f89..2a20fe4 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -27,6 +27,26 @@ Otava requires Python 3.10 or later. pip install apache-otava ``` +The default installation includes the Otava library and CLI, with support for CSV, +JSON, HistoStat, and Graphite data. Install an extra when you need another service: + +| Extra | Adds support for | +| --- | --- | +| `bigquery` | Google BigQuery | +| `postgres` | PostgreSQL | +| `influxdb` | InfluxDB 3 | +| `grafana` | Grafana annotations | +| `slack` | Slack notifications | +| `all` | All optional integrations | + +Extras are additive. For example: + +```bash +pip install 'apache-otava[bigquery]' +pip install 'apache-otava[bigquery,postgres]' +pip install 'apache-otava[all]' +``` + ## Install using Docker Pull the official Docker image: @@ -34,3 +54,5 @@ Pull the official Docker image: ```bash docker pull apache/otava ``` + +The Docker image includes all optional integrations. diff --git a/docs/POSTGRESQL.md b/docs/POSTGRESQL.md index 870aa5d..e700bf9 100644 --- a/docs/POSTGRESQL.md +++ b/docs/POSTGRESQL.md @@ -22,6 +22,12 @@ > [!TIP] > See [otava.yaml](../examples/postgresql/config/otava.yaml) for the full > example configuration. +## Installation + +```bash +pip install 'apache-otava[postgres]' +``` + ## PostgreSQL Connection The following block contains PostgreSQL connection details: diff --git a/otava/_optional.py b/otava/_optional.py new file mode 100644 index 0000000..3fdafd9 --- /dev/null +++ b/otava/_optional.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from importlib import import_module +from types import ModuleType + + +class MissingOptionalDependencyError(ModuleNotFoundError): + pass + + +def import_optional_dependency(module_name: str, extra_name: str) -> ModuleType: + try: + return import_module(module_name) + except ModuleNotFoundError as err: + missing_module = err.name + if missing_module and ( + module_name == missing_module or module_name.startswith(f"{missing_module}.") + ): + raise MissingOptionalDependencyError( + f"Optional dependency '{module_name}' is required for this operation. " + f"Install it with: pip install 'apache-otava[{extra_name}]'", + name=module_name, + ) from err + raise diff --git a/otava/bigquery.py b/otava/bigquery.py index 8ceb5a5..4708863 100644 --- a/otava/bigquery.py +++ b/otava/bigquery.py @@ -15,17 +15,31 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from dataclasses import dataclass from datetime import datetime -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from google.cloud import bigquery -from google.oauth2 import service_account +if TYPE_CHECKING: + from google.cloud.bigquery import Client, ScalarQueryParameter +else: + Client = Any + ScalarQueryParameter = Any +from otava._optional import import_optional_dependency from otava.change_point_divisive.base import ChangePointGroup, ChangePointSerializer from otava.test_config import BigQueryTestConfig +def _bigquery_module(): + return import_optional_dependency("google.cloud.bigquery", "bigquery") + + +def _service_account_module(): + return import_optional_dependency("google.oauth2.service_account", "bigquery") + + @dataclass class BigQueryConfig: NAME = "bigquery" @@ -62,8 +76,10 @@ class BigQuery: self.__config = config @property - def client(self) -> bigquery.Client: + def client(self) -> Client: if self.__client is None: + bigquery = _bigquery_module() + service_account = _service_account_module() credentials = service_account.Credentials.from_service_account_file( self.__config.credentials, scopes=["https://www.googleapis.com/auth/cloud-platform";], @@ -72,8 +88,9 @@ class BigQuery: return self.__client def fetch_data( - self, query: str, params: Optional[List[bigquery.ScalarQueryParameter]] = None + self, query: str, params: Optional[List[ScalarQueryParameter]] = None ): + bigquery = _bigquery_module() job_config = None if params: job_config = bigquery.QueryJobConfig(query_parameters=params) diff --git a/otava/grafana.py b/otava/grafana.py index b853f3a..ea0e7fa 100644 --- a/otava/grafana.py +++ b/otava/grafana.py @@ -19,9 +19,13 @@ from dataclasses import asdict, dataclass from datetime import datetime from typing import List, Optional -import requests from pytz import UTC -from requests.exceptions import HTTPError + +from otava._optional import import_optional_dependency + + +def _requests_module(): + return import_optional_dependency("requests", "grafana") @dataclass @@ -77,6 +81,7 @@ class Grafana: Reference: - https://grafana.com/docs/grafana/latest/http_api/annotations/#find-annotations """ + requests = _requests_module() url = f"{self.url}api/annotations" query_parameters = {} if start is not None: @@ -105,7 +110,7 @@ class Grafana: except KeyError as err: raise GrafanaError(f"Missing field {err.args[0]}") - except HTTPError as err: + except requests.exceptions.HTTPError as err: raise GrafanaError(str(err)) def delete_annotations(self, *ids: int): @@ -113,13 +118,14 @@ class Grafana: Reference: - https://grafana.com/docs/grafana/latest/http_api/annotations/#delete-annotation-by-id """ + requests = _requests_module() url = f"{self.url}api/annotations" for annotation_id in ids: annotation_url = f"{url}/{annotation_id}" try: response = requests.delete(url=annotation_url, auth=(self.__user, self.__password)) response.raise_for_status() - except HTTPError as err: + except requests.exceptions.HTTPError as err: raise GrafanaError(str(err)) def create_annotations(self, *annotations: Annotation): @@ -127,6 +133,7 @@ class Grafana: Reference: - https://grafana.com/docs/grafana/latest/http_api/annotations/#create-annotation """ + requests = _requests_module() try: url = f"{self.url}api/annotations" for annotation in annotations: @@ -135,5 +142,5 @@ class Grafana: del data["id"] response = requests.post(url=url, json=data, auth=(self.__user, self.__password)) response.raise_for_status() - except HTTPError as err: + except requests.exceptions.HTTPError as err: raise GrafanaError(str(err)) diff --git a/otava/importer.py b/otava/importer.py index 0552405..a6b3084 100644 --- a/otava/importer.py +++ b/otava/importer.py @@ -24,9 +24,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Dict, List, Optional, Set -from google.cloud import bigquery - -from otava.bigquery import BigQuery +from otava.bigquery import BigQuery, _bigquery_module from otava.config import Config from otava.data_selector import DataSelector from otava.graphite import DataPoint, Graphite, GraphiteError @@ -758,7 +756,9 @@ class BigQueryImporter(Importer): ) # Replace placeholder with @branch for BigQuery parameterized query query = query.replace("%{BRANCH}", "@branch") - params = [bigquery.ScalarQueryParameter("branch", "STRING", selector.branch)] + params = [ + _bigquery_module().ScalarQueryParameter("branch", "STRING", selector.branch) + ] columns, rows = self.__bigquery.fetch_data(query, params) diff --git a/otava/influxdb.py b/otava/influxdb.py index af5a53b..c4aa450 100644 --- a/otava/influxdb.py +++ b/otava/influxdb.py @@ -15,9 +15,22 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from influxdb_client_3 import InfluxDBClient3 +else: + InfluxDBClient3 = Any + +from otava._optional import import_optional_dependency + -from influxdb_client_3 import InfluxDBClient3 +def _influxdb_client_class(): + influxdb = import_optional_dependency("influxdb_client_3", "influxdb") + return influxdb.InfluxDBClient3 @dataclass @@ -51,7 +64,7 @@ class InfluxDB: @property def client(self) -> InfluxDBClient3: if self._client is None: - self._client = InfluxDBClient3( + self._client = _influxdb_client_class()( host=self.config.host, database=self.config.database, token=self.config.token, diff --git a/otava/main.py b/otava/main.py index 3bb164d..395061f 100644 --- a/otava/main.py +++ b/otava/main.py @@ -22,9 +22,9 @@ from typing import Dict, List, Optional import configargparse as argparse import pytz -from slack_sdk import WebClient from otava import config +from otava._optional import MissingOptionalDependencyError from otava.attributes import get_back_links from otava.bigquery import BigQuery, BigQueryError from otava.config import Config @@ -35,7 +35,7 @@ from otava.importer import DataImportError, Importers from otava.postgres import Postgres, PostgresError from otava.report import Report, ReportType from otava.series import AnalysisOptions, AnalyzedSeries -from otava.slack import NotificationError, SlackNotifier +from otava.slack import NotificationError, SlackNotifier, _create_slack_notifier from otava.test_config import ( BigQueryTestConfig, GraphiteTestConfig, @@ -63,7 +63,7 @@ class Otava: self.__conf = conf self.__importers = Importers(conf) self.__grafana = None - self.__slack = self.__maybe_create_slack_notifier() + self.__slack = None self.__postgres = None self.__bigquery = None @@ -251,10 +251,12 @@ class Otava: for cpg in cpg_list: bigquery.insert_change_point(test, metric_name, cpg.attributes, cpg) - def __maybe_create_slack_notifier(self): - if not self.__conf.slack: + def __get_slack_notifier(self): + if not self.__conf.slack or not self.__conf.slack.bot_token: return None - return SlackNotifier(WebClient(token=self.__conf.slack.bot_token)) + if self.__slack is None: + self.__slack = _create_slack_notifier(self.__conf.slack.bot_token) + return self.__slack def notify_slack( self, @@ -263,12 +265,13 @@ class Otava: channels: List[str], since: datetime, ): - if not self.__slack: + slack = self.__get_slack_notifier() + if not slack: logging.error( "Slack definition is missing from the configuration, cannot send notification" ) return - self.__slack.notify(test_change_points, selector=selector, channels=channels, since=since) + slack.notify(test_change_points, selector=selector, channels=channels, since=since) def validate(self): valid = True @@ -644,6 +647,9 @@ def script_main(conf: Config = None, args: List[str] = None): except NotificationError as err: logging.error(err.message) exit(1) + except MissingOptionalDependencyError as err: + logging.error(str(err)) + exit(1) def main(): diff --git a/otava/postgres.py b/otava/postgres.py index cbf22af..d00631d 100644 --- a/otava/postgres.py +++ b/otava/postgres.py @@ -15,16 +15,26 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from dataclasses import dataclass from datetime import datetime -from typing import Dict +from typing import TYPE_CHECKING, Any, Dict -import pg8000 +if TYPE_CHECKING: + from pg8000.dbapi import Connection +else: + Connection = Any +from otava._optional import import_optional_dependency from otava.change_point_divisive.base import ChangePointGroup, ChangePointSerializer from otava.test_config import PostgresTestConfig +def _pg8000_module(): + return import_optional_dependency("pg8000", "postgres") + + @dataclass class PostgresConfig: NAME = "postgres" @@ -66,8 +76,9 @@ class Postgres: def __init__(self, config: PostgresConfig): self.__config = config - def __get_conn(self) -> pg8000.dbapi.Connection: + def __get_conn(self) -> Connection: if self.__conn is None: + pg8000 = _pg8000_module() self.__conn = pg8000.dbapi.Connection( host=self.__config.hostname, port=self.__config.port, diff --git a/otava/slack.py b/otava/slack.py index 441f083..22ee8b5 100644 --- a/otava/slack.py +++ b/otava/slack.py @@ -15,14 +15,21 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from dataclasses import dataclass from datetime import datetime from math import isinf -from typing import Dict, List +from typing import TYPE_CHECKING, Any, Dict, List from pytz import UTC -from slack_sdk import WebClient +if TYPE_CHECKING: + from slack_sdk import WebClient +else: + WebClient = Any + +from otava._optional import import_optional_dependency from otava.change_point_divisive.base import ChangePointGroup, ChangePointSerializer from otava.data_selector import DataSelector from otava.series import AnalyzedSeries @@ -263,3 +270,8 @@ class SlackNotifier: for channel in channels: for blocks in dispatches: self.__client.chat_postMessage(channel=channel, blocks=blocks) + + +def _create_slack_notifier(token: str) -> SlackNotifier: + slack_sdk = import_optional_dependency("slack_sdk", "slack") + return SlackNotifier(slack_sdk.WebClient(token=token)) diff --git a/pyproject.toml b/pyproject.toml index 53e3762..6a1ded2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,14 +46,9 @@ dependencies = [ "python-dateutil>=2.9.0", "ruamel.yaml==0.18.16", - "requests>=2.32.5", "pystache>=0.6.8", "tabulate>=0.9.0", "validators>=0.35.0", - "slack-sdk>=3.39.0", - "google-cloud-bigquery>=3.38.0", - "pg8000>=1.31.5", - "influxdb3-python>=0.20.0", "configargparse>=1.7.1", "pydantic>=2,<3", @@ -65,6 +60,28 @@ dependencies = [ ] [project.optional-dependencies] +bigquery = [ + "google-cloud-bigquery>=3.38.0", +] +postgres = [ + "pg8000>=1.31.5", +] +influxdb = [ + "influxdb3-python>=0.20.0", +] +grafana = [ + "requests>=2.32.5", +] +slack = [ + "slack-sdk>=3.39.0", +] +all = [ + "google-cloud-bigquery>=3.38.0", + "pg8000>=1.31.5", + "influxdb3-python>=0.20.0", + "requests>=2.32.5", + "slack-sdk>=3.39.0", +] dev = [ "hypothesis>=6.0", "pytest>=9.0.1", diff --git a/tests/core_install_smoke.py b/tests/core_install_smoke.py new file mode 100644 index 0000000..72b300d --- /dev/null +++ b/tests/core_install_smoke.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import csv +import os +import subprocess +import sys +import tempfile +from importlib import metadata +from pathlib import Path + +from otava.bigquery import BigQuery, BigQueryConfig +from otava.grafana import Grafana, GrafanaConfig +from otava.influxdb import InfluxDB, InfluxDBConfig +from otava.postgres import Postgres, PostgresConfig +from otava.slack import _create_slack_notifier + +OPTIONAL_DISTRIBUTIONS = { + "google-cloud-bigquery": "bigquery", + "pg8000": "postgres", + "influxdb3-python": "influxdb", + "requests": "grafana", + "slack-sdk": "slack", +} + + +def assert_optional_distributions_are_absent(): + for distribution in OPTIONAL_DISTRIBUTIONS: + try: + metadata.version(distribution) + except metadata.PackageNotFoundError: + continue + raise AssertionError(f"{distribution} was installed by the default package") + + +def assert_cli_help_works(): + result = subprocess.run( + [sys.executable, "-m", "otava.main", "--help"], capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr + assert "usage:" in result.stdout + + +def assert_csv_analysis_works(): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + data_dir = root / "data" + data_dir.mkdir() + with (data_dir / "sample.csv").open("w", newline="") as csv_file: + writer = csv.writer(csv_file) + writer.writerow(["time", "metric"]) + for day, value in enumerate([10, 11, 9, 10, 30, 31, 29, 30], start=1): + writer.writerow([f"2026-01-{day:02d}T00:00:00+00:00", value]) + + config = root / "otava.yaml" + config.write_text( + """tests: + local.sample: + type: csv + file: data/sample.csv + time_column: time + metrics: [metric] +""", + encoding="utf-8", + ) + + result = subprocess.run( + [sys.executable, "-m", "otava.main", "analyze", "local.sample"], + cwd=root, + capture_output=True, + text=True, + env={**os.environ, "OTAVA_CONFIG": str(config)}, + ) + assert result.returncode == 0, result.stderr + assert "metric" in result.stdout + + +def assert_missing_extra(extra_name, operation): + try: + operation() + except ModuleNotFoundError as err: + expected = f"pip install 'apache-otava[{extra_name}]'" + assert expected in str(err), str(err) + return + raise AssertionError(f"{extra_name} operation did not report its missing extra") + + +def assert_optional_operations_name_their_extras(): + operations = { + "bigquery": lambda: BigQuery( + BigQueryConfig("project", "dataset", "credentials.json") + ).client, + "postgres": lambda: Postgres( + PostgresConfig("localhost", 5432, "user", "password", "database") + ).fetch_data("SELECT 1"), + "influxdb": lambda: InfluxDB( + InfluxDBConfig("http://localhost:8181";, "database", "token") + ).client, + "grafana": lambda: Grafana( + GrafanaConfig("https://example.invalid/";, "user", "password") + ).fetch_annotations(None, None), + "slack": lambda: _create_slack_notifier("token"), + } + for extra_name, operation in operations.items(): + assert_missing_extra(extra_name, operation) + + +def main(): + assert_optional_distributions_are_absent() + assert_cli_help_works() + assert_csv_analysis_works() + assert_optional_operations_name_their_extras() + + +if __name__ == "__main__": + main() diff --git a/tests/optional_dependencies_test.py b/tests/optional_dependencies_test.py new file mode 100644 index 0000000..46a270c --- /dev/null +++ b/tests/optional_dependencies_test.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import subprocess +import sys +import textwrap +from importlib import import_module + +import pytest + + +def test_missing_optional_dependency_names_install_extra(monkeypatch): + optional = import_module("otava._optional") + + def missing_google(_): + raise ModuleNotFoundError("No module named 'google'", name="google") + + monkeypatch.setattr(optional, "import_module", missing_google) + + with pytest.raises(ModuleNotFoundError) as exc_info: + optional.import_optional_dependency("google.cloud.bigquery", "bigquery") + + assert str(exc_info.value) == ( + "Optional dependency 'google.cloud.bigquery' is required for this operation. " + "Install it with: pip install 'apache-otava[bigquery]'" + ) + + +def test_optional_dependency_preserves_unrelated_import_failure(monkeypatch): + optional = import_module("otava._optional") + original_error = ModuleNotFoundError("No module named 'transitive_package'", name="transitive_package") + + def missing_transitive(_): + raise original_error + + monkeypatch.setattr(optional, "import_module", missing_transitive) + + with pytest.raises(ModuleNotFoundError) as exc_info: + optional.import_optional_dependency("google.cloud.bigquery", "bigquery") + + assert exc_info.value is original_error + + +def test_cli_import_does_not_load_optional_service_clients(): + script = textwrap.dedent( + """ + import builtins + + blocked = {"google", "influxdb_client_3", "pg8000", "requests", "slack_sdk"} + original_import = builtins.__import__ + + def import_without_optional_clients(name, globals=None, locals=None, fromlist=(), level=0): + top_level = name.partition(".")[0] + if top_level in blocked: + raise ModuleNotFoundError(f"No module named '{top_level}'", name=top_level) + return original_import(name, globals, locals, fromlist, level) + + builtins.__import__ = import_without_optional_clients + + from otava.main import create_otava_cli_parser + + assert "usage:" in create_otava_cli_parser().format_help() + """ + ) + + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + assert result.returncode == 0, result.stderr + + +def test_runtime_type_hints_do_not_require_optional_service_clients(): + script = textwrap.dedent( + """ + import builtins + from typing import get_type_hints + + blocked = {"google", "influxdb_client_3", "pg8000", "slack_sdk"} + original_import = builtins.__import__ + + def import_without_optional_clients(name, globals=None, locals=None, fromlist=(), level=0): + top_level = name.partition(".")[0] + if top_level in blocked: + raise ModuleNotFoundError(f"No module named '{top_level}'", name=top_level) + return original_import(name, globals, locals, fromlist, level) + + builtins.__import__ = import_without_optional_clients + + from otava.bigquery import BigQuery + from otava.influxdb import InfluxDB + from otava.postgres import Postgres + from otava.slack import SlackNotifier + + get_type_hints(BigQuery.client.fget) + get_type_hints(InfluxDB.client.fget) + get_type_hints(Postgres._Postgres__get_conn) + get_type_hints(SlackNotifier.__init__) + """ + ) + + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + assert result.returncode == 0, result.stderr diff --git a/uv.lock b/uv.lock index 9b08a4e..1623753 100644 --- a/uv.lock +++ b/uv.lock @@ -24,24 +24,29 @@ source = { editable = "." } dependencies = [ { name = "configargparse" }, { name = "dateparser" }, - { name = "google-cloud-bigquery" }, - { name = "influxdb3-python" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version < '3.14'" }, { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version >= '3.14'" }, - { name = "pg8000" }, { name = "pydantic" }, { name = "pystache" }, { name = "python-dateutil" }, - { name = "requests" }, { name = "ruamel-yaml" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version >= '3.11'" }, - { name = "slack-sdk" }, { name = "tabulate" }, { name = "validators" }, ] [package.optional-dependencies] +all = [ + { name = "google-cloud-bigquery" }, + { name = "influxdb3-python" }, + { name = "pg8000" }, + { name = "requests" }, + { name = "slack-sdk" }, +] +bigquery = [ + { name = "google-cloud-bigquery" }, +] dev = [ { name = "autoflake" }, { name = "flake8" }, @@ -54,6 +59,18 @@ dev = [ { name = "ruff" }, { name = "tox" }, ] +grafana = [ + { name = "requests" }, +] +influxdb = [ + { name = "influxdb3-python" }, +] +postgres = [ + { name = "pg8000" }, +] +slack = [ + { name = "slack-sdk" }, +] [package.dev-dependencies] dev = [ @@ -66,13 +83,16 @@ requires-dist = [ { name = "configargparse", specifier = ">=1.7.1" }, { name = "dateparser", specifier = ">=1.0.0" }, { name = "flake8", marker = "extra == 'dev'", specifier = ">=7.3.0" }, - { name = "google-cloud-bigquery", specifier = ">=3.38.0" }, + { name = "google-cloud-bigquery", marker = "extra == 'all'", specifier = ">=3.38.0" }, + { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.38.0" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.0" }, - { name = "influxdb3-python", specifier = ">=0.20.0" }, + { name = "influxdb3-python", marker = "extra == 'all'", specifier = ">=0.20.0" }, + { name = "influxdb3-python", marker = "extra == 'influxdb'", specifier = ">=0.20.0" }, { name = "isort", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "numpy", marker = "python_full_version < '3.14'", specifier = "==2.2.*" }, { name = "numpy", marker = "python_full_version >= '3.14'", specifier = ">=2.3.2,<2.4" }, - { name = "pg8000", specifier = ">=1.31.5" }, + { name = "pg8000", marker = "extra == 'all'", specifier = ">=1.31.5" }, + { name = "pg8000", marker = "extra == 'postgres'", specifier = ">=1.31.5" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = "==4.5.0" }, { name = "pydantic", specifier = ">=2,<3" }, { name = "pystache", specifier = ">=0.6.8" }, @@ -80,17 +100,19 @@ requires-dist = [ { name = "pytest-benchmark", marker = "extra == 'dev'", specifier = ">=5.2.3" }, { name = "python-dateutil", specifier = ">=2.9.0" }, { name = "pytz", marker = "extra == 'dev'", specifier = "==2025.2" }, - { name = "requests", specifier = ">=2.32.5" }, + { name = "requests", marker = "extra == 'all'", specifier = ">=2.32.5" }, + { name = "requests", marker = "extra == 'grafana'", specifier = ">=2.32.5" }, { name = "ruamel-yaml", specifier = "==0.18.16" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.14.8" }, { name = "scipy", marker = "python_full_version < '3.11'", specifier = ">=1.15,<1.16" }, { name = "scipy", marker = "python_full_version >= '3.11'", specifier = ">=1.16,<1.17" }, - { name = "slack-sdk", specifier = ">=3.39.0" }, + { name = "slack-sdk", marker = "extra == 'all'", specifier = ">=3.39.0" }, + { name = "slack-sdk", marker = "extra == 'slack'", specifier = ">=3.39.0" }, { name = "tabulate", specifier = ">=0.9.0" }, { name = "tox", marker = "extra == 'dev'", specifier = "==4.32.0" }, { name = "validators", specifier = ">=0.35.0" }, ] -provides-extras = ["dev"] +provides-extras = ["bigquery", "postgres", "influxdb", "grafana", "slack", "all", "dev"] [package.metadata.requires-dev] dev = [{ name = "pytest-cov", specifier = ">=7.1.0" }]
