This is an automated email from the ASF dual-hosted git repository. bossenti pushed a commit to branch feature/retrieve-crendetials-from-envs in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 66c088c0928c32490e2834341d0c31027b434457 Author: bossenti <[email protected]> AuthorDate: Tue May 30 20:38:11 2023 +0200 feature: retrieve credentials from SP environment variables Signed-off-by: bossenti <[email protected]> --- .../docs/getting-started/quickstart.md | 5 +- streampipes-client-python/docs/index.md | 3 +- ...introduction-to-streampipes-python-client.ipynb | 18 ++--- ...cting-data-from-the-streampipes-data-lake.ipynb | 6 +- ...ive-data-from-the-streampipes-data-stream.ipynb | 6 +- ...ine-learning-on-a-streampipes-data-stream.ipynb | 6 +- streampipes-client-python/setup.py | 3 +- .../streampipes/client/credential_provider.py | 76 +++++++++++++++++--- .../tests/client/test_credential_provider.py | 81 +++++++++++++++++++--- 9 files changed, 159 insertions(+), 45 deletions(-) diff --git a/streampipes-client-python/docs/getting-started/quickstart.md b/streampipes-client-python/docs/getting-started/quickstart.md index cef53c1f9..84b9e5a0c 100644 --- a/streampipes-client-python/docs/getting-started/quickstart.md +++ b/streampipes-client-python/docs/getting-started/quickstart.md @@ -66,10 +66,11 @@ Alternatively, you can provide your credentials via environment variables. Simply define your credential provider as follows: ```python ->>> from streampipes.client.credential_provider import StreamPipesApiKeyCredentials +from streampipes.client.credential_provider import StreamPipesApiKeyCredentials -StreamPipesApiKeyCredentials.from_env(username_env="USER", api_key_env="API-KEY") +StreamPipesApiKeyCredentials() ``` +This requires to set the following environment variables: `SP_API_KEY` and `SP_USERNAME` <br> `username` is always the username that is used to log in into StreamPipes. <br> diff --git a/streampipes-client-python/docs/index.md b/streampipes-client-python/docs/index.md index 4ccf4a0c0..8caeb6759 100644 --- a/streampipes-client-python/docs/index.md +++ b/streampipes-client-python/docs/index.md @@ -89,8 +89,9 @@ Simply define your credential provider as follows: ```python from streampipes.client.credential_provider import StreamPipesApiKeyCredentials -StreamPipesApiKeyCredentials.from_env(username_env="USER", api_key_env="API-KEY") +StreamPipesApiKeyCredentials() ``` +This requires to set the following environment variables: `SP_API_KEY` and `SP_USERNAME` <br> `username` is always the username that is used to log in into StreamPipes. <br> diff --git a/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb b/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb index cf5ff5cff..9ec18392c 100644 --- a/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb +++ b/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb @@ -149,7 +149,7 @@ "execution_count": null, "outputs": [], "source": [ - "StreamPipesApiKeyCredentials.from_env(username_env=\"USER\", api_key_env=\"API-KEY\")" + "StreamPipesApiKeyCredentials()" ], "metadata": { "collapsed": false @@ -158,9 +158,8 @@ { "cell_type": "markdown", "source": [ - "Please note that you pass the names of the environment variables.\n", - "To ensure that the above code works, you must set the environment variables with the same name you specified in `from_env`.\n", - "In this scenario this would look like the following:" + "To ensure that the above code works, you must set the environment variables as expected.\n", + "This can be done like following:" ], "metadata": { "collapsed": false @@ -171,8 +170,8 @@ "execution_count": null, "outputs": [], "source": [ - "%export USER=\"<USERNAME>\"\n", - "%export API_KEY=\"<API-KEY>\"" + "%export SP_USERNAME=\"<USERNAME>\"\n", + "%export SP_API_KEY=\"<API-KEY>\"" ], "metadata": { "collapsed": false @@ -256,13 +255,6 @@ "metadata": { "collapsed": false } - }, - { - "cell_type": "markdown", - "source": [], - "metadata": { - "collapsed": false - } } ], "metadata": { diff --git a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb index 66a71f08b..000f6fa96 100644 --- a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb +++ b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb @@ -46,8 +46,8 @@ "outputs": [], "source": [ "import os\n", - "os.environ[\"USER\"] = \"[email protected]\"\n", - "os.environ[\"API-KEY\"] = \"XXX\"\n" + "os.environ[\"SP_USERNAME\"] = \"[email protected]\"\n", + "os.environ[\"SP_API_KEY\"] = \"XXX\"\n" ], "metadata": { "collapsed": false @@ -59,7 +59,7 @@ "outputs": [], "source": [ "config = StreamPipesClientConfig(\n", - " credential_provider=StreamPipesApiKeyCredentials.from_env(username_env=\"USER\", api_key_env=\"API-KEY\"),\n", + " credential_provider=StreamPipesApiKeyCredentials(),\n", " host_address=\"localhost\",\n", " https_disabled=True,\n", " port=80\n", diff --git a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb index 5d2dcef89..5b206df5f 100644 --- a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb +++ b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb @@ -51,8 +51,8 @@ "source": [ "import os\n", "\n", - "os.environ[\"USER\"] = \"[email protected]\"\n", - "os.environ[\"API-KEY\"] = \"XXX\"\n", + "os.environ[\"SP_USERNAME\"] = \"[email protected]\"\n", + "os.environ[\"SP_API_KEY\"] = \"XXX\"\n", "\n", "# Use this if you work locally:\n", "os.environ[\"BROKER-HOST\"] = \"localhost\" \n", @@ -74,7 +74,7 @@ ], "source": [ "client_config = StreamPipesClientConfig(\n", - " credential_provider=StreamPipesApiKeyCredentials.from_env(username_env=\"USER\", api_key_env=\"API-KEY\"),\n", + " credential_provider=StreamPipesApiKeyCredentials(),\n", " host_address=\"localhost\",\n", " port=80,\n", " https_disabled=True,\n", diff --git a/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb b/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb index 780464d3d..a797f041b 100644 --- a/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb +++ b/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb @@ -40,8 +40,8 @@ "source": [ "import os\n", "\n", - "os.environ[\"USER\"] = \"[email protected]\"\n", - "os.environ[\"API-KEY\"] = \"XXX\"\n", + "os.environ[\"SP_USERNAME\"] = \"[email protected]\"\n", + "os.environ[\"SP_API_KEY\"] = \"XXX\"\n", "\n", "# Use this if you work locally:\n", "os.environ[\"BROKER-HOST\"] = \"localhost\" \n", @@ -63,7 +63,7 @@ ], "source": [ "client_config = StreamPipesClientConfig(\n", - " credential_provider=StreamPipesApiKeyCredentials.from_env(username_env=\"USER\", api_key_env=\"API-KEY\"),\n", + " credential_provider=StreamPipesApiKeyCredentials(),\n", " host_address=\"localhost\",\n", " port=80,\n", " https_disabled=True,\n", diff --git a/streampipes-client-python/setup.py b/streampipes-client-python/setup.py index ac26f49e0..7825ca403 100644 --- a/streampipes-client-python/setup.py +++ b/streampipes-client-python/setup.py @@ -41,7 +41,8 @@ base_packages = [ "pydantic>=1.10.2", "requests>=2.28.1", "nats-py>=2.2.0", - "confluent-kafka>=2.0.2" + "confluent-kafka>=2.0.2", + "typing-extensions>=4.5.0" ] dev_packages = base_packages + [ diff --git a/streampipes-client-python/streampipes/client/credential_provider.py b/streampipes-client-python/streampipes/client/credential_provider.py index a29598c9f..2cafabf30 100644 --- a/streampipes-client-python/streampipes/client/credential_provider.py +++ b/streampipes-client-python/streampipes/client/credential_provider.py @@ -32,6 +32,8 @@ __all__ = [ "StreamPipesApiKeyCredentials", ] +from typing_extensions import deprecated + class CredentialProvider(ABC): """Abstract implementation of a credential provider. @@ -81,12 +83,20 @@ class StreamPipesApiKeyCredentials(CredentialProvider): The required token can be generated via the StreamPipes UI (see the description on our [start-page](../../../). + Both parameters can either be passed as arguments or remain unset. + If they are not passed, they are retrieved from environment variables: + + * `SP_USERNAME` is expected to contain the username + * `SP_API_KEY` is expected to contain the API key + Parameters ---------- - username: str - The username to which the API token is granted, e.g., `[email protected]`. - api_key: str - The StreamPipes API key as it is displayed in the UI. + username: Optional[str] + The username to which the API token is granted, e.g., `[email protected]`.<br> + If not passed, the username is retrieved from environment variable `SP_USERNAME`. + api_key: Optional[str] + The StreamPipes API key as it is displayed in the UI.<br> + If not passed, the api key is retrieved from environment variable `SP_API_KEY` Examples -------- @@ -94,9 +104,15 @@ class StreamPipesApiKeyCredentials(CredentialProvider): """ + _ENV_KEY_API = "SP_API_KEY" + _ENV_KEY_USERNAME = "SP_USERNAME" + @classmethod + @deprecated("deprecated since 0.93.0; please use the class constructor instead.") def from_env(cls, username_env: str, api_key_env: str) -> StreamPipesApiKeyCredentials: - """Returns an api key provider parameterized via environment variables. + """DEPRECATED - use the class constructor instead + + Returns an api key provider parameterized via environment variables. Parameters ---------- @@ -123,19 +139,57 @@ class StreamPipesApiKeyCredentials(CredentialProvider): raise KeyError( f"Ups, the following environment variables have not been found: " f"{'`' + username_env + '`,' if username is None else ''}" - f"{'`' + api_key_env +'`' if api_key is None else ''}. " # noqa: W291 + f"{'`' + api_key_env + '`' if api_key is None else ''}. " # noqa: W291 "Please check them to be properly set." ) return cls(username=username, api_key=api_key) def __init__( - self, - username: str, - api_key: str, + self, + username: Optional[str] = None, + api_key: Optional[str] = None, ): - self.username = username - self.api_key = api_key + + # if both parameters are passed we can add them directly to the instance + if all({username, api_key}): + self.username = username + self.api_key = api_key + + # otherwise we need to check if environment variables are properly set + else: + + retrieved_api_key = os.environ.get(self._ENV_KEY_API, None) + retrieved_user_name = os.environ.get(self._ENV_KEY_USERNAME, None) + + self.username = retrieved_user_name + self.api_key = retrieved_api_key + + if username: + self.username = username + + if retrieved_api_key is None: + raise AttributeError("API key not found in the environment variables - please provide the API key " + "via the parameter `api_key` or ensure that it is passed to " + f"the environment variable `{self._ENV_KEY_API}`.") + self.api_key = retrieved_api_key + + if api_key: + self.api_key = api_key + + if retrieved_user_name is None: + raise AttributeError("Username not found - please provide the username " + "via the parameter `username` or ensure that it is passed to " + f"the environment variable `{self._ENV_KEY_USERNAME}`.") + self.username = retrieved_user_name + + if not all({self.username, self.api_key}): + + if None in {retrieved_user_name, retrieved_api_key}: + raise AttributeError("Both parameters not found - please provide both api key and username " + "either via the parameters (`api_key` and `username`) or via " + f"the corresponding environment variables (`{self._ENV_KEY_API}` and " + f"`{self._ENV_KEY_USERNAME}`).") @property def _authentication_headers(self) -> Dict[str, str]: diff --git a/streampipes-client-python/tests/client/test_credential_provider.py b/streampipes-client-python/tests/client/test_credential_provider.py index 23ed251e6..57a7b975c 100644 --- a/streampipes-client-python/tests/client/test_credential_provider.py +++ b/streampipes-client-python/tests/client/test_credential_provider.py @@ -21,6 +21,16 @@ from streampipes.client.credential_provider import StreamPipesApiKeyCredentials class TestStreamPipesApiKeyCredentials(TestCase): + + @staticmethod + def _clear_envs(): + + if StreamPipesApiKeyCredentials._ENV_KEY_API in os.environ.keys(): + del os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] + + if StreamPipesApiKeyCredentials._ENV_KEY_USERNAME in os.environ.keys(): + del os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] + def test_api_key_credentials(self): credentials = StreamPipesApiKeyCredentials(username="test-user", api_key="test-key") result = credentials.make_headers() @@ -34,14 +44,69 @@ class TestStreamPipesApiKeyCredentials(TestCase): self.assertDictEqual(expected_extended, result_extended) - def test_api_key_from_env(self): - os.environ["USER"] = "user" - os.environ["KEY"] = "api-key" - credentials = StreamPipesApiKeyCredentials.from_env(username_env="USER", api_key_env="KEY") + def test_pass_credentials(self): + credentials = StreamPipesApiKeyCredentials(username="username", api_key="api-key") + + self.assertEqual("username", credentials.username) + self.assertEqual("api-key", credentials.api_key) + + def test_pass_credentials_envs_set(self): + + os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] = "another-api-key" + os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] = "another-user-name" + + credentials = StreamPipesApiKeyCredentials(username="username", api_key="api-key") + + self.assertEqual("username", credentials.username) + self.assertEqual("api-key", credentials.api_key) + + def test_pass_username(self): + + self._clear_envs() + os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] = "another-api-key" + + credentials = StreamPipesApiKeyCredentials(username="username") + + self.assertEqual("username", credentials.username) + self.assertEqual("another-api-key", credentials.api_key) - self.assertEqual("user", credentials.username) + def test_pass_username_api_key_not_set(self): + + self._clear_envs() + + with self.assertRaises(AttributeError): + StreamPipesApiKeyCredentials(username="username") + + def test_pass_api_key(self): + + self._clear_envs() + os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] = "another-username" + + credentials = StreamPipesApiKeyCredentials(api_key="api-key") + + self.assertEqual("another-username", credentials.username) self.assertEqual("api-key", credentials.api_key) - def test_api_key_from_env_not_set(self): - with self.assertRaises(KeyError): - StreamPipesApiKeyCredentials.from_env(username_env="test", api_key_env="key") + def test_pass_api_key_username_not_set(self): + + self._clear_envs() + + with self.assertRaises(AttributeError): + StreamPipesApiKeyCredentials(api_key="api-key") + + def test_nothing_set(self): + + self._clear_envs() + + with self.assertRaises(AttributeError): + StreamPipesApiKeyCredentials() + + def test_all_from_envs(self): + + os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] = "another-api-key" + os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] = "another-username" + + credentials = StreamPipesApiKeyCredentials() + + self.assertEqual("another-username", credentials.username) + self.assertEqual("another-api-key", credentials.api_key)
