This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 42b5c9e17 feature: retrieve credentials from SP environment variables
(#1640)
42b5c9e17 is described below
commit 42b5c9e17486eaddf2b71e66d3c1e3567d251796
Author: Tim <[email protected]>
AuthorDate: Wed May 31 20:12:58 2023 +0200
feature: retrieve credentials from SP environment variables (#1640)
---
.../docs/getting-started/quickstart.md | 5 +-
streampipes-client-python/docs/index.md | 3 +-
...introduction-to-streampipes-python-client.ipynb | 18 ++---
...cting-data-from-the-streampipes-data-lake.ipynb | 6 +-
...ive-data-from-the-streampipes-data-stream.ipynb | 6 +-
...ine-learning-on-a-streampipes-data-stream.ipynb | 6 +-
streampipes-client-python/setup.py | 3 +-
.../streampipes/client/credential_provider.py | 85 +++++++++++++++++++---
.../tests/client/test_credential_provider.py | 81 +++++++++++++++++++--
9 files changed, 167 insertions(+), 46 deletions(-)
diff --git a/streampipes-client-python/docs/getting-started/quickstart.md
b/streampipes-client-python/docs/getting-started/quickstart.md
index cef53c1f9..84b9e5a0c 100644
--- a/streampipes-client-python/docs/getting-started/quickstart.md
+++ b/streampipes-client-python/docs/getting-started/quickstart.md
@@ -66,10 +66,11 @@ Alternatively, you can provide your credentials via
environment variables.
Simply define your credential provider as follows:
```python
->>> from streampipes.client.credential_provider import
StreamPipesApiKeyCredentials
+from streampipes.client.credential_provider import StreamPipesApiKeyCredentials
-StreamPipesApiKeyCredentials.from_env(username_env="USER",
api_key_env="API-KEY")
+StreamPipesApiKeyCredentials()
```
+This requires to set the following environment variables: `SP_API_KEY` and
`SP_USERNAME`
<br>
`username` is always the username that is used to log in into StreamPipes. <br>
diff --git a/streampipes-client-python/docs/index.md
b/streampipes-client-python/docs/index.md
index 4ccf4a0c0..8caeb6759 100644
--- a/streampipes-client-python/docs/index.md
+++ b/streampipes-client-python/docs/index.md
@@ -89,8 +89,9 @@ Simply define your credential provider as follows:
```python
from streampipes.client.credential_provider import StreamPipesApiKeyCredentials
-StreamPipesApiKeyCredentials.from_env(username_env="USER",
api_key_env="API-KEY")
+StreamPipesApiKeyCredentials()
```
+This requires to set the following environment variables: `SP_API_KEY` and
`SP_USERNAME`
<br>
`username` is always the username that is used to log in into StreamPipes. <br>
diff --git
a/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb
b/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb
index cf5ff5cff..9ec18392c 100644
---
a/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb
+++
b/streampipes-client-python/docs/tutorials/1-introduction-to-streampipes-python-client.ipynb
@@ -149,7 +149,7 @@
"execution_count": null,
"outputs": [],
"source": [
- "StreamPipesApiKeyCredentials.from_env(username_env=\"USER\",
api_key_env=\"API-KEY\")"
+ "StreamPipesApiKeyCredentials()"
],
"metadata": {
"collapsed": false
@@ -158,9 +158,8 @@
{
"cell_type": "markdown",
"source": [
- "Please note that you pass the names of the environment variables.\n",
- "To ensure that the above code works, you must set the environment
variables with the same name you specified in `from_env`.\n",
- "In this scenario this would look like the following:"
+ "To ensure that the above code works, you must set the environment
variables as expected.\n",
+ "This can be done like following:"
],
"metadata": {
"collapsed": false
@@ -171,8 +170,8 @@
"execution_count": null,
"outputs": [],
"source": [
- "%export USER=\"<USERNAME>\"\n",
- "%export API_KEY=\"<API-KEY>\""
+ "%export SP_USERNAME=\"<USERNAME>\"\n",
+ "%export SP_API_KEY=\"<API-KEY>\""
],
"metadata": {
"collapsed": false
@@ -256,13 +255,6 @@
"metadata": {
"collapsed": false
}
- },
- {
- "cell_type": "markdown",
- "source": [],
- "metadata": {
- "collapsed": false
- }
}
],
"metadata": {
diff --git
a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
index 66a71f08b..000f6fa96 100644
---
a/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
+++
b/streampipes-client-python/docs/tutorials/2-extracting-data-from-the-streampipes-data-lake.ipynb
@@ -46,8 +46,8 @@
"outputs": [],
"source": [
"import os\n",
- "os.environ[\"USER\"] = \"[email protected]\"\n",
- "os.environ[\"API-KEY\"] = \"XXX\"\n"
+ "os.environ[\"SP_USERNAME\"] = \"[email protected]\"\n",
+ "os.environ[\"SP_API_KEY\"] = \"XXX\"\n"
],
"metadata": {
"collapsed": false
@@ -59,7 +59,7 @@
"outputs": [],
"source": [
"config = StreamPipesClientConfig(\n",
- "
credential_provider=StreamPipesApiKeyCredentials.from_env(username_env=\"USER\",
api_key_env=\"API-KEY\"),\n",
+ " credential_provider=StreamPipesApiKeyCredentials(),\n",
" host_address=\"localhost\",\n",
" https_disabled=True,\n",
" port=80\n",
diff --git
a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
index 5d2dcef89..5b206df5f 100644
---
a/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
+++
b/streampipes-client-python/docs/tutorials/3-getting-live-data-from-the-streampipes-data-stream.ipynb
@@ -51,8 +51,8 @@
"source": [
"import os\n",
"\n",
- "os.environ[\"USER\"] = \"[email protected]\"\n",
- "os.environ[\"API-KEY\"] = \"XXX\"\n",
+ "os.environ[\"SP_USERNAME\"] = \"[email protected]\"\n",
+ "os.environ[\"SP_API_KEY\"] = \"XXX\"\n",
"\n",
"# Use this if you work locally:\n",
"os.environ[\"BROKER-HOST\"] = \"localhost\" \n",
@@ -74,7 +74,7 @@
],
"source": [
"client_config = StreamPipesClientConfig(\n",
- "
credential_provider=StreamPipesApiKeyCredentials.from_env(username_env=\"USER\",
api_key_env=\"API-KEY\"),\n",
+ " credential_provider=StreamPipesApiKeyCredentials(),\n",
" host_address=\"localhost\",\n",
" port=80,\n",
" https_disabled=True,\n",
diff --git
a/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb
b/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb
index 780464d3d..a797f041b 100644
---
a/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb
+++
b/streampipes-client-python/docs/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream.ipynb
@@ -40,8 +40,8 @@
"source": [
"import os\n",
"\n",
- "os.environ[\"USER\"] = \"[email protected]\"\n",
- "os.environ[\"API-KEY\"] = \"XXX\"\n",
+ "os.environ[\"SP_USERNAME\"] = \"[email protected]\"\n",
+ "os.environ[\"SP_API_KEY\"] = \"XXX\"\n",
"\n",
"# Use this if you work locally:\n",
"os.environ[\"BROKER-HOST\"] = \"localhost\" \n",
@@ -63,7 +63,7 @@
],
"source": [
"client_config = StreamPipesClientConfig(\n",
- "
credential_provider=StreamPipesApiKeyCredentials.from_env(username_env=\"USER\",
api_key_env=\"API-KEY\"),\n",
+ " credential_provider=StreamPipesApiKeyCredentials(),\n",
" host_address=\"localhost\",\n",
" port=80,\n",
" https_disabled=True,\n",
diff --git a/streampipes-client-python/setup.py
b/streampipes-client-python/setup.py
index ac26f49e0..7825ca403 100644
--- a/streampipes-client-python/setup.py
+++ b/streampipes-client-python/setup.py
@@ -41,7 +41,8 @@ base_packages = [
"pydantic>=1.10.2",
"requests>=2.28.1",
"nats-py>=2.2.0",
- "confluent-kafka>=2.0.2"
+ "confluent-kafka>=2.0.2",
+ "typing-extensions>=4.5.0"
]
dev_packages = base_packages + [
diff --git
a/streampipes-client-python/streampipes/client/credential_provider.py
b/streampipes-client-python/streampipes/client/credential_provider.py
index a29598c9f..461088e8f 100644
--- a/streampipes-client-python/streampipes/client/credential_provider.py
+++ b/streampipes-client-python/streampipes/client/credential_provider.py
@@ -32,6 +32,8 @@ __all__ = [
"StreamPipesApiKeyCredentials",
]
+from typing_extensions import deprecated
+
class CredentialProvider(ABC):
"""Abstract implementation of a credential provider.
@@ -81,12 +83,20 @@ class StreamPipesApiKeyCredentials(CredentialProvider):
The required token can be generated via the StreamPipes UI (see the
description on our [start-page](../../../).
+ Both parameters can either be passed as arguments or remain unset.
+ If they are not passed, they are retrieved from environment variables:
+
+ * `SP_USERNAME` is expected to contain the username
+ * `SP_API_KEY` is expected to contain the API key
+
Parameters
----------
- username: str
- The username to which the API token is granted, e.g.,
`[email protected]`.
- api_key: str
- The StreamPipes API key as it is displayed in the UI.
+ username: Optional[str]
+ The username to which the API token is granted, e.g.,
`[email protected]`.<br>
+ If not passed, the username is retrieved from environment variable
`SP_USERNAME`.
+ api_key: Optional[str]
+ The StreamPipes API key as it is displayed in the UI.<br>
+ If not passed, the api key is retrieved from environment variable
`SP_API_KEY`
Examples
--------
@@ -94,9 +104,15 @@ class StreamPipesApiKeyCredentials(CredentialProvider):
"""
+ _ENV_KEY_API = "SP_API_KEY"
+ _ENV_KEY_USERNAME = "SP_USERNAME"
+
@classmethod
+ @deprecated("deprecated since 0.93.0; please use the class constructor
instead.")
def from_env(cls, username_env: str, api_key_env: str) ->
StreamPipesApiKeyCredentials:
- """Returns an api key provider parameterized via environment variables.
+ """DEPRECATED - use the class constructor instead
+
+ Returns an api key provider parameterized via environment variables.
Parameters
----------
@@ -123,7 +139,7 @@ class StreamPipesApiKeyCredentials(CredentialProvider):
raise KeyError(
f"Ups, the following environment variables have not been
found: "
f"{'`' + username_env + '`,' if username is None else ''}"
- f"{'`' + api_key_env +'`' if api_key is None else ''}. " #
noqa: W291
+ f"{'`' + api_key_env + '`' if api_key is None else ''}. " #
noqa: W291
"Please check them to be properly set."
)
@@ -131,11 +147,52 @@ class StreamPipesApiKeyCredentials(CredentialProvider):
def __init__(
self,
- username: str,
- api_key: str,
+ username: Optional[str] = None,
+ api_key: Optional[str] = None,
):
- self.username = username
- self.api_key = api_key
+ # if both parameters are passed we can add them directly to the
instance
+ if all({username, api_key}):
+ self.username = username
+ self.api_key = api_key
+
+ # otherwise we need to check if environment variables are properly set
+ else:
+ retrieved_api_key = os.environ.get(self._ENV_KEY_API, None)
+ retrieved_user_name = os.environ.get(self._ENV_KEY_USERNAME, None)
+
+ self.username = retrieved_user_name
+ self.api_key = retrieved_api_key
+
+ if username:
+ self.username = username
+
+ if retrieved_api_key is None:
+ raise AttributeError(
+ "API key not found in the environment variables -
please provide the API key "
+ "via the parameter `api_key` or ensure that it is
passed to "
+ f"the environment variable `{self._ENV_KEY_API}`."
+ )
+ self.api_key = retrieved_api_key
+
+ if api_key:
+ self.api_key = api_key
+
+ if retrieved_user_name is None:
+ raise AttributeError(
+ "Username not found - please provide the username "
+ "via the parameter `username` or ensure that it is
passed to "
+ f"the environment variable `{self._ENV_KEY_USERNAME}`."
+ )
+ self.username = retrieved_user_name
+
+ if not all({self.username, self.api_key}):
+ if None in {retrieved_user_name, retrieved_api_key}:
+ raise AttributeError(
+ "Both parameters not found - please provide both api
key and username "
+ "either via the parameters (`api_key` and `username`)
or via "
+ f"the corresponding environment variables
(`{self._ENV_KEY_API}` and "
+ f"`{self._ENV_KEY_USERNAME}`)."
+ )
@property
def _authentication_headers(self) -> Dict[str, str]:
@@ -146,7 +203,11 @@ class StreamPipesApiKeyCredentials(CredentialProvider):
Dictionary with authentication headers as string key-value pairs.
"""
+
+ user: str = self.username # type: ignore # mypy mistakenly detects
this attribute as optional
+ token: str = self.api_key # type: ignore # mypy mistakenly detects
this attribute as optional
+
return {
- "X-API-User": self.username,
- "X-API-Key": self.api_key,
+ "X-API-User": user,
+ "X-API-Key": token,
}
diff --git a/streampipes-client-python/tests/client/test_credential_provider.py
b/streampipes-client-python/tests/client/test_credential_provider.py
index 23ed251e6..57a7b975c 100644
--- a/streampipes-client-python/tests/client/test_credential_provider.py
+++ b/streampipes-client-python/tests/client/test_credential_provider.py
@@ -21,6 +21,16 @@ from streampipes.client.credential_provider import
StreamPipesApiKeyCredentials
class TestStreamPipesApiKeyCredentials(TestCase):
+
+ @staticmethod
+ def _clear_envs():
+
+ if StreamPipesApiKeyCredentials._ENV_KEY_API in os.environ.keys():
+ del os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API]
+
+ if StreamPipesApiKeyCredentials._ENV_KEY_USERNAME in os.environ.keys():
+ del os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME]
+
def test_api_key_credentials(self):
credentials = StreamPipesApiKeyCredentials(username="test-user",
api_key="test-key")
result = credentials.make_headers()
@@ -34,14 +44,69 @@ class TestStreamPipesApiKeyCredentials(TestCase):
self.assertDictEqual(expected_extended, result_extended)
- def test_api_key_from_env(self):
- os.environ["USER"] = "user"
- os.environ["KEY"] = "api-key"
- credentials =
StreamPipesApiKeyCredentials.from_env(username_env="USER", api_key_env="KEY")
+ def test_pass_credentials(self):
+ credentials = StreamPipesApiKeyCredentials(username="username",
api_key="api-key")
+
+ self.assertEqual("username", credentials.username)
+ self.assertEqual("api-key", credentials.api_key)
+
+ def test_pass_credentials_envs_set(self):
+
+ os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] =
"another-api-key"
+ os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] =
"another-user-name"
+
+ credentials = StreamPipesApiKeyCredentials(username="username",
api_key="api-key")
+
+ self.assertEqual("username", credentials.username)
+ self.assertEqual("api-key", credentials.api_key)
+
+ def test_pass_username(self):
+
+ self._clear_envs()
+ os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] =
"another-api-key"
+
+ credentials = StreamPipesApiKeyCredentials(username="username")
+
+ self.assertEqual("username", credentials.username)
+ self.assertEqual("another-api-key", credentials.api_key)
- self.assertEqual("user", credentials.username)
+ def test_pass_username_api_key_not_set(self):
+
+ self._clear_envs()
+
+ with self.assertRaises(AttributeError):
+ StreamPipesApiKeyCredentials(username="username")
+
+ def test_pass_api_key(self):
+
+ self._clear_envs()
+ os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] =
"another-username"
+
+ credentials = StreamPipesApiKeyCredentials(api_key="api-key")
+
+ self.assertEqual("another-username", credentials.username)
self.assertEqual("api-key", credentials.api_key)
- def test_api_key_from_env_not_set(self):
- with self.assertRaises(KeyError):
- StreamPipesApiKeyCredentials.from_env(username_env="test",
api_key_env="key")
+ def test_pass_api_key_username_not_set(self):
+
+ self._clear_envs()
+
+ with self.assertRaises(AttributeError):
+ StreamPipesApiKeyCredentials(api_key="api-key")
+
+ def test_nothing_set(self):
+
+ self._clear_envs()
+
+ with self.assertRaises(AttributeError):
+ StreamPipesApiKeyCredentials()
+
+ def test_all_from_envs(self):
+
+ os.environ[StreamPipesApiKeyCredentials._ENV_KEY_API] =
"another-api-key"
+ os.environ[StreamPipesApiKeyCredentials._ENV_KEY_USERNAME] =
"another-username"
+
+ credentials = StreamPipesApiKeyCredentials()
+
+ self.assertEqual("another-username", credentials.username)
+ self.assertEqual("another-api-key", credentials.api_key)
