This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a4336cefe Support token limited to single organization (#4822)
a4336cefe is described below
commit a4336cefe665f696393622c512978fabb4566860
Author: Camille Teruel <[email protected]>
AuthorDate: Thu Mar 30 17:57:11 2023 +0200
Support token limited to single organization (#4822)
* feat: Support token limited to one organization
If a token is limited to a single organization, the accounts API is not
accessible.
In that case, it is impossible to get the name of that organization, so it
has to be provided in the connection.
We have those 4 cases:
* single org token & connection.organization unset: unsupported, can't know
the org name
* single org token & connection.organization set: support projects from
given organization
* all orgs token & connection.organization unset: support projects from all
organizations accessible to token owner
* all orgs token & connection.organization set: support only projects from
given organization
* fix: RemoteScope.scope -> RemoteScope.data
* fix: Remove test warning
* fix: Remote scope groups parentId must be null not empty string
---------
Co-authored-by: Camille Teruel <[email protected]>
---
.../python/plugins/azuredevops/azuredevops/main.py | 29 ++++++++++++++++------
.../plugins/azuredevops/azuredevops/models.py | 1 +
backend/python/pydevlake/pydevlake/__init__.py | 2 ++
backend/python/pydevlake/pydevlake/message.py | 2 +-
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
.../python/pydevlake/pydevlake/testing/__init__.py | 3 ---
.../services/remote/plugin/remote_scope_api.go | 2 +-
backend/test/integration/helper/models.go | 2 +-
.../test/integration/remote/python_plugin_test.go | 6 +++--
9 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/backend/python/plugins/azuredevops/azuredevops/main.py
b/backend/python/plugins/azuredevops/azuredevops/main.py
index 23e0b0751..da81a34bf 100644
--- a/backend/python/plugins/azuredevops/azuredevops/main.py
+++ b/backend/python/plugins/azuredevops/azuredevops/main.py
@@ -26,6 +26,7 @@ from pydevlake import Plugin, RemoteScopeGroup, DomainType,
ScopeTxRulePair
from pydevlake.domain_layer.code import Repo
from pydevlake.domain_layer.devops import CicdScope
from pydevlake.pipeline_tasks import gitextractor, refdiff
+from pydevlake.api import APIException
class AzureDevOpsPlugin(Plugin):
@@ -57,10 +58,14 @@ class AzureDevOpsPlugin(Plugin):
def remote_scope_groups(self, connection) -> list[RemoteScopeGroup]:
api = AzureDevOpsAPI(connection)
- member_id = api.my_profile().json['id']
- accounts = api.accounts(member_id).json
- for account in accounts['value']:
- org = account['accountName']
+ if connection.organization:
+ orgs = [connection.organization]
+ else:
+ member_id = api.my_profile().json['id']
+ accounts = api.accounts(member_id).json
+ orgs = [account['accountName'] for account in accounts['value']]
+
+ for org in orgs:
for proj in api.projects(org):
proj_name = proj['name']
@@ -86,9 +91,19 @@ class AzureDevOpsPlugin(Plugin):
yield repo
def test_connection(self, connection: AzureDevOpsConnection):
- resp = AzureDevOpsAPI(connection).my_profile()
- if resp.status != 200:
- raise Exception(f"Invalid token: {connection.token}")
+ api = AzureDevOpsAPI(connection)
+ if connection.organization is None:
+ try:
+ api.my_profile()
+ except APIException as e:
+ if e.response.status == 401:
+ raise Exception(f"Invalid token {e}. You may need to set
organization name in connection or edit your token to set organization to 'All
accessible organizations'")
+ raise
+ else:
+ try:
+ api.projects(connection.organization)
+ except APIException as e:
+ raise Exception(f"Invalid token: {e}")
def extra_tasks(self, scope: GitRepository, tx_rule:
AzureDevOpsTransformationRule, entity_types: list[str], connection:
AzureDevOpsConnection):
if DomainType.CODE in entity_types:
diff --git a/backend/python/plugins/azuredevops/azuredevops/models.py
b/backend/python/plugins/azuredevops/azuredevops/models.py
index 96ec88472..b19782a86 100644
--- a/backend/python/plugins/azuredevops/azuredevops/models.py
+++ b/backend/python/plugins/azuredevops/azuredevops/models.py
@@ -27,6 +27,7 @@ from pydevlake.pipeline_tasks import RefDiffOptions
class AzureDevOpsConnection(Connection):
token: str
+ organization: Optional[str]
class AzureDevOpsTransformationRule(TransformationRule):
diff --git a/backend/python/pydevlake/pydevlake/__init__.py
b/backend/python/pydevlake/pydevlake/__init__.py
index bf7622c6a..5374e7bbe 100644
--- a/backend/python/pydevlake/pydevlake/__init__.py
+++ b/backend/python/pydevlake/pydevlake/__init__.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import pytest
+pytest.register_assert_rewrite('pydevlake.testing')
from .model import ToolModel, ToolScope, DomainScope, Connection,
TransformationRule
from .logger import logger
diff --git a/backend/python/pydevlake/pydevlake/message.py
b/backend/python/pydevlake/pydevlake/message.py
index 9d2021087..e3e72b912 100644
--- a/backend/python/pydevlake/pydevlake/message.py
+++ b/backend/python/pydevlake/pydevlake/message.py
@@ -112,7 +112,7 @@ class RemoteScopeGroup(RemoteScopeTreeNode):
class RemoteScope(RemoteScopeTreeNode):
type: str = Field("scope", const=True)
parent_id: str = Field(..., alias="parentId")
- scope: ToolScope
+ data: ToolScope
class RemoteScopes(Message):
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index c6395fc13..c227ad50b 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -116,7 +116,7 @@ class Plugin(ABC):
id=tool_scope.id,
parent_id=group_id,
name=tool_scope.name,
- scope=tool_scope
+ data=tool_scope
)
)
else:
diff --git a/backend/python/pydevlake/pydevlake/testing/__init__.py
b/backend/python/pydevlake/pydevlake/testing/__init__.py
index 01d7fa9e8..ca237aa49 100644
--- a/backend/python/pydevlake/pydevlake/testing/__init__.py
+++ b/backend/python/pydevlake/pydevlake/testing/__init__.py
@@ -13,9 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import pytest
-pytest.register_assert_rewrite('pydevlake.testing')
-
from .testing import (
assert_stream_convert,
assert_stream_run,
diff --git a/backend/server/services/remote/plugin/remote_scope_api.go
b/backend/server/services/remote/plugin/remote_scope_api.go
index 3ae77a907..f40458739 100644
--- a/backend/server/services/remote/plugin/remote_scope_api.go
+++ b/backend/server/services/remote/plugin/remote_scope_api.go
@@ -32,7 +32,7 @@ type RemoteScopesOutput struct {
type RemoteScopesTreeNode struct {
Type string `json:"type"`
- ParentId string `json:"parentId"`
+ ParentId *string `json:"parentId"`
Id string `json:"id"`
Name string `json:"name"`
Data interface{} `json:"data"`
diff --git a/backend/test/integration/helper/models.go
b/backend/test/integration/helper/models.go
index 37da37282..5c3541fa8 100644
--- a/backend/test/integration/helper/models.go
+++ b/backend/test/integration/helper/models.go
@@ -51,7 +51,7 @@ type BlueprintV2Config struct {
}
type RemoteScopesChild struct {
Type string `json:"type"`
- ParentId string `json:"parentId"`
+ ParentId *string `json:"parentId"`
Id string `json:"id"`
Name string `json:"name"`
Data interface{} `json:"data"`
diff --git a/backend/test/integration/remote/python_plugin_test.go
b/backend/test/integration/remote/python_plugin_test.go
index 077005c32..ae9de7a81 100644
--- a/backend/test/integration/remote/python_plugin_test.go
+++ b/backend/test/integration/remote/python_plugin_test.go
@@ -50,8 +50,9 @@ func TestRemoteScopeGroups(t *testing.T) {
scope := scopeGroups[0]
require.Equal(t, "Group 1", scope.Name)
require.Equal(t, "group1", scope.Id)
- require.Equal(t, "", scope.ParentId)
require.Equal(t, "group", scope.Type)
+ require.Nil(t, scope.ParentId)
+ require.Nil(t, scope.Data)
}
func TestRemoteScopes(t *testing.T) {
@@ -69,8 +70,9 @@ func TestRemoteScopes(t *testing.T) {
scope := scopes[0]
require.Equal(t, "Project 1", scope.Name)
require.Equal(t, "p1", scope.Id)
- require.Equal(t, "group1", scope.ParentId)
+ require.Equal(t, "group1", *scope.ParentId)
require.Equal(t, "scope", scope.Type)
+ require.NotNil(t, scope.Data)
}
func TestCreateScope(t *testing.T) {
