Re: [OE-core] [PATCH 4/5] recipetool: add go recipe generator

2022-05-11 Thread Mark Asselstine 



On 2022-05-06 03:15, Alexander Kanavin wrote:

This is a lot of code. Can you add some documentation for it, what it
does and how it works? If someone would want to understand it, how
would they go about it?

Alex

On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
 wrote:


From: Lukas Funke 

Signed-off-by: Lukas Funke 
Signed-off-by: Stefan Herbrechtsmeier 
---

  scripts/lib/recipetool/create_go.py | 394 
  1 file changed, 394 insertions(+)
  create mode 100644 scripts/lib/recipetool/create_go.py

diff --git a/scripts/lib/recipetool/create_go.py 
b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 00..4552e9b470
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,394 @@
+# Recipe creation tool - go support plugin
+#
+# Copyright (C) 2022 Weidmueller GmbH & Co KG
+# Author: Lukas Funke 
+#
+# Copyright (c) 2009 The Go Authors. All rights reserved.
+#
+#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
+#
+import bb.utils
+from collections import namedtuple
+from enum import Enum
+from html.parser import HTMLParser
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import shutil
+from urllib.error import URLError, HTTPError
+import urllib.parse
+import urllib.request
+
+from recipetool.create import RecipeHandler, handle_license_vars, 
ensure_native_cmd
+
+GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+re_pseudo_semver = 
re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P[0-9]+)-(?P[0-9Aa-zA-Z]+)")
+re_semver = 
re.compile(r"^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
+
+def tinfoil_init(instance):
+global tinfoil
+tinfoil = instance
+
+class GoRecipeHandler(RecipeHandler):
+
+def _resolve_repository_static(self, modulepath):
+_rootpath = None
+_vcs = None
+_repourl = None
+_suffix = None
+
+host, _, path = modulepath.partition('/')
+
+class vcs(Enum):
+pathprefix = "pathprefix"
+regexp = "regexp"
+vcs = "vcs"
+repo = "repo"
+check = "check"
+schemelessRepo = "schemelessRepo"
+
+# GitHub
+vcsGitHub = {}
+vcsGitHub[vcs.pathprefix] = "github.com"
+vcsGitHub[vcs.regexp] = 
re.compile(r'^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsGitHub[vcs.vcs] = "git"
+vcsGitHub[vcs.repo] = "https://\g"
+
+# Bitbucket
+vcsBitbucket = {}
+vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
+vcsBitbucket[vcs.regexp] = 
re.compile(r'^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsBitbucket[vcs.vcs] = "git"
+vcsBitbucket[vcs.repo] = "https://\g"
+
+# IBM DevOps Services (JazzHub)
+vcsIBMDevOps = {}
+vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
+vcsIBMDevOps[vcs.regexp] = 
re.compile(r'^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsIBMDevOps[vcs.vcs] = "git"
+vcsIBMDevOps[vcs.repo] = "https://\g"
+
+# Git at Apache
+vcsApacheGit = {}
+vcsApacheGit[vcs.pathprefix] = "git.apache.org"
+vcsApacheGit[vcs.regexp] = 
re.compile(r'^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsApacheGit[vcs.vcs] = "git"
+vcsApacheGit[vcs.repo] = "https://\g"
+
+# Git at OpenStack
+vcsOpenStackGit = {}
+vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
+vcsOpenStackGit[vcs.regexp] = 
re.compile(r'^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsOpenStackGit[vcs.vcs] = "git"
+vcsOpenStackGit[vcs.repo] = "https://\g"
+
+# chiselapp.com for fossil
+vcsChiselapp = {}
+vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
+vcsChiselapp[vcs.regexp] = 
re.compile(r'^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
+vcsChiselapp[vcs.vcs] = "fossil"
+vcsChiselapp[vcs.repo] = "https://\g"
+
+# General syntax for any server.
+# Must be last.
+vcsGeneralServer = {}
+vcsGeneralServer[vcs.regexp] = 
re.compile("(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|fossil|git|hg|svn))(/~?(?P[A-Za-z0-9_.\-]+))*$")
+vcsGeneralServer[vcs.schemelessRepo] = True
+
+vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, vcsApacheGit, 
vcsOpenStackGit, vcsChiselapp, vcsGeneralServer]
+
+if modulepath.startswith("example.net") or modulepath == "rsc.io":
+logger.warning("Suspicious module path

Re: [OE-core] [PATCH 4/5] recipetool: add go recipe generator

2022-05-06 Thread Alexander Kanavin 
This is a lot of code. Can you add some documentation for it, what it
does and how it works? If someone would want to understand it, how
would they go about it?

Alex

On Fri, 6 May 2022 at 09:00, Stefan Herbrechtsmeier
 wrote:
>
> From: Lukas Funke 
>
> Signed-off-by: Lukas Funke 
> Signed-off-by: Stefan Herbrechtsmeier 
> ---
>
>  scripts/lib/recipetool/create_go.py | 394 
>  1 file changed, 394 insertions(+)
>  create mode 100644 scripts/lib/recipetool/create_go.py
>
> diff --git a/scripts/lib/recipetool/create_go.py 
> b/scripts/lib/recipetool/create_go.py
> new file mode 100644
> index 00..4552e9b470
> --- /dev/null
> +++ b/scripts/lib/recipetool/create_go.py
> @@ -0,0 +1,394 @@
> +# Recipe creation tool - go support plugin
> +#
> +# Copyright (C) 2022 Weidmueller GmbH & Co KG
> +# Author: Lukas Funke 
> +#
> +# Copyright (c) 2009 The Go Authors. All rights reserved.
> +#
> +#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
> +#
> +import bb.utils
> +from collections import namedtuple
> +from enum import Enum
> +from html.parser import HTMLParser
> +import json
> +import logging
> +import os
> +import re
> +import subprocess
> +import sys
> +import tempfile
> +import shutil
> +from urllib.error import URLError, HTTPError
> +import urllib.parse
> +import urllib.request
> +
> +from recipetool.create import RecipeHandler, handle_license_vars, 
> ensure_native_cmd
> +
> +GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
> +logger = logging.getLogger('recipetool')
> +
> +tinfoil = None
> +
> +re_pseudo_semver = 
> re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P[0-9]+)-(?P[0-9Aa-zA-Z]+)")
> +re_semver = 
> re.compile(r"^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
> +
> +def tinfoil_init(instance):
> +global tinfoil
> +tinfoil = instance
> +
> +class GoRecipeHandler(RecipeHandler):
> +
> +def _resolve_repository_static(self, modulepath):
> +_rootpath = None
> +_vcs = None
> +_repourl = None
> +_suffix = None
> +
> +host, _, path = modulepath.partition('/')
> +
> +class vcs(Enum):
> +pathprefix = "pathprefix"
> +regexp = "regexp"
> +vcs = "vcs"
> +repo = "repo"
> +check = "check"
> +schemelessRepo = "schemelessRepo"
> +
> +# GitHub
> +vcsGitHub = {}
> +vcsGitHub[vcs.pathprefix] = "github.com"
> +vcsGitHub[vcs.regexp] = 
> re.compile(r'^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$')
> +vcsGitHub[vcs.vcs] = "git"
> +vcsGitHub[vcs.repo] = "https://\g"
> +
> +# Bitbucket
> +vcsBitbucket = {}
> +vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
> +vcsBitbucket[vcs.regexp] = 
> re.compile(r'^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P[A-Za-z0-9_.\-]+))*$')
> +vcsBitbucket[vcs.vcs] = "git"
> +vcsBitbucket[vcs.repo] = "https://\g"
> +
> +# IBM DevOps Services (JazzHub)
> +vcsIBMDevOps = {}
> +vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
> +vcsIBMDevOps[vcs.regexp] = 
> re.compile(r'^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$')
> +vcsIBMDevOps[vcs.vcs] = "git"
> +vcsIBMDevOps[vcs.repo] = "https://\g"
> +
> +# Git at Apache
> +vcsApacheGit = {}
> +vcsApacheGit[vcs.pathprefix] = "git.apache.org"
> +vcsApacheGit[vcs.regexp] = 
> re.compile(r'^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P[A-Za-z0-9_.\-]+))*$')
> +vcsApacheGit[vcs.vcs] = "git"
> +vcsApacheGit[vcs.repo] = "https://\g"
> +
> +# Git at OpenStack
> +vcsOpenStackGit = {}
> +vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
> +vcsOpenStackGit[vcs.regexp] = 
> re.compile(r'^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P[A-Za-z0-9_.\-]+))*$')
> +vcsOpenStackGit[vcs.vcs] = "git"
> +vcsOpenStackGit[vcs.repo] = "https://\g"
> +
> +# chiselapp.com for fossil
> +vcsChiselapp = {}
> +vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
> +vcsChiselapp[vcs.regexp] = 
> re.compile(r'^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
> +vcsChiselapp[vcs.vcs] = "fossil"
> +vcsChiselapp[vcs.repo] = "https://\g"
> +
> +# General syntax for any server.
> +# Must be last.
> +vcsGeneralServer = {}
> +vcsGeneralServer[vcs.regexp] = 
> re.compile("(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|fossil|git|hg|svn))(/~?(?P[A-Za-z0-9_.\-]+))*$")
> +vcsGeneralServer[vcs.schemelessRepo] = True
> +
> +vcsPaths = [vcsGitHub, vcsBitbucket,

[OE-core] [PATCH 4/5] recipetool: add go recipe generator

2022-05-06 Thread Stefan Herbrechtsmeier 
From: Lukas Funke 

Signed-off-by: Lukas Funke 
Signed-off-by: Stefan Herbrechtsmeier 
---

 scripts/lib/recipetool/create_go.py | 394 
 1 file changed, 394 insertions(+)
 create mode 100644 scripts/lib/recipetool/create_go.py

diff --git a/scripts/lib/recipetool/create_go.py 
b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 00..4552e9b470
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,394 @@
+# Recipe creation tool - go support plugin
+#
+# Copyright (C) 2022 Weidmueller GmbH & Co KG
+# Author: Lukas Funke 
+#
+# Copyright (c) 2009 The Go Authors. All rights reserved.
+#
+#  SPDX-License-Identifier: BSD-3-Clause AND GPL-2.0-only
+#
+import bb.utils
+from collections import namedtuple
+from enum import Enum
+from html.parser import HTMLParser
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import shutil
+from urllib.error import URLError, HTTPError
+import urllib.parse
+import urllib.request
+
+from recipetool.create import RecipeHandler, handle_license_vars, 
ensure_native_cmd
+
+GoImport = namedtuple('GoImport', 'reporoot vcs repourl suffix')
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+re_pseudo_semver = 
re.compile(r"v([0-9]+)\.([0-9]+).([0-9]+|\([0-9]+\+1\))-(pre\.[0-9]+\.)?([0-9]+\.)?(?P[0-9]+)-(?P[0-9Aa-zA-Z]+)")
+re_semver = 
re.compile(r"^v(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
+
+def tinfoil_init(instance):
+global tinfoil
+tinfoil = instance
+
+class GoRecipeHandler(RecipeHandler):
+
+def _resolve_repository_static(self, modulepath):
+_rootpath = None
+_vcs = None
+_repourl = None
+_suffix = None
+
+host, _, path = modulepath.partition('/')
+
+class vcs(Enum):
+pathprefix = "pathprefix"
+regexp = "regexp"
+vcs = "vcs"
+repo = "repo"
+check = "check"
+schemelessRepo = "schemelessRepo"
+
+# GitHub
+vcsGitHub = {}
+vcsGitHub[vcs.pathprefix] = "github.com"
+vcsGitHub[vcs.regexp] = 
re.compile(r'^(?Pgithub\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsGitHub[vcs.vcs] = "git"
+vcsGitHub[vcs.repo] = "https://\g"
+
+# Bitbucket
+vcsBitbucket = {}
+vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
+vcsBitbucket[vcs.regexp] = 
re.compile(r'^(?Pbitbucket\.org/(?P[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsBitbucket[vcs.vcs] = "git"
+vcsBitbucket[vcs.repo] = "https://\g"
+
+# IBM DevOps Services (JazzHub)
+vcsIBMDevOps = {}
+vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
+vcsIBMDevOps[vcs.regexp] = 
re.compile(r'^(?Phub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsIBMDevOps[vcs.vcs] = "git"
+vcsIBMDevOps[vcs.repo] = "https://\g"
+
+# Git at Apache
+vcsApacheGit = {}
+vcsApacheGit[vcs.pathprefix] = "git.apache.org"
+vcsApacheGit[vcs.regexp] = 
re.compile(r'^(?Pgit\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsApacheGit[vcs.vcs] = "git"
+vcsApacheGit[vcs.repo] = "https://\g"
+
+# Git at OpenStack
+vcsOpenStackGit = {}
+vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
+vcsOpenStackGit[vcs.regexp] = 
re.compile(r'^(?Pgit\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P[A-Za-z0-9_.\-]+))*$')
+vcsOpenStackGit[vcs.vcs] = "git"
+vcsOpenStackGit[vcs.repo] = "https://\g"
+
+# chiselapp.com for fossil
+vcsChiselapp = {}
+vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
+vcsChiselapp[vcs.regexp] = 
re.compile(r'^(?Pchiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
+vcsChiselapp[vcs.vcs] = "fossil"
+vcsChiselapp[vcs.repo] = "https://\g"
+
+# General syntax for any server.
+# Must be last.
+vcsGeneralServer = {}
+vcsGeneralServer[vcs.regexp] = 
re.compile("(?P(?P([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?Pbzr|fossil|git|hg|svn))(/~?(?P[A-Za-z0-9_.\-]+))*$")
+vcsGeneralServer[vcs.schemelessRepo] = True
+
+vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps, vcsApacheGit, 
vcsOpenStackGit, vcsChiselapp, vcsGeneralServer]
+
+if modulepath.startswith("example.net") or modulepath == "rsc.io":
+logger.warning("Suspicious module path %s" % modulepath)
+return None
+if modulepath.startswith("http:") or modulepath.startswith("https:"):
+logger.warning("Import path should not start with %s %s" % 
("http", "https"))
+return None
+
+for srv in vcsPaths:
+