On Sun, Aug 06, 2023 at 07:00:49PM +0200, Marc Espie wrote:
[...]
> > > I'm also wondering if keeping the main GH_* stuff in bsd.port.mk makes a
> > > lot
> > > of sense, instead of grouping everything in github.port.mk
> >
> > I'm for it, maybe as a second step after the module for just the
> > submodule handling is done because there would be a lot of ports churn
> > with moving the main GH_* stuff out of bsd.port.mk.
>
> Probably not. We have a few "big" modules that don't depend on explicitly
> adding to modules, but on some variable triggering it (historic imake stuff
> or configure), having github.port.mk brought in from one of the GH_* variables
> (probably don't need to test them all) would be acceptable.
I'm sharing a new version after some testing, this time as a diff
because it turns out the handling of GH_*-related DISTNAME juggling
needs to happen before setting up the DISTFILES in the module. So this
diff now effectively moves all the GH_* bits from bsd.port.mk into
github.port.mk. The module is hooked up by defining GH_ACCOUNT or
GH_SUBMODULES.
Some rework was needed while testing to handle the different situations
for the target directory of the submodule. This ended up with a test if
the directory exists, and depending on the result either rmdir'ing the
(presumed empty) directory, or mkdir'ing the parent directory.
I tested this with the about 30 ports I could identify that use GitHub
submodules, by adjusting the Makefile to use GH_SUBMODULES. Here a few
points from what I've observed:
1. It's well possible to mix and match with the old way of defining
MASTER_SITES0 etc and DISTFILES=filename.ext:0. An example is
devel/fpm, where I kept this line:
MASTER_SITES0 =
https://github.com/fortran-lang/fpm/releases/download/v${V}/
while replacing MASTER_SITES{1,2} with the new GH_SUBMODULES way.
2. Setting GH_WRKSRC allows keeping the diff from the current Makefile
small, for example in devel/fpm:
+GH_WRKSRC = ${WRKSRC}/vendor
and in editors/neovim:
+GH_WRKSRC = ${STATIC_DEPS_WRKSRC}
3. It's possible to either use the commit hash or a tagname, like here
for lang/jruby:
+GH_SUBMODULES+= jnr jffi refs/tags/jffi-1.3.10 jffi
4. When using EXTRACT_ONLY, the name to use is more complicated now,
but can be identified with `$ make show=CHECKSUMFILES`. Here is what I
had to change it to with print/lilypond:
-EXTRACT_ONLY= ${DISTNAME}.tar.gz urw-base35-fonts-${URW_V}.tar.gz
+EXTRACT_ONLY= ${DISTNAME}.tar.gz \
+
gh-submodules/ArtifexSoftware-urw-base35-fonts-${URW_V}.tar.gz
The full table of what I tested and the result up to if the port still
packages is here: https://thfr.info/tmp/github-submodule-ports.txt
I also tested megaglest as a port that uses GH_ACCOUNT etc without using any
submodules. Tested also with dxx-rebirth as an example of a port not
using any GH_*.
This draft hijacks MASTER_SITES8 for the modules purposes - not
MASTER_SITES9, as I noticed that is used by some modules like cargo and
go. I grep'd through the ports tree and couldn't identify any explicit
use of MASTER_SITES8.
I think this is ready for more general testing:
$ cd /usr/ports/infrastructure/mk; patch < /path/to/github-submodules.diff
Index: bsd.port.mk
===================================================================
RCS file: /cvs/ports/infrastructure/mk/bsd.port.mk,v
retrieving revision 1.1595
diff -u -p -r1.1595 bsd.port.mk
--- bsd.port.mk 8 Jul 2023 10:20:16 -0000 1.1595
+++ bsd.port.mk 7 Aug 2023 16:42:28 -0000
@@ -136,8 +136,8 @@ _ALL_VARIABLES += BROKEN COMES_WITH \
CONFIGURE_STYLE USE_LIBTOOL SEPARATE_BUILD \
SHARED_LIBS TARGETS PSEUDO_FLAVOR \
AUTOCONF_VERSION AUTOMAKE_VERSION CONFIGURE_ARGS \
- GH_ACCOUNT GH_COMMIT GH_PROJECT GH_TAGNAME MAKEFILE_LIST \
- USE_LLD USE_NOEXECONLY USE_WXNEEDED USE_NOBTCFI \
+ GH_ACCOUNT GH_COMMIT GH_PROJECT GH_SUBMODULES GH_TAGNAME \
+ MAKEFILE_LIST USE_LLD USE_NOEXECONLY USE_WXNEEDED USE_NOBTCFI \
COMPILER COMPILER_LANGS COMPILER_LINKS \
SUBST_VARS UPDATE_PLIST_ARGS \
PKGPATHS DEBUG_PACKAGES DEBUG_CONFIGURE_ARGS \
@@ -313,6 +313,10 @@ MODULES += ${_i}
. endif
.endfor
+.if defined(GH_ACCOUNT) || defined(GH_SUBMODULES)
+MODULES += github
+.endif
+
MODULES ?=
.if !empty(MODULES) || !empty(COMPILER)
_MODULES_DONE =
@@ -611,18 +615,6 @@ BUILD_DEPENDS += textproc/groff>=1.21
_PKG_ARGS += -DUSE_GROFF=1
.endif
-# github related variables
-GH_TAGNAME ?=
-GH_COMMIT ?=
-GH_ACCOUNT ?=
-GH_PROJECT ?=
-
-.if !empty(GH_PROJECT) && !empty(GH_TAGNAME)
-_GH_TAG_DIST = ${GH_TAGNAME:C/^(v|V|ver|[Rr]el|[Rr]elease)[-._]?([0-9])/\2/}
-DISTNAME ?= ${GH_PROJECT}-${_GH_TAG_DIST}
-GH_DISTFILE = ${GH_PROJECT}-${_GH_TAG_DIST}${EXTRACT_SUFX}
-.endif
-
PKGNAME ?= ${DISTNAME}
FULLPKGNAME ?= ${PKGNAME}${FLAVOR_EXT}
_MASTER ?=
@@ -871,11 +863,7 @@ _WRKDIRS = ${WRKOBJDIR_${PKGPATH}}/${_WR
_WRKDIRS += ${WRKOBJDIR}/${_WRKDIR_STEM}
_WRKDIRS += ${WRKOBJDIR_MFS}/${_WRKDIR_STEM}
-.if !empty(GH_TAGNAME)
-WRKDIST ?= ${WRKDIR}/${GH_PROJECT}-${GH_TAGNAME:C/^[vV]([0-9])/\1/}
-.elif !empty(GH_COMMIT)
-WRKDIST ?= ${WRKDIR}/${GH_PROJECT}-${GH_COMMIT}
-.elif !defined(DISTNAME)
+.if !defined(DISTNAME)
WRKDIST ?= ${WRKDIR}
.else
WRKDIST ?= ${WRKDIR}/${DISTNAME}
@@ -1259,26 +1247,8 @@ ECHO_MSG ?= echo
.include "${PORTSDIR}/infrastructure/db/network.conf"
-.if !empty(GH_ACCOUNT) && !empty(GH_PROJECT)
-. if !empty(GH_COMMIT) && !empty(GH_TAGNAME)
-ERRORS += "Fatal: specifying both GH_TAGNAME and GH_COMMIT is invalid"
-. endif
-. if !empty(GH_TAGNAME)
-MASTER_SITES_GITHUB += \
-
https://github.com/${GH_ACCOUNT}/${GH_PROJECT}/archive/refs/tags/${GH_TAGNAME:S/$/\//}
-. elif !empty(GH_COMMIT)
-MASTER_SITES_GITHUB += \
- https://github.com/${GH_ACCOUNT}/${GH_PROJECT}/archive/
-. else
-ERRORS += "Fatal: if using GH_*, one of GH_TAGNAME or GH_COMMIT must be set"
-.endif
-
-MASTER_SITES ?= ${MASTER_SITES_GITHUB}
-HOMEPAGE ?= https://github.com/${GH_ACCOUNT}/${GH_PROJECT}
-.else
# Empty declarations to avoid "variable XXX is recursive" errors
MASTER_SITES ?=
-.endif
# I guess we're in the master distribution business! :) As we gain mirror
# sites for distfiles, add them to MASTER_SITE_BACKUP
@@ -1299,10 +1269,7 @@ _warn_checksum += ;echo ">>> MASTER_SITE
EXTRACT_SUFX ?= .tar.gz
-.if !empty(GH_COMMIT)
-GH_DISTFILE =
${DISTNAME}-${GH_COMMIT:C/(........).*/\1/}{${GH_COMMIT}}${EXTRACT_SUFX}
-DISTFILES ?= ${GH_DISTFILE}
-.elif defined(DISTNAME)
+.if defined(DISTNAME)
DISTFILES ?= ${DISTNAME}${EXTRACT_SUFX}
.endif
Index: github.port.mk
===================================================================
RCS file: github.port.mk
diff -N github.port.mk
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ github.port.mk 7 Aug 2023 16:42:28 -0000
@@ -0,0 +1,78 @@
+# GitHub variables, see bsd.port.mk(5)
+
+GH_TAGNAME ?=
+GH_COMMIT ?=
+GH_ACCOUNT ?=
+GH_PROJECT ?=
+
+# List of static dependencies. The format is:
+# account project tag_or_commit target_dir # license
+# Example:
+# GH_SUBMODULES += moonlight-stream moonlight-common-c \
+# c9426a6a71c4162e65dde8c0c71a25f1dbca46ba \
+# third-party/moonlight-common-c # GPL-v3.0+
+GH_SUBMODULES ?=
+
+# Master site for github tarballs
+GH_MASTER_SITES ?= https://github.com/
+
+# where submodule distfiles will be stored
+GH_DIST_SUBDIR ?= gh-submodules
+
+# where submodules will be extracted to
+GH_WRKSRC ?= ${WRKSRC}
+
+# Grab submodules by default with MASTER_SITES8. (Don't use 9 to avoid
collision
+# with language-specific mechanisms, like devel/cargo or lang/go.)
+GH_MASTER_SITESN ?= 8
+MASTER_SITES${GH_MASTER_SITESN} ?= ${GH_MASTER_SITES}
+
+# Default GitHub distfile suffix
+GH_SUFX ?= .tar.gz
+
+.if !empty(GH_PROJECT) && !empty(GH_TAGNAME)
+_GH_TAG_DIST = ${GH_TAGNAME:C/^(v|V|ver|[Rr]el|[Rr]elease)[-._]?([0-9])/\2/}
+DISTNAME ?= ${GH_PROJECT}-${_GH_TAG_DIST}
+GH_DISTFILE = ${GH_PROJECT}-${_GH_TAG_DIST}${EXTRACT_SUFX}
+.endif
+
+.if !empty(GH_TAGNAME)
+WRKDIST ?= ${WRKDIR}/${GH_PROJECT}-${GH_TAGNAME:C/^[vV]([0-9])/\1/}
+.elif !empty(GH_COMMIT)
+WRKDIST ?= ${WRKDIR}/${GH_PROJECT}-${GH_COMMIT}
+.endif
+
+.if !empty(GH_ACCOUNT) && !empty(GH_PROJECT)
+. if !empty(GH_COMMIT) && !empty(GH_TAGNAME)
+ERRORS += "Fatal: specifying both GH_TAGNAME and GH_COMMIT is invalid"
+. endif
+. if !empty(GH_TAGNAME)
+MASTER_SITES_GITHUB += \
+
https://github.com/${GH_ACCOUNT}/${GH_PROJECT}/archive/refs/tags/${GH_TAGNAME:S/$/\//}
+. elif !empty(GH_COMMIT)
+MASTER_SITES_GITHUB += \
+ https://github.com/${GH_ACCOUNT}/${GH_PROJECT}/archive/
+. else
+ERRORS += "Fatal: if using GH_*, one of GH_TAGNAME or GH_COMMIT must be set"
+. endif
+MASTER_SITES ?= ${MASTER_SITES_GITHUB}
+HOMEPAGE ?= https://github.com/${GH_ACCOUNT}/${GH_PROJECT}
+.endif
+
+.if !empty(GH_COMMIT)
+GH_DISTFILE =
${DISTNAME}-${GH_COMMIT:C/(........).*/\1/}{${GH_COMMIT}}${EXTRACT_SUFX}
+DISTFILES ?= ${GH_DISTFILE}
+.endif
+
+# post-extract target for moving the submodules to GH_WRKSRC
+.if !empty(GH_SUBMODULES)
+MODGITHUB_post-extract = \
+ @${ECHO_MSG} "moving GitHub submodules to ${GH_WRKSRC}" ;
+. for _ghaccount _ghproject _ghtagcommit _targetdir in ${GH_SUBMODULES}
+DISTFILES +=
${GH_DIST_SUBDIR}/${_ghaccount}-${_ghproject}-${_ghtagcommit:S/refs\/tags\///}{${_ghaccount}/${_ghproject}/archive/${_ghtagcommit}}${GH_SUFX}:${GH_MASTER_SITESN}
+MODGITHUB_post-extract += \
+ [[ -d ${GH_WRKSRC}/${_targetdir} ]] && rmdir ${GH_WRKSRC}/${_targetdir}
\
+ || mkdir -p `dirname ${GH_WRKSRC}/${_targetdir}` ; \
+ mv ${WRKDIR}/${_ghproject}-${_ghtagcommit:S/refs\/tags\///:S/^v//}
${GH_WRKSRC}/${_targetdir} ;
+. endfor
+.endif
