Hello!
Nalaginrut reported that Guix fails to build with Guile 2.2, which was a
bit of a shame, hence commits e465d9e19087ab150f7e31f21c09e4a147b93b36
and 9d126aa2b504bb9fad536eac186805ff623e96be.
Now, the hack in build-aux/compile-all.scm doesn’t quite work with 2.2,
and it was already quite fragile.
So I think we need a different approach. A more robust approach that
comes to mind would be to look at our module dependency graph (with help
from (guix modules)), run N Guile processes (instead of threads), and
have each of them build a subset of the graph. It would essentially
require us to reimplement a subset of Make (or guix-daemon).
I’ve tried generating makefile rules from module dependencies. That’s
pretty cool, but compilation of the package modules remains damn slow
(2nd patch attached).
What do people think? Anyone has a better idea?
The attached patch allows us to build the modules sequentially but in
topologically-sorted order, which means we can avoid the ugly
load-modules-first workaround. Still, it’s a bit too slow.
Ludo’.
diff --git a/build-aux/compile-all.scm b/build-aux/compile-all.scm
index 7c937a0..3703187 100644
--- a/build-aux/compile-all.scm
+++ b/build-aux/compile-all.scm
@@ -21,6 +21,7 @@
(system base message)
(ice-9 match)
(ice-9 threads)
+ (guix modules)
(guix build utils))
(define warnings
@@ -55,19 +56,15 @@
(map string->symbol
(string-split module-path #\/))))
-;;; To work around <http://bugs.gnu.org/15602> (FIXME), we want to load all
-;;; files to be compiled first. We do this via resolve-interface so that the
-;;; top-level of each file (module) is only executed once.
-(define (load-module-file file)
- (let ((module (file->module file)))
- (format #t " LOAD ~a~%" module)
- (resolve-interface module)))
+(define (module->file module)
+ (string-append srcdir "/"
+ (string-join (map symbol->string module) "/")
+ ".scm"))
-(define (compile-file* file output-mutex)
+(define (compile-file* file)
(let ((go (scm->go file)))
- (with-mutex output-mutex
- (format #t " GUILEC ~a~%" go)
- (force-output))
+ (format #t " GUILEC ~a~%" go)
+ (force-output)
(mkdir-p (dirname go))
(with-fluids ((*current-warning-prefix* ""))
(with-target host
@@ -78,12 +75,12 @@
(match (command-line)
((_ . files)
- (let ((files (filter file-needs-compilation? files)))
- (for-each load-module-file files)
- (let ((mutex (make-mutex)))
- (par-for-each (lambda (file)
- (compile-file* file mutex))
- files)))))
+ (for-each (lambda (module)
+ (let ((file (module->file module)))
+ (when (file-needs-compilation? file)
+ (compile-file* file))))
+ (source-module-closure
+ (map file->module files)))))
;;; Local Variables:
;;; eval: (put 'with-target 'scheme-indent-function 1)
diff --git a/guix/modules.scm b/guix/modules.scm
index 24f613f..837d150 100644
--- a/guix/modules.scm
+++ b/guix/modules.scm
@@ -118,14 +118,19 @@ module names. Only modules that match SELECT? are considered."
(visited (set)))
(match modules
(()
- (reverse result))
+ (values result visited))
((module rest ...)
(cond ((set-contains? visited module)
(loop rest result visited))
((select? module)
- (loop (append (dependencies module) rest)
- (cons module result)
- (set-insert module visited)))
+ (call-with-values
+ (lambda ()
+ (loop rest result
+ (set-insert module visited)))
+ (lambda (result visited)
+ (loop (dependencies module)
+ (cons module result)
+ visited))))
(else
(loop rest result visited)))))))
>From 9621aa00b4378036e906f58dc40e431748ba9790 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= <l...@gnu.org>
Date: Wed, 21 Sep 2016 23:45:07 +0900
Subject: [PATCH] build: Generate makefile rules from .scm files.
* guix/modules.scm (source-module-dependencies): Export.
* build-aux/generate-make-rules.scm: New file.
* Makefile.am (EXTRA_DIST): Add it.
(AM_V_GUILEC, AM_V_GUILEC_, AM_V_GUILEC_0)
(GUILD_COMPILE_FLAGS): New variables.
(.scm.go): New rule.
(%.go, make-go): Remove.
(build-aux/module-rules.mk): New rule. Add "-include" statement.
(CLEANFILES): Add this file.
---
Makefile.am | 45 +++++++++++++++++++------
build-aux/generate-make-rules.scm | 71 +++++++++++++++++++++++++++++++++++++++
guix/modules.scm | 3 +-
3 files changed, 108 insertions(+), 11 deletions(-)
create mode 100644 build-aux/generate-make-rules.scm
diff --git a/Makefile.am b/Makefile.am
index f9fe141..6720773 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -355,6 +355,7 @@ EXTRA_DIST = \
.dir-locals.el \
build-aux/build-self.scm \
build-aux/compile-all.scm \
+ build-aux/generate-make-rules.scm \
build-aux/hydra/evaluate.scm \
build-aux/hydra/gnu-system.scm \
build-aux/hydra/demo-os.scm \
@@ -390,26 +391,50 @@ CLEANFILES = \
$(GOBJECTS) \
$(SCM_TESTS:tests/%.scm=%.log)
+AM_V_GUILEC = $(AM_V_GUILEC_$(V))
+AM_V_GUILEC_ = $(AM_V_GUILEC_$(AM_DEFAULT_VERBOSITY))
+AM_V_GUILEC_0 = @echo " GUILEC" $@;
+
+# Flags passed to 'guild compile'.
+GUILD_COMPILE_FLAGS = \
+ -Wformat -Wunbound-variable -Warity-mismatch
+
# Unset 'GUILE_LOAD_COMPILED_PATH' altogether while compiling. Otherwise, if
# $GUILE_LOAD_COMPILED_PATH contains $(moduledir), we may find .go files in
# there that are newer than the local .scm files (for instance because the
# user ran 'make install' recently). When that happens, we end up loading
# those previously-installed .go files, which may be stale, thereby breaking
-# the whole thing. Likewise, set 'XDG_CACHE_HOME' to avoid loading possibly
-# stale files from ~/.cache/guile/ccache.
-%.go: make-go ; @:
-make-go: $(MODULES) guix/config.scm guix/tests.scm
- $(AM_V_at)echo "Compiling Scheme modules..." ; \
+# the whole thing.
+#
+# XXX: Use the C locale for when Guile lacks
+# <http://git.sv.gnu.org/cgit/guile.git/commit/?h=stable-2.0&id=e2c6bf3866d1186c60bacfbd4fe5037087ee5e3f>.
+.scm.go:
+ $(AM_V_GUILEC)$(MKDIR_P) `dirname "$@"` ; \
+ $(AM_V_P) && out=1 || out=- ; \
unset GUILE_LOAD_COMPILED_PATH ; \
- XDG_CACHE_HOME=/nowhere \
- host=$(host) srcdir="$(top_srcdir)" \
+ LC_ALL=C \
$(top_builddir)/pre-inst-env \
- $(GUILE) -L "$(top_builddir)" -L "$(top_srcdir)" \
- --no-auto-compile \
- -s "$(top_srcdir)"/build-aux/compile-all.scm $^
+ $(GUILD) compile -L "$(top_builddir)" -L "$(top_srcdir)" \
+ $(GUILD_COMPILE_FLAGS) --target="$(host)" \
+ -o "$@" "$<" >&$$out
+
SUFFIXES = .go
+# Dependency among Scheme modules.
+-include build-aux/module-rules.mk
+
+build-aux/module-rules.mk: $(MODULES)
+ $(AM_V_GEN)$(MKDIR_P) `dirname "$@"` ; \
+ unset GUILE_LOAD_COMPILED_PATH ; \
+ LC_ALL=C srcdir="$(srcdir)" \
+ $(top_builddir)/pre-inst-env \
+ $(GUILE) --no-auto-compile \
+ build-aux/generate-make-rules.scm $^ > "$@.tmp"
+ mv "$@.tmp" "$@"
+
+CLEANFILES += build-aux/module-rules.mk
+
# Make sure source files are installed first, so that the mtime of
# installed compiled files is greater than that of installed source
# files. See
diff --git a/build-aux/generate-make-rules.scm b/build-aux/generate-make-rules.scm
new file mode 100644
index 0000000..c38b364
--- /dev/null
+++ b/build-aux/generate-make-rules.scm
@@ -0,0 +1,71 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2016 Ludovic Courtès <l...@gnu.org>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
+
+(use-modules (guix modules)
+ (guix utils)
+ (srfi srfi-26)
+ (ice-9 match))
+
+(define (relevant-module? module)
+ (match module
+ (('guix _ ...) #t)
+ (('gnu 'packages _ ...) #f)
+ (('gnu _ ...) #t)
+ (_ #f)))
+
+(define srcdir (or (getenv "srcdir")"."))
+
+(define (relative-file file)
+ (if (string-prefix? (string-append srcdir "/") file)
+ (string-drop file (+ 1 (string-length srcdir)))
+ file))
+
+(define (file->module file)
+ (let* ((relative (relative-file file))
+ (module-path (string-drop-right relative 4)))
+ (map string->symbol
+ (string-split module-path #\/))))
+
+(define (module->file module)
+ (string-append srcdir "/"
+ (string-join (map symbol->string module) "/")))
+
+(define (display-module-rule module dependencies port)
+ (format port "~a.go: ~a.scm~{ ~a~}~%"
+ (module->file module)
+ (module->file module)
+ (map (compose (cut string-append <> ".go")
+ module->file) dependencies)))
+
+(match (command-line)
+ ((command . files)
+ (format #t "# Automatically generated. -*- read-only -*-\n")
+ (for-each (lambda (file)
+ (match (file->module file)
+ (('gnu 'packages _ ...)
+ ;; We don't want to rebuild package modules too often,
+ ;; especially since it's rarely useful. Thus, generate a
+ ;; simple static rule.
+ (format #t "~a.go: ~a guix/packages.go~%"
+ (file-sans-extension file) file))
+ (module
+ (display-module-rule module
+ (filter relevant-module?
+ (source-module-dependencies module))
+ (current-output-port)))))
+ files)))
diff --git a/guix/modules.scm b/guix/modules.scm
index 24f613f..d78faf2 100644
--- a/guix/modules.scm
+++ b/guix/modules.scm
@@ -21,7 +21,8 @@
#:use-module (guix sets)
#:use-module (srfi srfi-26)
#:use-module (ice-9 match)
- #:export (source-module-closure
+ #:export (source-module-dependencies
+ source-module-closure
live-module-closure
guix-module-name?))
--
2.10.0
