Source: grass
Version: 7.0.4-1
Severity: wishlist
Tags: patch upstream
User: reproducible-builds@lists.alioth.debian.org
Usertags: fileordering randomness
X-Debbugs-Cc: reproducible-builds@lists.alioth.debian.org
Control: block -1 by 825088
Dear Maintainer,
While working on the “reproducible builds” effort [1], we have noticed
that 'grass' could not be built reproducibly.
There are several reproducibility issues:
1) File ordering issues - the build result depends on the order of the
files listed with readdir or equivalent.
* in tools/build_modules_xml.py - see patch sort-build-modules-list
* in lib/db/dbmi_base/dbmscap.c (this affects options order in the
html/db.*.html files) - see patch sort-dbmscap that builds an ordered list.
* in include/Make/Vars.make (this affects the order in which object
files are merged) - see patch sort-obj-files
2) Randomness issue: html/colortables/random.png is built using a
pseudo-random generator seeded with build-time value. See patch
srand48_auto-from-SOURCE_DATE_EPOCH that uses the SOURCE_DATE_EPOCH [2]
environment variable (when set) to set a seed from last debian/changelog
entry date.
3) Makefile mistake: from
https://buildd.debian.org/status/fetch.php?pkg=grass&arch=i386&ver=7.0.4-1&stamp=1462121195,
it seems to me that the binary NAD files are not installed properly:
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/prvi
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/hawaii
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/alaska
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/stgeorge
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/FL
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/WO
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/TN
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/stlrnc
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/stpaul
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/conus
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/MD
/usr/bin/install -c -m 644 OBJ.i686-pc-linux-gnu/prvi
/«PKGBUILDDIR»/dist.i686-pc-linux-gnu/etc/proj/nad/WI
The single OBJ.i686-pc-linux-gnu/prvi file is here installed to *all*
/etc/proj/nad files.
See the patch binary-nad-install for a fix.
4) nad2bin issue: nad2bin has unreproducible output (see #825088)
Once these proposed patches are applied (and #825088 fixed), grass can
be built reproducibly in our current experimental framework.
Regards,
Alexis Bienvenüe.
[1] https://wiki.debian.org/ReproducibleBuilds
[2] https://reproducible-builds.org/specs/source-date-epoch/
Description: Sort build modules list
Sort modules in module_items.xml, to make the build reproducible.
Author: Alexis Bienvenüe <p...@passoire.fr>
--- grass-7.0.4.orig/gui/wxpython/tools/build_modules_xml.py
+++ grass-7.0.4/gui/wxpython/tools/build_modules_xml.py
@@ -51,7 +51,7 @@ def parse_modules(fd):
# TODO: what about ms windows? does gtask handle this?
mlist = list(gcore.get_commands()[0])
indent = 4
- for m in mlist:
+ for m in sorted(mlist):
# TODO: get rid of g.mapsets_picker.py
if m == 'g.mapsets_picker.py' or m == 'g.parser':
continue
Description: Sort dbmscap list
To get reproducible results.
Author: Alexis Bienvenüe <p...@passoire.fr>
Index: grass-7.0.4/lib/db/dbmi_base/dbmscap.c
===================================================================
--- grass-7.0.4.orig/lib/db/dbmi_base/dbmscap.c
+++ grass-7.0.4/lib/db/dbmi_base/dbmscap.c
@@ -209,19 +209,22 @@ dbDbmscap *db_read_dbmscap(void)
return list;
}
+static int cmp_entry(dbDbmscap *a, dbDbmscap *b) {
+ return( a->driverName && b->driverName ? strcmp(a->driverName,b->driverName)
: 0 );
+}
+
static void add_entry(dbDbmscap ** list, char *name, char *startup, char
*comment)
{
- dbDbmscap *head, *cur, *tail;
+ /* add an entry to the list, so that the list remains ordered (by
driverName) */
- /* add this entry to the head of a linked list */
- tail = head = *list;
- while (tail && tail->next)
- tail = tail->next;
- *list = NULL;
+ dbDbmscap *head, *cur, *tail;
cur = (dbDbmscap *) db_malloc(sizeof(dbDbmscap));
- if (cur == NULL)
- return; /* out of memory */
+ if (cur == NULL) {
+ *list = NULL;
+ return;
+ /* out of memory */
+ }
cur->next = NULL;
/* copy each item to the dbmscap structure */
@@ -229,11 +232,21 @@ static void add_entry(dbDbmscap ** list,
strcpy(cur->startup, startup);
strcpy(cur->comment, comment);
+ /* find the last entry that is less than cur */
+ tail = head = *list;
+ while (tail && tail->next && cmp_entry(tail->next,cur)<0)
+ tail = tail->next;
+
/* handle the first call (head == NULL) */
- if (tail)
- tail->next = cur;
- else
- head = cur;
+ if (tail && cmp_entry(tail,cur)<0) {
+ /* insert right after tail */
+ cur->next = tail->next;
+ tail->next = cur;
+ } else {
+ /* insert at first position */
+ cur->next = head;
+ head = cur;
+ }
*list = head;
}
Description: Sort object files
to make the build reproducible.
Author: Alexis Bienvenüe <p...@passoire.fr>
--- grass-7.0.4.orig/include/Make/Vars.make
+++ grass-7.0.4/include/Make/Vars.make
@@ -13,10 +13,10 @@ LEX_SOURCES := $(wildcard *.l)
YACC_SOURCES := $(wildcard *.y)
AUTO_OBJS := \
- $(subst .c,.o,$(C_SOURCES)) \
- $(subst .cpp,.o,$(CPP_SOURCES)) \
- $(subst .l,.yy.o,$(LEX_SOURCES)) \
- $(subst .y,.tab.o,$(YACC_SOURCES))
+ $(sort $(subst .c,.o,$(C_SOURCES))) \
+ $(sort $(subst .cpp,.o,$(CPP_SOURCES))) \
+ $(sort $(subst .l,.yy.o,$(LEX_SOURCES))) \
+ $(sort $(subst .y,.tab.o,$(YACC_SOURCES)))
ifndef MOD_OBJS
MOD_OBJS = $(AUTO_OBJS)
Description: Make srand48_auto use SOURCE_DATE_EPOCH if set
If SOURCE_DATE_EPOCH is set, use it to seed the random generator when
G_srand48_auto is called. This helps makeing the build reproducible
(html/random.png)
See https://reproducible-builds.org/specs/source-date-epoch/
Author: Alexis Bienvenüe <p...@passoire.fr>
--- grass-7.0.4.orig/lib/gis/lrand48.c
+++ grass-7.0.4/lib/gis/lrand48.c
@@ -70,7 +70,12 @@ void G_srand48(long seedval)
long G_srand48_auto(void)
{
- unsigned long seed = (unsigned long) getpid();
+ unsigned long seed;
+ char *source_date_epoch = getenv("SOURCE_DATE_EPOCH");
+ if(source_date_epoch) {
+ seed = strtoull(source_date_epoch, NULL, 10);
+ } else {
+ seed = (unsigned long) getpid();
#ifdef HAVE_GETTIMEOFDAY
{
@@ -86,6 +91,7 @@ long G_srand48_auto(void)
seed += (unsigned long) t;
}
#endif
+ }
G_srand48((long) seed);
return (long) seed;
Description: Binary NAD files install
Corrects binary NAD files install rule.
Author: Alexis Bienvenüe <p...@passoire.fr>
--- grass-7.0.4.orig/lib/proj/Makefile
+++ grass-7.0.4/lib/proj/Makefile
@@ -43,7 +43,7 @@ $(FTOL_OBJ): $(OBJDIR)/ftol.o
$(INSTALL) $< $@
endif
-$(NAD_DSTFILES): $(NAD_DIR)/%: $(NAD_BINFILES) | $(NAD_DIR)
+$(NAD_DSTFILES): $(NAD_DIR)/%: $(OBJDIR)/% | $(NAD_DIR)
$(INSTALL_DATA) $< $@
$(NAD_BINFILES): $(OBJDIR)/%: %.lla
_______________________________________________
Reproducible-builds mailing list
Reproducible-builds@lists.alioth.debian.org
http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/reproducible-builds
