Jérémy Bobbio: > Jérémy Bobbio: > > Here are four patches based on the current master (1e059955) that will > > write files in deterministic order in the control and data archives. > > File names are sorted by forking `sort` before being piped to `tar`. > > Attached are the patch based on current master (36eda4c1bc).
Attached are the patch based on master (1b8c20ad2). -- Lunar .''`. [email protected] : :Ⓐ : # apt-get install anarchism `. `'` `-
From a09849333b2ca211a1fa2ed02674c6af7b49c112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <[email protected]> Date: Tue, 27 Aug 2013 18:10:15 +0200 Subject: [PATCH 1/4] Ensure deterministic file order in data.tar.* files Address: #719845 --- dpkg-deb/build.c | 42 ++++++++++++++++++++++++++++++------------ lib/dpkg/dpkg.h | 1 + 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c index 94d75ff..db9aaf3 100644 --- a/dpkg-deb/build.c +++ b/dpkg-deb/build.c @@ -168,19 +168,36 @@ file_info_list_free(struct file_info *fi) static void file_treewalk_feed(const char *dir, int fd_out) { - int pipefd[2]; - pid_t pid; + int sort_pipefd[2], find_pipefd[2]; + pid_t sort_pid, find_pid; struct file_info *fi; struct file_info *symlist = NULL; struct file_info *symlist_end = NULL; - m_pipe(pipefd); + m_pipe(find_pipefd); + m_pipe(sort_pipefd); + + sort_pid = subproc_fork(); + if (sort_pid == 0) { + m_dup2(find_pipefd[0], 0); + close(find_pipefd[0]); + close(find_pipefd[1]); + m_dup2(sort_pipefd[1], 1); + close(sort_pipefd[0]); + close(sort_pipefd[1]); + if (setenv("LC_ALL", "C", 1 /* overwrite */)) + ohshite(_("unable to setenv")); + execlp(SORT, "sort", "--zero-terminated", NULL); + ohshite(_("unable to execute %s (%s)"), "sort", SORT); + } + close(find_pipefd[0]); + close(sort_pipefd[1]); - pid = subproc_fork(); - if (pid == 0) { - m_dup2(pipefd[1], 1); - close(pipefd[0]); - close(pipefd[1]); + find_pid = subproc_fork(); + if (find_pid == 0) { + m_dup2(find_pipefd[1], 1); + close(find_pipefd[0]); + close(find_pipefd[1]); if (chdir(dir)) ohshite(_("failed to chdir to '%.255s'"), dir); @@ -189,11 +206,11 @@ file_treewalk_feed(const char *dir, int fd_out) "-print0", NULL); ohshite(_("unable to execute %s (%s)"), "find", FIND); } - close(pipefd[1]); + close(find_pipefd[1]); /* We need to reorder the files so we can make sure that symlinks * will not appear before their target. */ - while ((fi = file_info_get(dir, pipefd[0])) != NULL) { + while ((fi = file_info_get(dir, sort_pipefd[0])) != NULL) { if (strchr(fi->fn, '\n')) ohshit(_("newline not allowed in pathname '%s'"), fi->fn); if (S_ISLNK(fi->st.st_mode)) { @@ -206,8 +223,9 @@ file_treewalk_feed(const char *dir, int fd_out) } } - close(pipefd[0]); - subproc_reap(pid, "find", 0); + close(sort_pipefd[0]); + subproc_reap(find_pid, "find", 0); + subproc_reap(sort_pid, "sort", 0); for (fi = symlist; fi; fi = fi->next) if (fd_write(fd_out, fi->fn, strlen(fi->fn) + 1) < 0) diff --git a/lib/dpkg/dpkg.h b/lib/dpkg/dpkg.h index 23b3dd3..0cfa340 100644 --- a/lib/dpkg/dpkg.h +++ b/lib/dpkg/dpkg.h @@ -110,6 +110,7 @@ DPKG_BEGIN_DECLS #define CAT "cat" #define FIND "find" #define DIFF "diff" +#define SORT "sort" #define FIND_EXPRSTARTCHARS "-(),!" -- 2.1.4
From d1ea3f10adec307715bd9e7ee277b5528dbd9698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <[email protected]> Date: Fri, 17 Jan 2014 12:56:13 +0100 Subject: [PATCH 2/4] Extract the creation of the control tarball to a dedicated function --- dpkg-deb/build.c | 77 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c index db9aaf3..363fb52 100644 --- a/dpkg-deb/build.c +++ b/dpkg-deb/build.c @@ -234,6 +234,40 @@ file_treewalk_feed(const char *dir, int fd_out) file_info_list_free(symlist); } +static void +create_control_tar(const char *dir, struct compress_params *control_compress_params, + int gzfd) +{ + int p1[2]; + pid_t c1, c2; + + /* Fork a tar to package the control-section of the package. */ + unsetenv("TAR_OPTIONS"); + m_pipe(p1); + c1 = subproc_fork(); + if (!c1) { + m_dup2(p1[1],1); close(p1[0]); close(p1[1]); + if (chdir(dir)) + ohshite(_("failed to chdir to '%.255s'"), dir); + if (chdir(BUILDCONTROLDIR)) + ohshite(_("failed to chdir to '%.255s'"), ".../DEBIAN"); + execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL); + ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR); + } + close(p1[1]); + + /* And run the compressor on our control archive. */ + + c2 = subproc_fork(); + if (!c2) { + compress_filter(control_compress_params, p1[0], gzfd, _("compressing control member")); + exit(0); + } + close(p1[0]); + subproc_reap(c2, _("<compress> from tar -cf"), 0); + subproc_reap(c1, "tar -cf", 0); +} + static const char *const maintainerscripts[] = { PREINSTFILE, POSTINSTFILE, @@ -512,20 +546,17 @@ do_build(const char *const *argv) arfd = creat(debar, 0644); if (arfd < 0) ohshite(_("unable to create '%.255s'"), debar); - /* Fork a tar to package the control-section of the package. */ - unsetenv("TAR_OPTIONS"); - m_pipe(p1); - c1 = subproc_fork(); - if (!c1) { - m_dup2(p1[1],1); close(p1[0]); close(p1[1]); - if (chdir(dir)) - ohshite(_("failed to chdir to '%.255s'"), dir); - if (chdir(BUILDCONTROLDIR)) - ohshite(_("failed to chdir to '%.255s'"), ".../DEBIAN"); - execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL); - ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR); + + if (opt_uniform_compression) { + control_compress_params = compress_params; + } else { + control_compress_params.type = COMPRESSOR_TYPE_GZIP; + control_compress_params.strategy = COMPRESSOR_STRATEGY_NONE; + control_compress_params.level = -1; + if (!compressor_check_params(&control_compress_params, &err)) + internerr("invalid control member compressor params: %s", err.str); } - close(p1[1]); + /* Create a temporary file to store the control data in. Immediately * unlink our temporary file so others can't mess with it. */ tfbuf = path_make_temp_template("dpkg-deb"); @@ -538,25 +569,7 @@ do_build(const char *const *argv) tfbuf); free(tfbuf); - /* And run the compressor on our control archive. */ - if (opt_uniform_compression) { - control_compress_params = compress_params; - } else { - control_compress_params.type = COMPRESSOR_TYPE_GZIP; - control_compress_params.strategy = COMPRESSOR_STRATEGY_NONE; - control_compress_params.level = -1; - if (!compressor_check_params(&control_compress_params, &err)) - internerr("invalid control member compressor params: %s", err.str); - } - - c2 = subproc_fork(); - if (!c2) { - compress_filter(&control_compress_params, p1[0], gzfd, _("compressing control member")); - exit(0); - } - close(p1[0]); - subproc_reap(c2, _("<compress> from tar -cf"), 0); - subproc_reap(c1, "tar -cf", 0); + create_control_tar(dir, &control_compress_params, gzfd); if (lseek(gzfd, 0, SEEK_SET)) ohshite(_("failed to rewind temporary file (%s)"), _("control member")); -- 2.1.4
From f9a28b246bab95f5cbf0b9af584c9a358bdf8b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <[email protected]> Date: Fri, 17 Jan 2014 12:58:15 +0100 Subject: [PATCH 3/4] Rename create_control_tar() variables to more meaningful names --- dpkg-deb/build.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c index 363fb52..ba15007 100644 --- a/dpkg-deb/build.c +++ b/dpkg-deb/build.c @@ -238,15 +238,15 @@ static void create_control_tar(const char *dir, struct compress_params *control_compress_params, int gzfd) { - int p1[2]; - pid_t c1, c2; + int tar_pipefd[2]; + pid_t tar_pid, compressor_pid; /* Fork a tar to package the control-section of the package. */ unsetenv("TAR_OPTIONS"); - m_pipe(p1); - c1 = subproc_fork(); - if (!c1) { - m_dup2(p1[1],1); close(p1[0]); close(p1[1]); + m_pipe(tar_pipefd); + tar_pid = subproc_fork(); + if (!tar_pid) { + m_dup2(tar_pipefd[1],1); close(tar_pipefd[0]); close(tar_pipefd[1]); if (chdir(dir)) ohshite(_("failed to chdir to '%.255s'"), dir); if (chdir(BUILDCONTROLDIR)) @@ -254,18 +254,18 @@ create_control_tar(const char *dir, struct compress_params *control_compress_par execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL); ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR); } - close(p1[1]); + close(tar_pipefd[1]); /* And run the compressor on our control archive. */ - c2 = subproc_fork(); - if (!c2) { - compress_filter(control_compress_params, p1[0], gzfd, _("compressing control member")); + compressor_pid = subproc_fork(); + if (!compressor_pid) { + compress_filter(control_compress_params, tar_pipefd[0], gzfd, _("compressing control member")); exit(0); } - close(p1[0]); - subproc_reap(c2, _("<compress> from tar -cf"), 0); - subproc_reap(c1, "tar -cf", 0); + close(tar_pipefd[0]); + subproc_reap(compressor_pid, _("<compress> from tar -cf"), 0); + subproc_reap(tar_pid, "tar -cf", 0); } static const char *const maintainerscripts[] = { -- 2.1.4
From 0ef0918aeefce34844c62790bbb82db8ece0e362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20Bobbio?= <[email protected]> Date: Tue, 27 Aug 2013 23:03:49 +0200 Subject: [PATCH 4/4] Also write control.tar.gz in deterministic order Closes: #719845 --- dpkg-deb/build.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/dpkg-deb/build.c b/dpkg-deb/build.c index ba15007..264bf13 100644 --- a/dpkg-deb/build.c +++ b/dpkg-deb/build.c @@ -238,32 +238,80 @@ static void create_control_tar(const char *dir, struct compress_params *control_compress_params, int gzfd) { - int tar_pipefd[2]; - pid_t tar_pid, compressor_pid; + int tar_pipefd[2], compressor_pipefd[2], sort_pipefd[2]; + pid_t tar_pid, compressor_pid, sort_pid, find_pid; /* Fork a tar to package the control-section of the package. */ unsetenv("TAR_OPTIONS"); m_pipe(tar_pipefd); + m_pipe(compressor_pipefd); tar_pid = subproc_fork(); if (!tar_pid) { - m_dup2(tar_pipefd[1],1); close(tar_pipefd[0]); close(tar_pipefd[1]); + m_dup2(tar_pipefd[0], 0); + close(tar_pipefd[0]); + close(tar_pipefd[1]); + m_dup2(compressor_pipefd[1], 1); + close(compressor_pipefd[0]); + close(compressor_pipefd[1]); if (chdir(dir)) ohshite(_("failed to chdir to '%.255s'"), dir); if (chdir(BUILDCONTROLDIR)) ohshite(_("failed to chdir to '%.255s'"), ".../DEBIAN"); - execlp(TAR, "tar", "-cf", "-", "--format=gnu", ".", NULL); + execlp(TAR, "tar", "-cf", "-", "--format=gnu", "--null", "--no-unquote", + "-T", "-", "--no-recursion", NULL); ohshite(_("unable to execute %s (%s)"), "tar -cf", TAR); } - close(tar_pipefd[1]); + close(tar_pipefd[0]); + close(compressor_pipefd[1]); /* And run the compressor on our control archive. */ compressor_pid = subproc_fork(); if (!compressor_pid) { - compress_filter(control_compress_params, tar_pipefd[0], gzfd, _("compressing control member")); + close(tar_pipefd[1]); + compress_filter(control_compress_params, compressor_pipefd[0], gzfd, _("compressing control member")); exit(0); } - close(tar_pipefd[0]); + close(compressor_pipefd[0]); + + /* We pipe the filename to sort between find and tar to get deterministic + * builds. */ + m_pipe(sort_pipefd); + sort_pid = subproc_fork(); + if (!sort_pid) { + m_dup2(sort_pipefd[0], 0); + close(sort_pipefd[0]); + close(sort_pipefd[1]); + m_dup2(tar_pipefd[1], 1); + close(tar_pipefd[0]); + close(tar_pipefd[1]); + if (setenv("LC_ALL", "C", 1 /* overwrite */)) + ohshite(_("unable to setenv")); + execlp(SORT, "sort", "--zero-terminated", NULL); + ohshite(_("unable to execute %s (%s)"), "sort", SORT); + } + close(sort_pipefd[0]); + close(tar_pipefd[1]); + + /* All the pipes are set, now lets run find, and start feeding + * filenames to tar. */ + find_pid = subproc_fork(); + if (!find_pid) { + m_dup2(sort_pipefd[1], 1); + close(sort_pipefd[0]); + close(sort_pipefd[1]); + if (chdir(dir)) + ohshite(_("failed to chdir to `%.255s'"), dir); + if (chdir(BUILDCONTROLDIR)) + ohshite(_("failed to chdir to `%.255s'"), ".../DEBIAN"); + execlp(FIND, "find", ".", "-path", "./" BUILDCONTROLDIR, "-prune", "-o", + "-print0", NULL); + ohshite(_("unable to execute %s (%s)"), "find", FIND); + } + close(sort_pipefd[1]); + + subproc_reap(find_pid, "find", 0); + subproc_reap(sort_pid, "sort", 0); subproc_reap(compressor_pid, _("<compress> from tar -cf"), 0); subproc_reap(tar_pid, "tar -cf", 0); } -- 2.1.4
signature.asc
Description: Digital signature
