The branch main has been updated by des:
URL:
https://cgit.FreeBSD.org/src/commit/?id=f9349d427431eaa5b3bd7e9e7e87725a138facd1
commit f9349d427431eaa5b3bd7e9e7e87725a138facd1
Author: Dag-Erling Smørgrav <d...@freebsd.org>
AuthorDate: 2022-11-02 13:41:44 +0000
Commit: Dag-Erling Smørgrav <d...@freebsd.org>
CommitDate: 2022-11-04 12:42:09 +0000
gzip: Add support for decompressing zstd files.
Sponsored by: Klara, Inc.
Differential Revision: https://reviews.freebsd.org/D37236
---
usr.bin/gzip/Makefile | 4 +-
usr.bin/gzip/Makefile.depend | 1 +
usr.bin/gzip/gzip.1 | 4 +-
usr.bin/gzip/gzip.c | 60 ++++++++++++++++++++++-------
usr.bin/gzip/unzstd.c | 89 ++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 143 insertions(+), 15 deletions(-)
diff --git a/usr.bin/gzip/Makefile b/usr.bin/gzip/Makefile
index c8ec4743f98d..227dce362a45 100644
--- a/usr.bin/gzip/Makefile
+++ b/usr.bin/gzip/Makefile
@@ -6,7 +6,7 @@
PROG= gzip
MAN= gzip.1 gzexe.1 zdiff.1 zforce.1 zmore.1 znew.1
-LIBADD= z lzma
+LIBADD= z lzma zstd
.if ${MK_BZIP2_SUPPORT} != "no"
LIBADD+= bz2
@@ -14,6 +14,8 @@ LIBADD+= bz2
CFLAGS+= -DNO_BZIP2_SUPPORT
.endif
+CFLAGS+= -I${SRCTOP}/sys/contrib/zstd/lib
+
SCRIPTS= gzexe zdiff zforce zmore znew
MLINKS+= gzip.1 gunzip.1 \
diff --git a/usr.bin/gzip/Makefile.depend b/usr.bin/gzip/Makefile.depend
index 02d864779e18..9a4b70335fec 100644
--- a/usr.bin/gzip/Makefile.depend
+++ b/usr.bin/gzip/Makefile.depend
@@ -10,6 +10,7 @@ DIRDEPS = \
lib/liblzma \
lib/libthr \
lib/libz \
+ lib/libzstd \
.include <dirdeps.mk>
diff --git a/usr.bin/gzip/gzip.1 b/usr.bin/gzip/gzip.1
index a8293b748832..356568a2ba11 100644
--- a/usr.bin/gzip/gzip.1
+++ b/usr.bin/gzip/gzip.1
@@ -25,7 +25,7 @@
.\" SUCH DAMAGE.
.\"
.\" $FreeBSD$
-.Dd January 7, 2019
+.Dd November 2, 2022
.Dt GZIP 1
.Os
.Sh NAME
@@ -110,6 +110,7 @@ is also capable of decompressing files compressed using
.Xr compress 1 ,
.Xr bzip2 1 ,
.Ar lzip ,
+.Xr zstd 1 ,
or
.Xr xz 1 .
.Sh OPTIONS
@@ -203,6 +204,7 @@ Report progress to standard error.
.Sh SEE ALSO
.Xr bzip2 1 ,
.Xr compress 1 ,
+.Xr zstd 1 ,
.Xr xz 1 ,
.Xr fts 3 ,
.Xr zlib 3
diff --git a/usr.bin/gzip/gzip.c b/usr.bin/gzip/gzip.c
index bdafb9e0b457..a7f64b61d053 100644
--- a/usr.bin/gzip/gzip.c
+++ b/usr.bin/gzip/gzip.c
@@ -86,6 +86,9 @@ enum filetype {
#endif
#ifndef NO_LZ_SUPPORT
FT_LZ,
+#endif
+#ifndef NO_ZSTD_SUPPORT
+ FT_ZSTD,
#endif
FT_LAST,
FT_UNKNOWN
@@ -118,6 +121,12 @@ enum filetype {
#define LZ_MAGIC "LZIP"
#endif
+#ifndef NO_ZSTD_SUPPORT
+#include <zstd.h>
+#define ZSTD_SUFFIX ".zst"
+#define ZSTD_MAGIC "\050\265\057\375"
+#endif
+
#define GZ_SUFFIX ".gz"
#define BUFLEN (64 * 1024)
@@ -164,6 +173,9 @@ static suffixes_t suffixes[] = {
#endif
#ifndef NO_LZ_SUPPORT
SUFFIX(LZ_SUFFIX, ""),
+#endif
+#ifndef NO_ZSTD_SUPPORT
+ SUFFIX(ZSTD_SUFFIX, ""),
#endif
SUFFIX(GZ_SUFFIX, ""), /* Overwritten by -S "" */
#undef SUFFIX
@@ -221,7 +233,7 @@ static const char *infile; /* name of file
coming in */
static void maybe_err(const char *fmt, ...) __printflike(1, 2) __dead2;
#if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) || \
- !defined(NO_XZ_SUPPORT)
+ !defined(NO_XZ_SUPPORT) || !defined(NO_ZSTD_SUPPORT)
static void maybe_errx(const char *fmt, ...) __printflike(1, 2) __dead2;
#endif
static void maybe_warn(const char *fmt, ...) __printflike(1, 2);
@@ -284,6 +296,10 @@ static off_t unxz_len(int);
static off_t unlz(int, int, char *, size_t, off_t *);
#endif
+#ifndef NO_ZSTD_SUPPORT
+static off_t unzstd(int, int, char *, size_t, off_t *);
+#endif
+
static const struct option longopts[] = {
{ "stdout", no_argument, 0, 'c' },
{ "to-stdout", no_argument, 0, 'c' },
@@ -467,7 +483,7 @@ maybe_err(const char *fmt, ...)
}
#if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) || \
- !defined(NO_XZ_SUPPORT)
+ !defined(NO_XZ_SUPPORT) || !defined(NO_ZSTD_SUPPORT)
/* ... without an errno. */
void
maybe_errx(const char *fmt, ...)
@@ -1098,33 +1114,32 @@ file_gettype(u_char *buf)
if (buf[0] == GZIP_MAGIC0 &&
(buf[1] == GZIP_MAGIC1 || buf[1] == GZIP_OMAGIC1))
return FT_GZIP;
- else
#ifndef NO_BZIP2_SUPPORT
- if (memcmp(buf, BZIP2_MAGIC, 3) == 0 &&
+ else if (memcmp(buf, BZIP2_MAGIC, 3) == 0 &&
buf[3] >= '0' && buf[3] <= '9')
return FT_BZIP2;
- else
#endif
#ifndef NO_COMPRESS_SUPPORT
- if (memcmp(buf, Z_MAGIC, 2) == 0)
+ else if (memcmp(buf, Z_MAGIC, 2) == 0)
return FT_Z;
- else
#endif
#ifndef NO_PACK_SUPPORT
- if (memcmp(buf, PACK_MAGIC, 2) == 0)
+ else if (memcmp(buf, PACK_MAGIC, 2) == 0)
return FT_PACK;
- else
#endif
#ifndef NO_XZ_SUPPORT
- if (memcmp(buf, XZ_MAGIC, 4) == 0) /* XXX: We only have 4 bytes */
+ else if (memcmp(buf, XZ_MAGIC, 4) == 0) /* XXX: We only have 4 bytes */
return FT_XZ;
- else
#endif
#ifndef NO_LZ_SUPPORT
- if (memcmp(buf, LZ_MAGIC, 4) == 0)
+ else if (memcmp(buf, LZ_MAGIC, 4) == 0)
return FT_LZ;
- else
#endif
+#ifndef NO_ZSTD_SUPPORT
+ else if (memcmp(buf, ZSTD_MAGIC, 4) == 0)
+ return FT_ZSTD;
+#endif
+ else
return FT_UNKNOWN;
}
@@ -1586,6 +1601,16 @@ file_uncompress(char *file, char *outfile, size_t
outsize)
size = unlz(fd, zfd, NULL, 0, NULL);
break;
#endif
+
+#ifndef NO_ZSTD_SUPPORT
+ case FT_ZSTD:
+ if (lflag) {
+ maybe_warnx("no -l with zstd files");
+ goto lose;
+ }
+ size = unzstd(fd, zfd, NULL, 0, NULL);
+ break;
+#endif
case FT_UNKNOWN:
if (lflag) {
maybe_warnx("no -l for unknown filetypes");
@@ -1813,6 +1838,12 @@ handle_stdin(void)
usize = unlz(STDIN_FILENO, STDOUT_FILENO,
(char *)fourbytes, sizeof fourbytes, &gsize);
break;
+#endif
+#ifndef NO_ZSTD_SUPPORT
+ case FT_ZSTD:
+ usize = unzstd(STDIN_FILENO, STDOUT_FILENO,
+ (char *)fourbytes, sizeof fourbytes, &gsize);
+ break;
#endif
}
@@ -2188,6 +2219,9 @@ display_version(void)
#ifndef NO_LZ_SUPPORT
#include "unlz.c"
#endif
+#ifndef NO_ZSTD_SUPPORT
+#include "unzstd.c"
+#endif
static ssize_t
read_retry(int fd, void *buf, size_t sz)
diff --git a/usr.bin/gzip/unzstd.c b/usr.bin/gzip/unzstd.c
new file mode 100644
index 000000000000..4536f3119ace
--- /dev/null
+++ b/usr.bin/gzip/unzstd.c
@@ -0,0 +1,89 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2022 Klara, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* This file is #included by gzip.c */
+
+static off_t
+unzstd(int in, int out, char *pre, size_t prelen, off_t *bytes_in)
+{
+ static char *ibuf, *obuf;
+ ZSTD_inBuffer zib;
+ ZSTD_outBuffer zob;
+ ZSTD_DCtx *zds;
+ ssize_t res;
+ size_t zres;
+ size_t bytes_out = 0;
+ int eof = 0;
+
+ if (ibuf == NULL)
+ ibuf = malloc(BUFLEN);
+ if (obuf == NULL)
+ obuf = malloc(BUFLEN);
+ if (ibuf == NULL || obuf == NULL)
+ maybe_err("malloc");
+
+ zds = ZSTD_createDStream();
+ ZSTD_initDStream(zds);
+
+ zib.src = pre;
+ zib.size = prelen;
+ zib.pos = 0;
+ if (bytes_in != NULL)
+ *bytes_in = prelen;
+ zob.dst = obuf;
+ zob.size = BUFLEN;
+ zob.pos = 0;
+
+ while (!eof) {
+ if (zib.pos >= zib.size) {
+ res = read(in, ibuf, BUFLEN);
+ if (res < 0)
+ maybe_err("read");
+ if (res == 0)
+ eof = 1;
+ infile_newdata(res);
+ zib.src = ibuf;
+ zib.size = res;
+ zib.pos = 0;
+ if (bytes_in != NULL)
+ *bytes_in += res;
+ }
+ zres = ZSTD_decompressStream(zds, &zob, &zib);
+ if (ZSTD_isError(zres)) {
+ maybe_errx("%s", ZSTD_getErrorName(zres));
+ }
+ if (zob.pos > 0) {
+ res = write(out, obuf, zob.pos);
+ if (res < 0)
+ maybe_err("write");
+ zob.pos = 0;
+ bytes_out += res;
+ }
+ }
+ ZSTD_freeDStream(zds);
+ return (bytes_out);
+}
