Author: markj
Date: Tue Feb 13 19:28:02 2018
New Revision: 329240
URL: https://svnweb.freebsd.org/changeset/base/329240

Log:
  Add support for zstd-compressed user and kernel core dumps.
  
  This works similarly to the existing gzip compression support, but
  zstd is typically faster and gives better compression ratios.
  
  Support for this functionality must be configured by adding ZSTDIO to
  one's kernel configuration file. dumpon(8)'s new -Z option is used to
  configure zstd compression for kernel dumps. savecore(8) now recognizes
  and saves zstd-compressed kernel dumps with a .zst extension.
  
  Submitted by: cem (original version)
  Relnotes:     yes
  Sponsored by: Dell EMC Isilon
  Differential Revision:        https://reviews.freebsd.org/D13101,
                        https://reviews.freebsd.org/D13633

Modified:
  head/sbin/dumpon/dumpon.8
  head/sbin/dumpon/dumpon.c
  head/sbin/savecore/savecore.c
  head/share/man/man5/core.5
  head/sys/conf/NOTES
  head/sys/conf/files
  head/sys/conf/options
  head/sys/kern/kern_shutdown.c
  head/sys/kern/kern_sig.c
  head/sys/kern/subr_compressor.c
  head/sys/sys/compressor.h
  head/sys/sys/kerneldump.h

Modified: head/sbin/dumpon/dumpon.8
==============================================================================
--- head/sbin/dumpon/dumpon.8   Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sbin/dumpon/dumpon.8   Tue Feb 13 19:28:02 2018        (r329240)
@@ -28,7 +28,7 @@
 .\"     From: @(#)swapon.8     8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd October 24, 2017
+.Dd February 13, 2018
 .Dt DUMPON 8
 .Os
 .Sh NAME
@@ -39,6 +39,7 @@
 .Op Fl v
 .Op Fl k Ar public_key_file
 .Op Fl z
+.Op Fl Z
 .Ar special_file
 .Nm
 .Op Fl v
@@ -116,8 +117,10 @@ kernel option.
 .Pp
 The
 .Fl z
-option configures the kernel to compress the dump in gzip format before writing
-it to the dump device.
+and
+.Fl Z
+options configure the kernel to compress the dump before writing it to
+the dump device.
 This reduces the amount of space required for the dump and accelerates
 recovery with
 .Xr savecore 8
@@ -126,10 +129,22 @@ When compression is enabled, the
 .Nm
 utility will not verify that the dump device is sufficiently large for a full
 dump.
-This flag requires a kernel compiled with the
+The
+.Fl z
+and
+.Fl Z
+options cause the dump to be written in
+.Xr gzip 1
+and
+.Xr zstd 1
+format, respectively.
+These flags require a kernel compiled with the
 .Dv GZIO
-kernel option.
+or
+.Dv ZSTDIO
+kernel options.
 .Pp
+.Pp
 The
 .Fl l
 flag causes
@@ -269,15 +284,17 @@ The core was decrypted properly if
 .Xr kgdb 1
 does not print any errors.
 .Sh SEE ALSO
+.Xr gzip 1 ,
 .Xr kgdb 1 ,
+.Xr zstd 1 ,
 .Xr ddb 4 ,
 .Xr fstab 5 ,
 .Xr rc.conf 5 ,
 .Xr config 8 ,
+.Xr decryptcore 8 ,
 .Xr init 8 ,
 .Xr loader 8 ,
 .Xr rc 8 ,
-.Xr decryptcore 8 ,
 .Xr savecore 8 ,
 .Xr swapon 8 ,
 .Xr panic 9

Modified: head/sbin/dumpon/dumpon.c
==============================================================================
--- head/sbin/dumpon/dumpon.c   Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sbin/dumpon/dumpon.c   Tue Feb 13 19:28:02 2018        (r329240)
@@ -73,7 +73,7 @@ static void
 usage(void)
 {
        fprintf(stderr, "%s\n%s\n%s\n",
-           "usage: dumpon [-v] [-k public_key_file] [-z] special_file",
+           "usage: dumpon [-v] [-k public_key_file] [-Zz] special_file",
            "       dumpon [-v] off",
            "       dumpon [-v] -l");
        exit(EX_USAGE);
@@ -192,12 +192,12 @@ main(int argc, char *argv[])
        int ch;
        int i, fd;
        int do_listdumpdev = 0;
-       bool enable, gzip;
+       bool enable, gzip, zstd;
 
-       gzip = false;
+       gzip = zstd = false;
        pubkeyfile = NULL;
 
-       while ((ch = getopt(argc, argv, "k:lvz")) != -1)
+       while ((ch = getopt(argc, argv, "k:lvZz")) != -1)
                switch((char)ch) {
                case 'k':
                        pubkeyfile = optarg;
@@ -208,6 +208,9 @@ main(int argc, char *argv[])
                case 'v':
                        verbose = 1;
                        break;
+               case 'Z':
+                       zstd = true;
+                       break;
                case 'z':
                        gzip = true;
                        break;
@@ -215,6 +218,9 @@ main(int argc, char *argv[])
                        usage();
                }
 
+       if (gzip && zstd)
+               errx(EX_USAGE, "The -z and -Z options are mutually exclusive.");
+
        argc -= optind;
        argv += optind;
 
@@ -254,7 +260,7 @@ main(int argc, char *argv[])
                if (fd < 0)
                        err(EX_OSFILE, "%s", dumpdev);
 
-               if (!gzip)
+               if (!gzip && !zstd)
                        check_size(fd, dumpdev);
 
                bzero(&kda, sizeof(kda));
@@ -268,8 +274,11 @@ main(int argc, char *argv[])
 #endif
 
                kda.kda_enable = 1;
-               kda.kda_compression = gzip ? KERNELDUMP_COMP_GZIP :
-                   KERNELDUMP_COMP_NONE;
+               kda.kda_compression = KERNELDUMP_COMP_NONE;
+               if (zstd)
+                       kda.kda_compression = KERNELDUMP_COMP_ZSTD;
+               else if (gzip)
+                       kda.kda_compression = KERNELDUMP_COMP_GZIP;
                i = ioctl(fd, DIOCSKERNELDUMP, &kda);
                explicit_bzero(kda.kda_encryptedkey, kda.kda_encryptedkeysize);
                free(kda.kda_encryptedkey);

Modified: head/sbin/savecore/savecore.c
==============================================================================
--- head/sbin/savecore/savecore.c       Tue Feb 13 19:19:26 2018        
(r329239)
+++ head/sbin/savecore/savecore.c       Tue Feb 13 19:28:02 2018        
(r329240)
@@ -109,6 +109,7 @@ printheader(xo_handle_t *xo, const struct kerneldumphe
        uint64_t dumplen;
        time_t t;
        const char *stat_str;
+       const char *comp_str;
 
        xo_flush_h(xo);
        xo_emit_h(xo, "{Lwc:Dump header from device}{:dump_device/%s}\n",
@@ -123,10 +124,21 @@ printheader(xo_handle_t *xo, const struct kerneldumphe
            (long long)dumplen);
        xo_emit_h(xo, "{P:  }{Lwc:Blocksize}{:blocksize/%d}\n",
            dtoh32(h->blocksize));
-       xo_emit_h(xo, "{P:  }{Lwc:Compression}{:compression/%s}\n",
-           h->compression == KERNELDUMP_COMP_GZIP ?
-           "gzip" : "none");
-
+       switch (h->compression) {
+       case KERNELDUMP_COMP_NONE:
+               comp_str = "none";
+               break;
+       case KERNELDUMP_COMP_GZIP:
+               comp_str = "gzip";
+               break;
+       case KERNELDUMP_COMP_ZSTD:
+               comp_str = "zstd";
+               break;
+       default:
+               comp_str = "???";
+               break;
+       }
+       xo_emit_h(xo, "{P:  }{Lwc:Compression}{:compression/%s}\n", comp_str);
        t = dtoh64(h->dumptime);
        xo_emit_h(xo, "{P:  }{Lwc:Dumptime}{:dumptime/%s}", ctime(&t));
        xo_emit_h(xo, "{P:  }{Lwc:Hostname}{:hostname/%s}\n", h->hostname);
@@ -249,6 +261,8 @@ saved_dump_size(int bounds)
        dumpsize += file_size(path);
        (void)snprintf(path, sizeof(path), "vmcore.%d.gz", bounds);
        dumpsize += file_size(path);
+       (void)snprintf(path, sizeof(path), "vmcore.%d.zst", bounds);
+       dumpsize += file_size(path);
        (void)snprintf(path, sizeof(path), "textdump.tar.%d", bounds);
        dumpsize += file_size(path);
        (void)snprintf(path, sizeof(path), "textdump.tar.%d.gz", bounds);
@@ -268,6 +282,8 @@ saved_dump_remove(int bounds)
        (void)unlink(path);
        (void)snprintf(path, sizeof(path), "vmcore.%d.gz", bounds);
        (void)unlink(path);
+       (void)snprintf(path, sizeof(path), "vmcore.%d.zst", bounds);
+       (void)unlink(path);
        (void)snprintf(path, sizeof(path), "textdump.tar.%d", bounds);
        (void)unlink(path);
        (void)snprintf(path, sizeof(path), "textdump.tar.%d.gz", bounds);
@@ -282,6 +298,7 @@ symlinks_remove(void)
        (void)unlink("key.last");
        (void)unlink("vmcore.last");
        (void)unlink("vmcore.last.gz");
+       (void)unlink("vmcore.last.zstd");
        (void)unlink("vmcore_encrypted.last");
        (void)unlink("vmcore_encrypted.last.gz");
        (void)unlink("textdump.tar.last");
@@ -615,6 +632,7 @@ DoFile(const char *savedir, const char *device)
                case KERNELDUMP_COMP_NONE:
                        break;
                case KERNELDUMP_COMP_GZIP:
+               case KERNELDUMP_COMP_ZSTD:
                        if (compress && verbose)
                                printf("dump is already compressed\n");
                        compress = false;
@@ -743,7 +761,8 @@ DoFile(const char *savedir, const char *device)
                    (isencrypted ? "vmcore_encrypted" : "vmcore"), bounds);
                fp = zopen(corename, "w");
        } else if (iscompressed && !isencrypted) {
-               snprintf(corename, sizeof(corename), "vmcore.%d.gz", bounds);
+               snprintf(corename, sizeof(corename), "vmcore.%d.%s", bounds,
+                   (kdhl.compression == KERNELDUMP_COMP_GZIP) ? "gz" : "zst");
                fp = fopen(corename, "w");
        } else {
                snprintf(corename, sizeof(corename), "%s.%d",
@@ -845,9 +864,10 @@ DoFile(const char *savedir, const char *device)
                }
        }
        if (compress || iscompressed) {
-               snprintf(linkname, sizeof(linkname), "%s.last.gz",
+               snprintf(linkname, sizeof(linkname), "%s.last.%s",
                    istextdump ? "textdump.tar" :
-                   (isencrypted ? "vmcore_encrypted" : "vmcore"));
+                   (isencrypted ? "vmcore_encrypted" : "vmcore"),
+                   (kdhl.compression == KERNELDUMP_COMP_ZSTD) ? "zst" : "gz");
        } else {
                snprintf(linkname, sizeof(linkname), "%s.last",
                    istextdump ? "textdump.tar" :

Modified: head/share/man/man5/core.5
==============================================================================
--- head/share/man/man5/core.5  Tue Feb 13 19:19:26 2018        (r329239)
+++ head/share/man/man5/core.5  Tue Feb 13 19:28:02 2018        (r329240)
@@ -28,7 +28,7 @@
 .\"     @(#)core.5     8.3 (Berkeley) 12/11/93
 .\" $FreeBSD$
 .\"
-.Dd January 8, 2018
+.Dd February 13, 2018
 .Dt CORE 5
 .Os
 .Sh NAME
@@ -112,10 +112,17 @@ The following sysctl control core file compression:
 .Bl -tag -width "kern.compress_user_cores_level" -compact -offset "12345"
 .It Em kern.compress_user_cores
 Enable compression of user cores.
-A value of 1 configures gzip compression.
-gzip-compressed core files will have a suffix of
+A value of 1 configures
+.Xr gzip 1
+compression,
+and a value of 2 configures
+.Xr zstd 1
+compression.
+Compressed core files will have a suffix of
 .Ql .gz
-appended to their filenames.
+or
+.Ql .zst
+appended to their filenames depending on the selected format.
 .It Em kern.compress_user_cores_level
 Compression level.
 Defaults to 6.

Modified: head/sys/conf/NOTES
==============================================================================
--- head/sys/conf/NOTES Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sys/conf/NOTES Tue Feb 13 19:28:02 2018        (r329240)
@@ -3018,6 +3018,10 @@ options         IMAGACT_BINMISC
 # This enables support for compressed core dumps.
 options        GZIO
 
+# zstd I/O stream support
+# This enables support for Zstd compressed core dumps.
+options        ZSTDIO
+
 # BHND(4) drivers
 options                BHND_LOGLEVEL   # Logging threshold level
 

Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sys/conf/files Tue Feb 13 19:28:02 2018        (r329240)
@@ -3836,7 +3836,8 @@ kern/subr_bus_dma.c               standard
 kern/subr_bufring.c            standard
 kern/subr_capability.c         standard
 kern/subr_clock.c              standard
-kern/subr_compressor.c         standard
+kern/subr_compressor.c         standard \
+       compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd"
 kern/subr_counter.c            standard
 kern/subr_devstat.c            standard
 kern/subr_disk.c               standard

Modified: head/sys/conf/options
==============================================================================
--- head/sys/conf/options       Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sys/conf/options       Tue Feb 13 19:28:02 2018        (r329240)
@@ -222,6 +222,7 @@ TURNSTILE_PROFILING
 UMTX_PROFILING
 UMTX_CHAINS    opt_global.h
 VERBOSE_SYSINIT
+ZSTDIO         opt_zstdio.h
 
 # POSIX kernel options
 P1003_1B_MQUEUE                        opt_posix.h

Modified: head/sys/kern/kern_shutdown.c
==============================================================================
--- head/sys/kern/kern_shutdown.c       Tue Feb 13 19:19:26 2018        
(r329239)
+++ head/sys/kern/kern_shutdown.c       Tue Feb 13 19:28:02 2018        
(r329240)
@@ -174,6 +174,7 @@ struct kerneldumpcrypto {
 #endif
 
 struct kerneldumpcomp {
+       uint8_t                 kdc_format;
        struct compressor       *kdc_stream;
        uint8_t                 *kdc_buf;
        size_t                  kdc_resid;
@@ -987,12 +988,23 @@ static struct kerneldumpcomp *
 kerneldumpcomp_create(struct dumperinfo *di, uint8_t compression)
 {
        struct kerneldumpcomp *kdcomp;
+       int format;
 
-       if (compression != KERNELDUMP_COMP_GZIP)
+       switch (compression) {
+       case KERNELDUMP_COMP_GZIP:
+               format = COMPRESS_GZIP;
+               break;
+       case KERNELDUMP_COMP_ZSTD:
+               format = COMPRESS_ZSTD;
+               break;
+       default:
                return (NULL);
+       }
+
        kdcomp = malloc(sizeof(*kdcomp), M_DUMPER, M_WAITOK | M_ZERO);
+       kdcomp->kdc_format = compression;
        kdcomp->kdc_stream = compressor_init(kerneldumpcomp_write_cb,
-           COMPRESS_GZIP, di->maxiosize, kerneldump_gzlevel, di);
+           format, di->maxiosize, kerneldump_gzlevel, di);
        if (kdcomp->kdc_stream == NULL) {
                free(kdcomp, M_DUMPER);
                return (NULL);
@@ -1293,7 +1305,7 @@ dump_start(struct dumperinfo *di, struct kerneldumphea
                         * will occupy, so try to use the whole swap partition
                         * (minus the first 64KB) in the hope that the
                         * compressed dump will fit. If that doesn't turn out to
-                        * be enouch, the bounds checking in dump_write()
+                        * be enough, the bounds checking in dump_write()
                         * will catch us and cause the dump to fail.
                         */
                        dumpextent = di->mediasize - SIZEOF_METADATA -
@@ -1463,7 +1475,7 @@ dump_init_header(const struct dumperinfo *di, struct k
        if (panicstr != NULL)
                strlcpy(kdh->panicstring, panicstr, sizeof(kdh->panicstring));
        if (di->kdcomp != NULL)
-               kdh->compression = KERNELDUMP_COMP_GZIP;
+               kdh->compression = di->kdcomp->kdc_format;
        kdh->parity = kerneldump_parity(kdh);
 }
 

Modified: head/sys/kern/kern_sig.c
==============================================================================
--- head/sys/kern/kern_sig.c    Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sys/kern/kern_sig.c    Tue Feb 13 19:28:02 2018        (r329240)
@@ -3253,7 +3253,8 @@ sysctl_debug_num_cores_check (SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_debug, OID_AUTO, ncores, CTLTYPE_INT|CTLFLAG_RW,
            0, sizeof(int), sysctl_debug_num_cores_check, "I", "");
 
-#define        GZ_SUFFIX       ".gz"
+#define        GZIP_SUFFIX     ".gz"
+#define        ZSTD_SUFFIX     ".zst"
 
 int compress_user_cores = 0;
 
@@ -3273,7 +3274,9 @@ sysctl_compress_user_cores(SYSCTL_HANDLER_ARGS)
 }
 SYSCTL_PROC(_kern, OID_AUTO, compress_user_cores, CTLTYPE_INT | CTLFLAG_RWTUN,
     0, sizeof(int), sysctl_compress_user_cores, "I",
-    "Enable compression of user corefiles (" __XSTRING(COMPRESS_GZIP) " = 
gzip)");
+    "Enable compression of user corefiles ("
+    __XSTRING(COMPRESS_GZIP) " = gzip, "
+    __XSTRING(COMPRESS_ZSTD) " = zstd)");
 
 int compress_user_cores_level = 6;
 SYSCTL_INT(_kern, OID_AUTO, compress_user_cores_level, CTLFLAG_RWTUN,
@@ -3377,7 +3380,9 @@ corefile_open(const char *comm, uid_t uid, pid_t pid, 
        sx_sunlock(&corefilename_lock);
        free(hostname, M_TEMP);
        if (compress == COMPRESS_GZIP)
-               sbuf_printf(&sb, GZ_SUFFIX);
+               sbuf_printf(&sb, GZIP_SUFFIX);
+       else if (compress == COMPRESS_ZSTD)
+               sbuf_printf(&sb, ZSTD_SUFFIX);
        if (sbuf_error(&sb) != 0) {
                log(LOG_ERR, "pid %ld (%s), uid (%lu): corename is too "
                    "long\n", (long)pid, comm, (u_long)uid);

Modified: head/sys/kern/subr_compressor.c
==============================================================================
--- head/sys/kern/subr_compressor.c     Tue Feb 13 19:19:26 2018        
(r329239)
+++ head/sys/kern/subr_compressor.c     Tue Feb 13 19:28:02 2018        
(r329240)
@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2014, 2017 Mark Johnston <ma...@freebsd.org>
+ * Copyright (c) 2017 Conrad Meyer <c...@freebsd.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -34,8 +35,10 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_gzio.h"
+#include "opt_zstdio.h"
 
 #include <sys/param.h>
+#include <sys/systm.h>
 
 #include <sys/compressor.h>
 #include <sys/kernel.h>
@@ -244,6 +247,232 @@ struct compressor_methods gzip_methods = {
 DATA_SET(compressors, gzip_methods);
 
 #endif /* GZIO */
+
+#ifdef ZSTDIO
+
+#define        ZSTD_STATIC_LINKING_ONLY
+#include <contrib/zstd/lib/zstd.h>
+
+struct zstdio_stream {
+       ZSTD_CCtx       *zst_stream;
+       ZSTD_inBuffer   zst_inbuffer;
+       ZSTD_outBuffer  zst_outbuffer;
+       uint8_t *       zst_buffer;     /* output buffer */
+       size_t          zst_maxiosz;    /* Max output IO size */
+       off_t           zst_off;        /* offset into the output stream */
+       void *          zst_static_wkspc;
+};
+
+static void    *zstdio_init(size_t maxiosize, int level);
+static void    zstdio_reset(void *stream);
+static int     zstdio_write(void *stream, void *data, size_t len,
+                   compressor_cb_t, void *);
+static void    zstdio_fini(void *stream);
+
+static void *
+zstdio_init(size_t maxiosize, int level)
+{
+       ZSTD_CCtx *dump_compressor;
+       struct zstdio_stream *s;
+       void *wkspc, *owkspc, *buffer;
+       size_t wkspc_size, buf_size;
+
+       wkspc_size = ZSTD_estimateCStreamSize(level);
+       owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
+           M_WAITOK | M_NODUMP);
+       /* Zstd API requires 8-byte alignment. */
+       if ((uintptr_t)wkspc % 8 != 0)
+               wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
+
+       dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
+       if (dump_compressor == NULL) {
+               free(owkspc, M_COMPRESS);
+               printf("%s: workspace too small.\n", __func__);
+               return (NULL);
+       }
+
+       (void)ZSTD_CCtx_setParameter(dump_compressor, ZSTD_p_checksumFlag, 1);
+
+       buf_size = ZSTD_CStreamOutSize() * 2;
+       buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
+
+       s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
+       s->zst_buffer = buffer;
+       s->zst_outbuffer.dst = buffer;
+       s->zst_outbuffer.size = buf_size;
+       s->zst_maxiosz = maxiosize;
+       s->zst_stream = dump_compressor;
+       s->zst_static_wkspc = owkspc;
+
+       zstdio_reset(s);
+
+       return (s);
+}
+
+static void
+zstdio_reset(void *stream)
+{
+       struct zstdio_stream *s;
+       size_t res;
+
+       s = stream;
+       res = ZSTD_resetCStream(s->zst_stream, 0);
+       if (ZSTD_isError(res))
+               panic("%s: could not reset stream %p: %s\n", __func__, s,
+                   ZSTD_getErrorName(res));
+
+       s->zst_off = 0;
+       s->zst_inbuffer.src = NULL;
+       s->zst_inbuffer.size = 0;
+       s->zst_inbuffer.pos = 0;
+       s->zst_outbuffer.pos = 0;
+}
+
+static int
+zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
+{
+       size_t bytes_to_dump;
+       int error;
+
+       /* Flush as many full output blocks as possible. */
+       /* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
+       while (s->zst_outbuffer.pos >= 4096) {
+               bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
+
+               if (bytes_to_dump > s->zst_maxiosz)
+                       bytes_to_dump = s->zst_maxiosz;
+
+               error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
+               if (error != 0)
+                       return (error);
+
+               /*
+                * Shift any non-full blocks up to the front of the output
+                * buffer.
+                */
+               s->zst_outbuffer.pos -= bytes_to_dump;
+               memmove(s->zst_outbuffer.dst,
+                   (char *)s->zst_outbuffer.dst + bytes_to_dump,
+                   s->zst_outbuffer.pos);
+               s->zst_off += bytes_to_dump;
+       }
+       return (0);
+}
+
+static int
+zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
+{
+       size_t rc, lastpos;
+       int error;
+
+       /*
+        * Positive return indicates unflushed data remaining; need to call
+        * endStream again after clearing out room in output buffer.
+        */
+       rc = 1;
+       lastpos = s->zst_outbuffer.pos;
+       while (rc > 0) {
+               rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
+               if (ZSTD_isError(rc)) {
+                       printf("%s: ZSTD_endStream failed (%s)\n", __func__,
+                           ZSTD_getErrorName(rc));
+                       return (EIO);
+               }
+               if (lastpos == s->zst_outbuffer.pos) {
+                       printf("%s: did not make forward progress endStream 
%zu\n",
+                           __func__, lastpos);
+                       return (EIO);
+               }
+
+               error = zst_flush_intermediate(s, cb, arg);
+               if (error != 0)
+                       return (error);
+
+               lastpos = s->zst_outbuffer.pos;
+       }
+
+       /*
+        * We've already done an intermediate flush, so all full blocks have
+        * been written.  Only a partial block remains.  Padding happens in a
+        * higher layer.
+        */
+       if (s->zst_outbuffer.pos != 0) {
+               error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
+                   arg);
+               if (error != 0)
+                       return (error);
+       }
+
+       return (0);
+}
+
+static int
+zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
+    void *arg)
+{
+       struct zstdio_stream *s;
+       size_t lastpos, rc;
+       int error;
+
+       s = stream;
+       if (data == NULL)
+               return (zstdio_flush(s, cb, arg));
+
+       s->zst_inbuffer.src = data;
+       s->zst_inbuffer.size = len;
+       s->zst_inbuffer.pos = 0;
+       lastpos = 0;
+
+       while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
+               rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
+                   &s->zst_inbuffer);
+               if (ZSTD_isError(rc)) {
+                       printf("%s: Compress failed on %p! (%s)\n",
+                           __func__, data, ZSTD_getErrorName(rc));
+                       return (EIO);
+               }
+
+               if (lastpos == s->zst_inbuffer.pos) {
+                       /*
+                        * XXX: May need flushStream to make forward progress
+                        */
+                       printf("ZSTD: did not make forward progress @pos %zu\n",
+                           lastpos);
+                       return (EIO);
+               }
+               lastpos = s->zst_inbuffer.pos;
+
+               error = zst_flush_intermediate(s, cb, arg);
+               if (error != 0)
+                       return (error);
+       }
+       return (0);
+}
+
+static void
+zstdio_fini(void *stream)
+{
+       struct zstdio_stream *s;
+
+       s = stream;
+       if (s->zst_static_wkspc != NULL)
+               free(s->zst_static_wkspc, M_COMPRESS);
+       else
+               ZSTD_freeCCtx(s->zst_stream);
+       free(s->zst_buffer, M_COMPRESS);
+       free(s, M_COMPRESS);
+}
+
+static struct compressor_methods zstd_methods = {
+       .format = COMPRESS_ZSTD,
+       .init = zstdio_init,
+       .reset = zstdio_reset,
+       .write = zstdio_write,
+       .fini = zstdio_fini,
+};
+DATA_SET(compressors, zstd_methods);
+
+#endif /* ZSTDIO */
 
 bool
 compressor_avail(int format)

Modified: head/sys/sys/compressor.h
==============================================================================
--- head/sys/sys/compressor.h   Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sys/sys/compressor.h   Tue Feb 13 19:28:02 2018        (r329240)
@@ -35,6 +35,7 @@
 
 /* Supported formats. */
 #define        COMPRESS_GZIP   1
+#define        COMPRESS_ZSTD   2
 
 typedef int (*compressor_cb_t)(void *, size_t, off_t, void *);
 

Modified: head/sys/sys/kerneldump.h
==============================================================================
--- head/sys/sys/kerneldump.h   Tue Feb 13 19:19:26 2018        (r329239)
+++ head/sys/sys/kerneldump.h   Tue Feb 13 19:28:02 2018        (r329240)
@@ -59,6 +59,7 @@
 
 #define        KERNELDUMP_COMP_NONE            0
 #define        KERNELDUMP_COMP_GZIP            1
+#define        KERNELDUMP_COMP_ZSTD            2
 
 #define        KERNELDUMP_ENC_NONE             0
 #define        KERNELDUMP_ENC_AES_256_CBC      1
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"

Reply via email to