RPM Package Manager, CVS Repository http://rpm5.org/cvs/ ____________________________________________________________________________
Server: rpm5.org Name: Jeff Johnson Root: /v/rpm/cvs Email: j...@rpm5.org Module: rpm Date: 31-May-2017 01:48:23 Branch: rpm-5_4 Handle: 2017053023482300 Added files: (Branch: rpm-5_4) rpm/rpmio zstio.c Modified files: (Branch: rpm-5_4) rpm/rpmio Makefile.am Log: - zstio: stub in a zstd implementation. Summary: Revision Changes Path 1.293.2.83 +6 -2 rpm/rpmio/Makefile.am 1.1.2.1 +486 -0 rpm/rpmio/zstio.c ____________________________________________________________________________ patch -p0 <<'@@ .' Index: rpm/rpmio/Makefile.am ============================================================================ $ cvs diff -u -r1.293.2.82 -r1.293.2.83 Makefile.am --- rpm/rpmio/Makefile.am 28 May 2017 18:39:22 -0000 1.293.2.82 +++ rpm/rpmio/Makefile.am 30 May 2017 23:48:23 -0000 1.293.2.83 @@ -31,7 +31,7 @@ dumpasn1 lookup3 trel twitter github tmicrojson #noinst_PROGRAMS += b2sum tset tblake2b tblake2bp tblake2s tblake2sp tgfs -EXTRA_PROGRAMS += b2sum tset tblake2b tblake2bp tblake2s tblake2sp test tcpuid +EXTRA_PROGRAMS += aiocat b2sum tset tblake2b tblake2bp tblake2s tblake2sp test tcpuid noinst_PROGRAMS += tgfs if WITH_LIBGIT2 noinst_PROGRAMS += tgit @@ -258,7 +258,8 @@ url.c \ xzdio.c \ yajl.c \ - yarn.c + yarn.c \ + zstio.c librpmio_la_LDFLAGS = -no-undefined -release $(LT_CURRENT).$(LT_REVISION) if HAVE_LD_VERSION_SCRIPT @@ -481,6 +482,9 @@ # testdata/wintestinput3 \ # testdata/wintestoutput3 +aiocat_SOURCES = aiocat.c +aiocat_LDADD = $(RPMIO_LDADD_COMMON) + b2sum_SOURCES = b2sum.c b2sum_LDADD = $(RPMIO_LDADD_COMMON) @@ . patch -p0 <<'@@ .' Index: rpm/rpmio/zstio.c ============================================================================ $ cvs diff -u -r0 -r1.1.2.1 zstio.c --- /dev/null 2017-05-31 01:45:42.000000000 +0200 +++ zstio.c 2017-05-31 01:48:23.586825184 +0200 @@ -0,0 +1,486 @@ +/** \ingroup rpmio + * \file rpmio/zstio.c + * Support for ZSTD compression library. + */ + +#include "system.h" + +#if defined(HAVE_STDBOOL_H) +#include <stdbool.h> +#else +typedef enum { true = 1, false = 0 } bool; +#endif + +#include "rpmio_internal.h" +#include <rpmmacro.h> +#include <rpmcb.h> + +#if defined(WITH_ZSTD) + +#include <zstd.h> + +#include "debug.h" + +#define ZSTONLY(fd) assert(fdGetIo(fd) == zstio) + +typedef struct cpio_state_s { + uint32_t n; /* byte progress in cpio header */ + uint32_t mode; /* file attributes */ + uint32_t nlnk; + uint32_t size; +} * cpio_state; + +#define RSYNC_WIN 4096 + +typedef struct rsync_state_s { + uint32_t n; /* number of elements in the window */ + uint32_t sum; /* current sum */ + unsigned char win[RSYNC_WIN]; /* window elements */ +} * rsync_state; + +typedef void * gzFile; /* XXX */ + +typedef struct zstdFILE_s { + gzFile gz; /* gzFile is a pointer */ + struct rsync_state_s rs; + struct cpio_state_s cs; + uint32_t nb; /* bytes pending for sync */ +} * zstdFILE; /* like FILE, to use with star */ + + +#ifdef __cplusplus +GENfree(zstdFILE) +#endif /* __cplusplus */ + +/* Should zstflush be called only after RSYNC_WIN boundaries? */ +static int enable_rsync = 1; + +/* =============================================================== */ +static +const char * zsferror(gzFile gz, int *errnum) +{ + return ""; +} + +static +gzFile zsfopen(const char *path, const char * fmode, int fdno) +{ + return NULL; +} + +static +int zsfread(gzFile gz, void *buf, unsigned count) +{ + int rc = -1; + return rc; +} + +static +int zsfwrite(gzFile gz, const void *buf, unsigned count) +{ + int rc = -1; + return rc; +} + +static +int zsfseek(gzFile gz, off_t offset, int whence) +{ + int rc = -1; + return rc; +} + +static +int zsfclose(gzFile gz) +{ + int rc = -1; + return rc; +} + +static +int zsfflush(gzFile gz, int flush) +{ + int rc = -1; + return rc; +} + +/* =============================================================== */ +/* from ../lib/cpio.h */ +#define CPIO_NEWC_MAGIC "070701" +#define PHYS_HDR_SIZE 110 + +#define OFFSET_MODE (sizeof(CPIO_NEWC_MAGIC)-1 + 1*8) +#define OFFSET_NLNK (sizeof(CPIO_NEWC_MAGIC)-1 + 4*8) +#define OFFSET_SIZE (sizeof(CPIO_NEWC_MAGIC)-1 + 6*8) + +static inline +int hex(char c) +{ + if (c >= '0' && c <= '9') + return (int)(c - '0'); + else if (c >= 'a' && c <= 'f') + return (int)(c - 'a') + 10; + else if (c >= 'A' && c <= 'F') + return (int)(c - 'A') + 10; + return -1; +} + +static inline +bool cpio_next(cpio_state s, unsigned char c) +{ + if (s->n >= sizeof(CPIO_NEWC_MAGIC)-1) { + int d = hex(c); + if (d < 0) { + s->n = 0; + return false; + } + if (0){} /* indent */ + else if (s->n >= OFFSET_MODE && s->n < OFFSET_MODE+8) { + if (s->n == OFFSET_MODE) + s->mode = 0; + else + s->mode <<= 4; + s->mode |= d; + } + else if (s->n >= OFFSET_NLNK && s->n < OFFSET_NLNK+8) { + if (s->n == OFFSET_NLNK) + s->nlnk = 0; + else + s->nlnk <<= 4; + s->nlnk |= d; + } + else if (s->n >= OFFSET_SIZE && s->n < OFFSET_SIZE+8) { + if (s->n == OFFSET_SIZE) + s->size = 0; + else + s->size <<= 4; + s->size |= d; + } + s->n++; + if (s->n >= PHYS_HDR_SIZE) { + s->n = 0; + if (!S_ISREG(s->mode) || s->nlnk != 1) + /* no file data */ + s->size = 0; + return true; + } + } + else if (CPIO_NEWC_MAGIC[s->n] == c) { + s->n++; + } + else { + s->n = 0; + } + return false; +} + +/* =============================================================== */ +static inline +bool rsync_next(rsync_state s, unsigned char c) +{ + uint32_t i; + + if (s->n < RSYNC_WIN) { /* not enough elements */ + s->sum += (uint32_t)c; /* update the sum */ + s->win[s->n++] = c; /* remember the element */ + return false; /* no match */ + } + i = s->n++ % RSYNC_WIN; /* wrap up */ + s->sum -= (uint32_t)s->win[i]; /* move the window on */ + s->sum += (uint32_t)c; + s->win[i] = c; + if (s->sum % RSYNC_WIN == 0) { /* match */ + s->n = 0; /* reset */ + s->sum = 0; + return true; + } + return false; +} + +#define CHUNK 4096 + +static inline +bool sync_hint(zstdFILE zsf, unsigned char c) +{ + bool cpio_hint; + bool rsync_hint; + + zsf->nb++; + cpio_hint = cpio_next(&zsf->cs, c); + if (cpio_hint) { + /* cpio header/data boundary */ + zsf->rs.n = zsf->rs.sum = 0; + if (zsf->nb >= 2*CHUNK) + /* better sync here */ + goto cpio_sync; + if (zsf->cs.size < CHUNK) + /* file is too small */ + return false; + if (zsf->nb < CHUNK/2) + /* not enough pending bytes */ + return false; + cpio_sync: + zsf->nb = 0; + return true; + } + rsync_hint = rsync_next(&zsf->rs, c); + if (rsync_hint) { + /* rolling checksum match */ + assert(zsf->nb >= RSYNC_WIN); + zsf->nb = 0; + return true; + } + return false; +} + +static ssize_t +rsyncable_write(zstdFILE zsf, const unsigned char *const buf, const size_t len) +{ + ssize_t rc; + size_t n; + ssize_t n_written = 0; + const unsigned char *begin = buf; + size_t i; + + for (i = 0; i < len; i++) { + if (!sync_hint(zsf, buf[i])) + continue; + n = i + 1 - (begin - buf); + rc = zsfwrite(zsf->gz, (const char *)begin, (unsigned)n); + if (rc < 0) + return (n_written ? n_written : rc); + n_written += rc; + if (rc < (ssize_t)n) + return n_written; + begin += n; + rc = zsfflush(zsf->gz, 0); + if (rc < 0) + return (n_written ? n_written : rc); + } + if (begin < buf + len) { + n = len - (begin - buf); + rc = zsfwrite(zsf->gz, (const char *)begin, (unsigned)n); + if (rc < 0) + return (n_written ? n_written : rc); + n_written += rc; + } + return n_written; +} + +/* =============================================================== */ + +RPM_GNUC_PURE +static inline +void * zstFileno(FD_t fd) +{ + void * rc = NULL; + int i; + + FDSANE(fd); + for (i = fd->nfps; i >= 0; i--) { + FDSTACK_t * fps = &fd->fps[i]; + if (fps->io != zstio) + continue; + rc = fps->fp; + break; + } + + return rc; +} + +static +FD_t zstOpen(const char * path, const char * fmode) +{ + FD_t fd; + zstdFILE zsf; + mode_t mode = (fmode && fmode[0] == 'w' ? O_WRONLY : O_RDONLY); + + zsf = (zstdFILE) xcalloc(1, sizeof(*zsf)); + zsf->gz = zsfopen(path, fmode, -1); + if (zsf->gz == NULL) { + zsf = _free(zsf); + return NULL; + } + fd = fdNew("open (zstOpen)"); + fdPop(fd); fdPush(fd, zstio, zsf, -1); + fdSetOpen(fd, path, -1, mode); + +DBGIO(fd, (stderr, "==>\tzstOpen(\"%s\", \"%s\") fd %p %s\n", path, fmode, (fd ? fd : NULL), fdbg(fd))); + return fdLink(fd, "zstOpen"); +} + +static FD_t zstFdopen(void * cookie, const char *fmode) +{ + FD_t fd = c2f(cookie); + int fdno; + zstdFILE zsf; + + if (fmode == NULL) return NULL; + fdno = fdSetFdno(fd, -1); /* XXX skip the fdio close */ + if (fdno < 0) return NULL; + zsf = (zstdFILE) xcalloc(1, sizeof(*zsf)); + zsf->gz = zsfopen(NULL, fmode, fdno); + if (zsf->gz == NULL) { + zsf = _free(zsf); + return NULL; + } + + fdPush(fd, zstio, zsf, fdno); /* Push zstio onto stack */ + + return fdLink(fd, "zstFdopen"); +} + +static int zstFlush(void * cookie) +{ + FD_t fd = c2f(cookie); + zstdFILE zsf; + zsf = (zstdFILE) zstFileno(fd); + if (zsf == NULL) return -2; + return zsfflush(zsf->gz, 0); +} + +/* =============================================================== */ +static ssize_t zstRead(void * cookie, char * buf, size_t count) +{ + FD_t fd = c2f(cookie); + zstdFILE zsf; + ssize_t rc; + + if (fd == NULL || fd->bytesRemain == 0) return 0; /* XXX simulate EOF */ + + zsf = (zstdFILE) zstFileno(fd); + if (zsf == NULL) return -2; /* XXX can't happen */ + + fdstat_enter(fd, FDSTAT_READ); + rc = zsfread(zsf->gz, buf, (unsigned)count); +DBGIO(fd, (stderr, "==>\tzstRead(%p,%p,%u) rc %lx %s\n", cookie, buf, (unsigned)count, (unsigned long)rc, fdbg(fd))); + if (rc < 0) { + int zerror = 0; + fd->errcookie = zsferror(zsf->gz, &zerror); +#ifdef DYING + if (zerror == Z_ERRNO) { + fd->syserrno = errno; + fd->errcookie = strerror(fd->syserrno); + } +#endif + } else { + fdstat_exit(fd, FDSTAT_READ, (rc > 0 ? rc : 0)); + if (fd->ndigests > 0 && rc > 0) fdUpdateDigests(fd, (const unsigned char *)buf, rc); + } + return rc; +} + +static ssize_t zstWrite(void * cookie, const char * buf, size_t count) +{ + FD_t fd = c2f(cookie); + zstdFILE zsf; + ssize_t rc; + + if (fd == NULL || fd->bytesRemain == 0) return 0; /* XXX simulate EOF */ + + if (fd->ndigests > 0 && count > 0) fdUpdateDigests(fd, (const unsigned char *)buf, count); + + zsf = (zstdFILE) zstFileno(fd); + if (zsf == NULL) return -2; /* XXX can't happen */ + + fdstat_enter(fd, FDSTAT_WRITE); + if (enable_rsync) + rc = rsyncable_write(zsf, (const unsigned char *)buf, (unsigned)count); + else + rc = zsfwrite(zsf->gz, (void *)buf, (unsigned)count); +DBGIO(fd, (stderr, "==>\tzstWrite(%p,%p,%u) rc %lx %s\n", cookie, buf, (unsigned)count, (unsigned long)rc, fdbg(fd))); + if (rc < (ssize_t)count) { + int zerror = 0; + fd->errcookie = zsferror(zsf->gz, &zerror); +#ifdef DYING + if (zerror == Z_ERRNO) { + fd->syserrno = errno; + fd->errcookie = strerror(fd->syserrno); + } +#endif + } + if (rc > 0) + fdstat_exit(fd, FDSTAT_WRITE, rc); + return rc; +} + +static int zstSeek(void * cookie, _libio_pos_t pos, int whence) +{ + int rc; +#ifdef USE_COOKIE_SEEK_POINTER + _IO_off64_t p = *pos; +#else + off_t p = pos; +#endif + FD_t fd = c2f(cookie); + zstdFILE zsf; + + if (fd == NULL) return -2; + assert(fd->bytesRemain == -1); /* XXX FIXME */ + + zsf = (zstdFILE) zstFileno(fd); + if (zsf == NULL) return -2; /* XXX can't happen */ + + fdstat_enter(fd, FDSTAT_SEEK); + rc = zsfseek(zsf->gz, (long)p, whence); +DBGIO(fd, (stderr, "==>\tzstSeek(%p,%ld,%d) rc %lx %s\n", cookie, (long)p, whence, (unsigned long)rc, fdbg(fd))); + if (rc < 0) { + int zerror = 0; + fd->errcookie = zsferror(zsf->gz, &zerror); +#ifdef DYING + if (zerror == Z_ERRNO) { + fd->syserrno = errno; + fd->errcookie = strerror(fd->syserrno); + } +#endif + } + if (rc > 0) + fdstat_exit(fd, FDSTAT_SEEK, rc); + return rc; +} + +static int zstClose(void * cookie) +{ + FD_t fd = c2f(cookie); + zstdFILE zsf; + int rc; + + zsf = (zstdFILE) zstFileno(fd); + if (zsf == NULL) return -2; /* XXX can't happen */ + + fdstat_enter(fd, FDSTAT_CLOSE); + rc = zsfclose(zsf->gz); + zsf->gz = NULL; + zsf = _free(zsf); + + /* XXX TODO: preserve fd if errors */ + + if (fd) { +DBGIO(fd, (stderr, "==>\tzstClose(%p) zerror %d %s\n", cookie, rc, fdbg(fd))); + if (rc < 0) { + fd->errcookie = "zstClose error"; +#ifdef DYING + if (rc == Z_ERRNO) { + fd->syserrno = errno; + fd->errcookie = strerror(fd->syserrno); + } +#endif + } else if (rc >= 0) { + fdstat_exit(fd, FDSTAT_CLOSE, rc); + } + } + +DBGIO(fd, (stderr, "==>\tzstClose(%p) rc %lx %s\n", cookie, (unsigned long)rc, fdbg(fd))); + + if (_rpmio_debug || rpmIsDebug()) fdstat_print(fd, "ZSTIO", stderr); + if (rc == 0) + fd = fdFree(fd, "open (zstClose)"); + return rc; +} + +static struct FDIO_s zstio_s = { + zstRead, zstWrite, zstSeek, zstClose, zstOpen, zstFdopen, zstFlush, +}; + +FDIO_t zstio = &zstio_s; + +#endif /* WITH_ZLIB */ @@ . ______________________________________________________________________ RPM Package Manager http://rpm5.org CVS Sources Repository rpm-cvs@rpm5.org