FWIW, I like this... makes it better for me to atleast grab some stuff
and copy over ***if*** it corrupts again.
thanks
On Thu, Sep 1, 2011 at 3:22 PM, Marco Peereboom <sl...@peereboom.us> wrote:
> Alright this diff keeps the file open and appends lines to HISTFILE. It
> only rewrites HISTFILE at 125% of HISTSIZE. Does fancy locking and
> deals with signals too. So unless someone finds some bugs I'll consider
> this version final.
>
> Yes, no on moving ksh history to text?
>
> Other comments?
>
> Index: alloc.c
> ===================================================================
> RCS file: /cvs/src/bin/ksh/alloc.c,v
> retrieving revision 1.8
> diff -u -p -r1.8 alloc.c
> --- alloc.c 21 Jul 2008 17:30:08 -0000 1.8
> +++ alloc.c 30 Aug 2011 18:05:47 -0000
> @@ -62,7 +62,7 @@ alloc(size_t size, Area *ap)
> {
> struct link *l;
>
> - l = malloc(sizeof(struct link) + size);
> + l = calloc(1, sizeof(struct link) + size);
> if (l == NULL)
> internal_errorf(1, "unable to allocate memory");
> l->next = ap->freelist;
> Index: history.c
> ===================================================================
> RCS file: /cvs/src/bin/ksh/history.c,v
> retrieving revision 1.39
> diff -u -p -r1.39 history.c
> --- history.c 19 May 2010 17:36:08 -0000 1.39
> +++ history.c 1 Sep 2011 20:14:44 -0000
> @@ -11,8 +11,7 @@
> * a) the original in-memory history mechanism
> * b) a more complicated mechanism done by p...@hillside.co.uk
> * that more closely follows the real ksh way of doing
> - * things. You need to have the mmap system call for this
> - * to work on your system
> + * things.
> */
>
> #include "sh.h"
> @@ -20,21 +19,11 @@
>
> #ifdef HISTORY
> # include <sys/file.h>
> -# include <sys/mman.h>
>
> -/*
> - * variables for handling the data file
> - */
> -static int histfd;
> -static int hsize;
> -
> -static int hist_count_lines(unsigned char *, int);
> -static int hist_shrink(unsigned char *, int);
> -static unsigned char *hist_skip_back(unsigned char *,int *,int);
> -static void histload(Source *, unsigned char *, int);
> -static void histinsert(Source *, int, unsigned char *);
> -static void writehistfile(int, char *);
> -static int sprinkle(int);
> +static void writehistfile(void);
> +static FILE *history_open(void);
> +static int history_load(Source *);
> +static void history_close(void);
>
> static int hist_execute(char *);
> static int hist_replace(char **, const char *, const char *, int);
> @@ -42,11 +31,14 @@ static char **hist_get(const char *, i
> static char **hist_get_oldest(void);
> static void histbackup(void);
>
> +static FILE *histfd;
> static char **current; /* current position in history[] */
> static char *hname; /* current name of history file */
> static int hstarted; /* set after hist_init() called */
> -static Source *hist_source;
> +static Source *hist_source;
> +static uint32_t line_co;
>
> +static struct stat last_sb;
>
> int
> c_fc(char **wp)
> @@ -529,15 +521,10 @@ sethistfile(const char *name)
> /* if the name is the same as the name we have */
> if (hname && strcmp(hname, name) == 0)
> return;
> -
> /*
> * its a new name - possibly
> */
> - if (histfd) {
> - /* yes the file is open */
> - (void) close(histfd);
> - histfd = 0;
> - hsize = 0;
> + if (hname) {
> afree(hname, APERM);
> hname = NULL;
> /* let's reset the history */
> @@ -545,6 +532,9 @@ sethistfile(const char *name)
> hist_source->line = 0;
> }
>
> + if (histfd)
> + history_close();
> +
> hist_init(hist_source);
> }
>
> @@ -561,6 +551,27 @@ init_histvec(void)
> }
> }
>
> +static void
> +history_lock(void)
> +{
> + while (flock(fileno(histfd), LOCK_EX) != 0) {
> + if (errno == EINTR || errno == EAGAIN)
> + continue;
> + else
> + break;
> + }
> +}
> +
> +static void
> +history_unlock(void)
> +{
> + while (flock(fileno(histfd), LOCK_UN) != 0) {
> + if (errno == EINTR || errno == EAGAIN)
> + continue;
> + else
> + break;
> + }
> +}
>
> /*
> * Routines added by Peter Collinson BSDI(Europe)/Hillside Systems to
> @@ -577,18 +588,29 @@ init_histvec(void)
> void
> histsave(int lno, const char *cmd, int dowrite)
> {
> - char **hp;
> - char *c, *cp;
> + char **hp;
> + char *c, *cp;
> + struct stat sb;
> +
> + if (dowrite && histfd) {
> + history_lock();
> + if (fstat(fileno(histfd), &sb) != -1) {
> + if (timespeccmp(&sb.st_mtim, &last_sb.st_mtim, ==))
> + ; /* file is unchanged */
> + else {
> + /* reset history */
> + histptr = history - 1;
> + hist_source->line = 0;
> + history_load(hist_source);
> + }
> + }
> + }
>
> c = str_save(cmd, APERM);
> if ((cp = strchr(c, '\n')) != NULL)
> *cp = '\0';
>
> - if (histfd && dowrite)
> - writehistfile(lno, c);
> -
> hp = histptr;
> -
> if (++hp >= history + histsize) { /* remove oldest command */
> afree((void*)*history, APERM);
> for (hp = history; hp < history + histsize - 1; hp++)
> @@ -596,371 +618,143 @@ histsave(int lno, const char *cmd, int d
> }
> *hp = c;
> histptr = hp;
> -}
> -
> -/*
> - * Write history data to a file nominated by HISTFILE
> - * if HISTFILE is unset then history still happens, but
> - * the data is not written to a file
> - * All copies of ksh looking at the file will maintain the
> - * same history. This is ksh behaviour.
> - *
> - * This stuff uses mmap()
> - * if your system ain't got it - then you'll have to undef HISTORYFILE
> - */
>
> -/*
> - * Open a history file
> - * Format is:
> - * Bytes 1, 2: HMAGIC - just to check that we are dealing with
> - * the correct object
> - * Then follows a number of stored commands
> - * Each command is
> - * <command byte><command number(4 bytes)><bytes><null>
> - */
> -#define HMAGIC1 0xab
> -#define HMAGIC2 0xcd
> -#define COMMAND 0xff
> + if (dowrite && histfd) {
> + /* append to file */
> + if (fseeko(histfd, 0, SEEK_END) == 0) {
> + fprintf(histfd, "%s", cmd);
> + fflush(histfd);
> + fstat(fileno(histfd), &last_sb);
> + line_co++;
> + writehistfile();
> + }
> + history_unlock();
> + }
> +}
>
> -void
> -hist_init(Source *s)
> +static FILE *
> +history_open(void)
> {
> - unsigned char *base;
> - int lines;
> - int fd;
> -
> - if (Flag(FTALKING) == 0)
> - return;
> -
> - hstarted = 1;
> -
> - hist_source = s;
> -
> - hname = str_val(global("HISTFILE"));
> - if (hname == NULL)
> - return;
> - hname = str_save(hname, APERM);
> + int fd, fddup;
> + FILE *f = NULL;
>
> - retry:
> - /* we have a file and are interactive */
> - if ((fd = open(hname, O_RDWR|O_CREAT|O_APPEND, 0600)) < 0)
> - return;
> -
> - histfd = savefd(fd);
> - if (histfd != fd)
> + if ((fd = open(hname, O_RDWR | O_CREAT | O_EXLOCK, 0600)) == -1)
> + return (NULL);
> + fddup = savefd(fd);
> + if (fddup != fd)
> close(fd);
>
> - (void) flock(histfd, LOCK_EX);
> + f = fdopen(fddup, "r+");
> + if (f == NULL) {
> + close(fddup);
> + goto bad;
> + }
>
> - hsize = lseek(histfd, 0L, SEEK_END);
> + fstat(fileno(f), &last_sb);
>
> - if (hsize == 0) {
> - /* add magic */
> - if (sprinkle(histfd)) {
> - hist_finish();
> - return;
> - }
> - }
> - else if (hsize > 0) {
> - /*
> - * we have some data
> - */
> - base = (unsigned char *)mmap(0, hsize, PROT_READ,
> - MAP_FILE|MAP_PRIVATE, histfd, 0);
> - /*
> - * check on its validity
> - */
> - if (base == MAP_FAILED || *base != HMAGIC1 || base[1] !=
HMAGIC2) {
> - if (base != MAP_FAILED)
> - munmap((caddr_t)base, hsize);
> - hist_finish();
> - if (unlink(hname) != 0)
> - return;
> - goto retry;
> - }
> - if (hsize > 2) {
> - lines = hist_count_lines(base+2, hsize-2);
> - if (lines > histsize) {
> - /* we need to make the file smaller */
> - if (hist_shrink(base, hsize))
> - if (unlink(hname) != 0)
> - return;
> - munmap((caddr_t)base, hsize);
> - hist_finish();
> - goto retry;
> - }
> - }
> - histload(hist_source, base+2, hsize-2);
> - munmap((caddr_t)base, hsize);
> - }
> - (void) flock(histfd, LOCK_UN);
> - hsize = lseek(histfd, 0L, SEEK_END);
> -}
> + return (f);
> +bad:
> + if (f)
> + fclose(f);
>
> -typedef enum state {
> - shdr, /* expecting a header */
> - sline, /* looking for a null byte to end the line */
> - sn1, /* bytes 1 to 4 of a line no */
> - sn2, sn3, sn4
> -} State;
> + return (NULL);
> +}
>
> -static int
> -hist_count_lines(unsigned char *base, int bytes)
> +static void
> +history_close(void)
> {
> - State state = shdr;
> - int lines = 0;
> -
> - while (bytes--) {
> - switch (state) {
> - case shdr:
> - if (*base == COMMAND)
> - state = sn1;
> - break;
> - case sn1:
> - state = sn2; break;
> - case sn2:
> - state = sn3; break;
> - case sn3:
> - state = sn4; break;
> - case sn4:
> - state = sline; break;
> - case sline:
> - if (*base == '\0')
> - lines++, state = shdr;
> - }
> - base++;
> + if (histfd) {
> + fflush(histfd);
> + fclose(histfd);
> + histfd = NULL;
> }
> - return lines;
> }
>
> -/*
> - * Shrink the history file to histsize lines
> - */
> static int
> -hist_shrink(unsigned char *oldbase, int oldbytes)
> +history_load(Source *s)
> {
> - int fd;
> - char nfile[1024];
> - struct stat statb;
> - unsigned char *nbase = oldbase;
> - int nbytes = oldbytes;
> + char *p, line[LINE + 1];
>
> - nbase = hist_skip_back(nbase, &nbytes, histsize);
> - if (nbase == NULL)
> - return 1;
> - if (nbase == oldbase)
> - return 0;
> + rewind(histfd);
>
> - /*
> - * create temp file
> - */
> - (void) shf_snprintf(nfile, sizeof(nfile), "%s.%d", hname, procpid);
> - if ((fd = open(nfile, O_CREAT | O_TRUNC | O_WRONLY, 0600)) < 0)
> - return 1;
> + /* just read it all; will auto resize history upon next command */
> + for (line_co = 1; ; line_co++) {
> + p = fgets(line, sizeof line, histfd);
> + if (p == NULL || feof(histfd) || ferror(histfd))
> + break;
> + if ((p = strchr(line, '\n')) == NULL) {
> + bi_errorf("history file is corrupt");
> + return (1);
> + }
> + *p = '\0';
>
> - if (sprinkle(fd)) {
> - close(fd);
> - unlink(nfile);
> - return 1;
> + s->line = line_co;
> + s->cmd_offset = line_co;
> + histsave(line_co, (char *)line, 0);
> }
> - if (write(fd, nbase, nbytes) != nbytes) {
> - close(fd);
> - unlink(nfile);
> - return 1;
> - }
> - /*
> - * worry about who owns this file
> - */
> - if (fstat(histfd, &statb) >= 0)
> - fchown(fd, statb.st_uid, statb.st_gid);
> - close(fd);
>
> - /*
> - * rename
> - */
> - if (rename(nfile, hname) < 0)
> - return 1;
> - return 0;
> -}
> + writehistfile();
>
> + return (0);
> +}
>
> -/*
> - * find a pointer to the data `no' back from the end of the file
> - * return the pointer and the number of bytes left
> - */
> -static unsigned char *
> -hist_skip_back(unsigned char *base, int *bytes, int no)
> +void
> +hist_init(Source *s)
> {
> - int lines = 0;
> - unsigned char *ep;
> + if (Flag(FTALKING) == 0)
> + return;
>
> - for (ep = base + *bytes; --ep > base; ) {
> - /* this doesn't really work: the 4 byte line number that is
> - * encoded after the COMMAND byte can itself contain the
> - * COMMAND byte....
> - */
> - for (; ep > base && *ep != COMMAND; ep--)
> - ;
> - if (ep == base)
> - break;
> - if (++lines == no) {
> - *bytes = *bytes - ((char *)ep - (char *)base);
> - return ep;
> - }
> - }
> - return NULL;
> -}
> + hstarted = 1;
>
> -/*
> - * load the history structure from the stored data
> - */
> -static void
> -histload(Source *s, unsigned char *base, int bytes)
> -{
> - State state;
> - int lno = 0;
> - unsigned char *line = NULL;
> -
> - for (state = shdr; bytes-- > 0; base++) {
> - switch (state) {
> - case shdr:
> - if (*base == COMMAND)
> - state = sn1;
> - break;
> - case sn1:
> - lno = (((*base)&0xff)<<24);
> - state = sn2;
> - break;
> - case sn2:
> - lno |= (((*base)&0xff)<<16);
> - state = sn3;
> - break;
> - case sn3:
> - lno |= (((*base)&0xff)<<8);
> - state = sn4;
> - break;
> - case sn4:
> - lno |= (*base)&0xff;
> - line = base+1;
> - state = sline;
> - break;
> - case sline:
> - if (*base == '\0') {
> - /* worry about line numbers */
> - if (histptr >= history && lno-1 != s->line)
{
> - /* a replacement ? */
> - histinsert(s, lno, line);
> - }
> - else {
> - s->line = lno;
> - s->cmd_offset = lno;
> - histsave(lno, (char *)line, 0);
> - }
> - state = shdr;
> - }
> - }
> - }
> -}
> + hist_source = s;
>
> -/*
> - * Insert a line into the history at a specified number
> - */
> -static void
> -histinsert(Source *s, int lno, unsigned char *line)
> -{
> - char **hp;
> + hname = str_val(global("HISTFILE"));
> + if (hname == NULL)
> + return;
> + hname = str_save(hname, APERM);
> + histfd = history_open();
> + if (histfd == NULL)
> + return;
>
> - if (lno >= s->line-(histptr-history) && lno <= s->line) {
> - hp = &histptr[lno-s->line];
> - if (*hp)
> - afree((void*)*hp, APERM);
> - *hp = str_save((char *)line, APERM);
> - }
> + history_load(s);
> +
> + history_unlock();
> }
>
> -/*
> - * write a command to the end of the history file
> - * This *MAY* seem easy but it's also necessary to check
> - * that the history file has not changed in size.
> - * If it has - then some other shell has written to it
> - * and we should read those commands to update our history
> - */
> static void
> -writehistfile(int lno, char *cmd)
> +writehistfile(void)
> {
> - int sizenow;
> - unsigned char *base;
> - unsigned char *new;
> - int bytes;
> - unsigned char hdr[5];
> -
> - (void) flock(histfd, LOCK_EX);
> - sizenow = lseek(histfd, 0L, SEEK_END);
> - if (sizenow != hsize) {
> - /*
> - * Things have changed
> - */
> - if (sizenow > hsize) {
> - /* someone has added some lines */
> - bytes = sizenow - hsize;
> - base = (unsigned char *)mmap(0, sizenow,
> - PROT_READ, MAP_FILE|MAP_PRIVATE, histfd, 0);
> - if (base == MAP_FAILED)
> - goto bad;
> - new = base + hsize;
> - if (*new != COMMAND) {
> - munmap((caddr_t)base, sizenow);
> - goto bad;
> - }
> - hist_source->line--;
> - histload(hist_source, new, bytes);
> - hist_source->line++;
> - lno = hist_source->line;
> - munmap((caddr_t)base, sizenow);
> - hsize = sizenow;
> - } else {
> - /* it has shrunk */
> - /* but to what? */
> - /* we'll give up for now */
> - goto bad;
> - }
> + int i;
> + char *cmd;
> +
> + /* see if file has grown over 25% */
> + if (line_co < histsize + (histsize / 4))
> + return;
> +
> + if (ftruncate(fileno(histfd), 0) == -1)
> + return;
> +
> + /* rewrite the whole caboodle */
> + rewind(histfd);
> + for (i = 0; i < histsize; i++) {
> + cmd = history[i];
> + if (cmd == NULL)
> + break;
> + if (fprintf(histfd, "%s\n", cmd) == -1)
> + return;
> }
> - /*
> - * we can write our bit now
> - */
> - hdr[0] = COMMAND;
> - hdr[1] = (lno>>24)&0xff;
> - hdr[2] = (lno>>16)&0xff;
> - hdr[3] = (lno>>8)&0xff;
> - hdr[4] = lno&0xff;
> - (void) write(histfd, hdr, 5);
> - (void) write(histfd, cmd, strlen(cmd)+1);
> - hsize = lseek(histfd, 0L, SEEK_END);
> - (void) flock(histfd, LOCK_UN);
> - return;
> -bad:
> - hist_finish();
> +
> + line_co = histsize;
> +
> + fflush(histfd);
> + fstat(fileno(histfd), &last_sb);
> }
>
> void
> hist_finish(void)
> {
> - (void) flock(histfd, LOCK_UN);
> - (void) close(histfd);
> - histfd = 0;
> + history_close();
> }
> -
> -/*
> - * add magic to the history file
> - */
> -static int
> -sprinkle(int fd)
> -{
> - static unsigned char mag[] = { HMAGIC1, HMAGIC2 };
> -
> - return(write(fd, mag, 2) != 2);
> -}
> -
> #else /* HISTORY */
>
> /* No history to be compiled in: dummy routines to avoid lots more ifdefs
*/
