Re: ksh history corruption

2011-09-01 Thread LEVAI Daniel 
On Wed, Aug 31, 2011 at 14:42:26 -0500, Marco Peereboom wrote:
 Version 4 fixes all reported bugs.
 
 Some folks have expressed doubt about the simplistic way of updating the
 history file.  Specifically the rewriting of all entries.  I am
 sensitive to that and know a couple of optimizations that can easily be
 applied.  However before I go there I'd like to get a thumbs up or down
 on this approach.  It trashes the binary history file format and
 replaces it with flat text.  Is this something we want?

Great, thanks, works with and without a HISTFILE.

Although, I've never used a history file, if this fixes something then
definitely yes, *but* please reconsider merging the various great
improvements too, that Alexander has created [1] in the past months.
(my favorites are the completion improvements and #8 :) ).

They are stable; I have been testing them since he had posted the diffs
on tech@.


Daniel

[1] http://plhk.ru/trash/ksh/

-- 
LIVAI Daniel
PGP key ID = 0x83B63A8F
Key fingerprint = DBEC C66B A47A DFA2 792D  650C C69B BE4C 83B6 3A8F

relayd session timeout

2011-09-01 Thread Alexander Bluhm 
Hi,

The relayd used the CHECK_TIMEOUT for connect and ssl handshake.
This is 200 milliseconds and too short.  Instead use the 600 seconds
session timeout that is used for accepted sessions everywhere else.

While there, make flag handling in relay_ssl_transaction() consistent
to the other functions.

ok?

bluhm


Index: usr.sbin/relayd/relay.c
===
RCS file: /data/mirror/openbsd/cvs/src/usr.sbin/relayd/relay.c,v
retrieving revision 1.138
diff -u -p -r1.138 relay.c
--- usr.sbin/relayd/relay.c 20 May 2011 09:43:53 -  1.138
+++ usr.sbin/relayd/relay.c 1 Sep 2011 07:58:40 -
@@ -2279,7 +2279,8 @@ relay_connect(struct rsession *con)
 
if (errno == EINPROGRESS)
event_again(con-se_ev, con-se_out.s, EV_WRITE|EV_TIMEOUT,
-   relay_connected, con-se_tv_start, env-sc_timeout, con);
+   relay_connected, con-se_tv_start, rlay-rl_conf.timeout, 
+   con);
else
relay_connected(con-se_out.s, EV_WRITE, con);
 
@@ -2625,7 +2626,7 @@ relay_ssl_transaction(struct rsession *c
SSL *ssl;
const SSL_METHOD*method;
void(*cb)(int, short, void *);
-   u_intflags = EV_TIMEOUT;
+   u_intflag;
 
ssl = SSL_new(rlay-rl_ssl_ctx);
if (ssl == NULL)
@@ -2634,11 +2635,11 @@ relay_ssl_transaction(struct rsession *c
if (cre-dir == RELAY_DIR_REQUEST) {
cb = relay_ssl_accept;
method = SSLv23_server_method();
-   flags |= EV_READ;
+   flag = EV_READ;
} else {
cb = relay_ssl_connect;
method = SSLv23_client_method();
-   flags |= EV_WRITE;
+   flag = EV_WRITE;
}
 
if (!SSL_set_ssl_method(ssl, method))
@@ -2653,8 +2654,10 @@ relay_ssl_transaction(struct rsession *c
 
cre-ssl = ssl;
 
-   event_again(con-se_ev, cre-s, EV_TIMEOUT|flags,
-   cb, con-se_tv_start, env-sc_timeout, con);
+   DPRINTF(%s: session %d: scheduling on %s, __func__, con-se_id,
+   (flag == EV_READ) ? EV_READ : EV_WRITE);
+   event_again(con-se_ev, cre-s, EV_TIMEOUT|flag, cb,
+   con-se_tv_start, rlay-rl_conf.timeout, con);
return;
 
  err:
@@ -2721,7 +2724,7 @@ retry:
DPRINTF(%s: session %d: scheduling on %s, __func__, con-se_id,
(retry_flag == EV_READ) ? EV_READ : EV_WRITE);
event_again(con-se_ev, fd, EV_TIMEOUT|retry_flag, relay_ssl_accept,
-   con-se_tv_start, env-sc_timeout, con);
+   con-se_tv_start, rlay-rl_conf.timeout, con);
 }
 
 void
@@ -2780,7 +2783,7 @@ retry:
DPRINTF(%s: session %d: scheduling on %s, __func__, con-se_id,
(retry_flag == EV_READ) ? EV_READ : EV_WRITE);
event_again(con-se_ev, fd, EV_TIMEOUT|retry_flag, relay_ssl_connect,
-   con-se_tv_start, env-sc_timeout, con);
+   con-se_tv_start, rlay-rl_conf.timeout, con);
 }
 
 void

Re: ksh history corruption

2011-09-01 Thread Marco Peereboom 
todd had his panties in a wad about backwards compatibility so I lifted
the ksh history load code out of ksh to dump it in a text file.  Compile
like:
cc kshconv.c -o kshconv

then run it like:
./kshconv -i ~/.hist -o texthist

Conversion code:
=== 8 ===
#include stdio.h
#include stdlib.h
#include err.h
#include sys/types.h
#include sys/mman.h

#define HMAGIC1 0xab
#define HMAGIC2 0xcd
#define COMMAND 0xff

extern char *__progname;

typedef enum state {
shdr,   /* expecting a header */
sline,  /* looking for a null byte to end the line */
sn1,/* bytes 1 to 4 of a line no */
sn2, sn3, sn4
} State;

void
histload(FILE *f, unsigned char *base, int bytes)
{
State   state;
int lno = 0;
unsigned char   *line = NULL;

for (state = shdr; bytes--  0; base++) {
switch (state) {
case shdr:
if (*base == COMMAND)
state = sn1;
break;
case sn1:
lno = (((*base)0xff)24);
state = sn2;
break;
case sn2:
lno |= (((*base)0xff)16);
state = sn3;
break;
case sn3:
lno |= (((*base)0xff)8);
state = sn4;
break;
case sn4:
lno |= (*base)0xff;
line = base+1;
state = sline;
break;
case sline:
if (*base == '\0') {
fprintf(f, %s\n, line);
state = shdr;
}
}
}
}
int
main(int argc, char *argv[])
{
int c;
char*in;
char*out;
FILE*fin;
FILE*fout;
off_t   hsize;
unsigned char   *base;

while ((c = getopt(argc, argv, i:o:)) != -1) {
switch (c) {
case 'i':
in = optarg;
break;
case 'o':
out = optarg;
break;
default:
errx(1, %s -i file -o file, __progname);
}
}

if (in == NULL || out == NULL)
errx(1, %s -i file -o file, __progname);

if ((fin = fopen(in, r)) == NULL)
err(1, %s, in);
if ((fout = fopen(out, w+)) == NULL)
err(1, %s, out);

hsize = lseek(fileno(fin), 0L, SEEK_END);
if (hsize == 0)
errx(1, nothing to do);
else if (hsize  0) {
/*
 * we have some data
 */
base = (unsigned char *)mmap(0, hsize, PROT_READ,
MAP_FILE|MAP_PRIVATE, fileno(fin), 0);
/*
 * check on its validity
 */
if (base == MAP_FAILED || *base != HMAGIC1 || base[1] != 
HMAGIC2) {
if (base != MAP_FAILED)
munmap((caddr_t)base, hsize);
goto done;
}

histload(fout, base+2, hsize-2);
munmap((caddr_t)base, hsize);
}
done:
fclose(fin);
fclose(fout);
}
=== 8 ===

V5 of diff with millert's suggestion to remove mmap.h.

Index: alloc.c
===
RCS file: /cvs/src/bin/ksh/alloc.c,v
retrieving revision 1.8
diff -u -p -r1.8 alloc.c
--- alloc.c 21 Jul 2008 17:30:08 -  1.8
+++ alloc.c 30 Aug 2011 18:05:47 -
@@ -62,7 +62,7 @@ alloc(size_t size, Area *ap)
 {
struct link *l;
 
-   l = malloc(sizeof(struct link) + size);
+   l = calloc(1, sizeof(struct link) + size);
if (l == NULL)
internal_errorf(1, unable to allocate memory);
l-next = ap-freelist;
Index: history.c
===
RCS file: /cvs/src/bin/ksh/history.c,v
retrieving revision 1.39
diff -u -p -r1.39 history.c
--- history.c   19 May 2010 17:36:08 -  1.39
+++ history.c   1 Sep 2011 13:49:13 -
@@ -11,8 +11,7 @@
  * a)  the original in-memory history  mechanism
  * b)  a more complicated mechanism done by  p...@hillside.co.uk
  * that more closely follows the real ksh way of doing
- * things. You need to have the mmap system call for this
- * to work on your system

relayd close before connect

2011-09-01 Thread Alexander Bluhm 
Hi,

Especially with SSL and short data transfers, it could happen that
the client closed before the connection to the server has been
established.  Then the relay closed immediately before transferring
any data.  The solution is to delay the close until the other side
has an event buffer.

ok?

bluhm


Index: usr.sbin/relayd/relay.c
===
RCS file: /data/mirror/openbsd/cvs/src/usr.sbin/relayd/relay.c,v
retrieving revision 1.138
diff -u -p -r1.138 relay.c
--- usr.sbin/relayd/relay.c 20 May 2011 09:43:53 -  1.138
+++ usr.sbin/relayd/relay.c 1 Sep 2011 12:14:05 -
@@ -1892,7 +1895,8 @@ relay_error(struct bufferevent *bev, sho
dst = EVBUFFER_OUTPUT(cre-dst-bev);
if (EVBUFFER_LENGTH(dst))
return;
-   }
+   } else
+   return;
 
relay_close(con, done);
return;

Re: ksh history corruption

2011-09-01 Thread Marco Peereboom 
Alright this diff keeps the file open and appends lines to HISTFILE.  It
only rewrites HISTFILE at 125% of HISTSIZE.  Does fancy locking and
deals with signals too.  So unless someone finds some bugs I'll consider
this version final.

Yes, no on moving ksh history to text?

Other comments?

Index: alloc.c
===
RCS file: /cvs/src/bin/ksh/alloc.c,v
retrieving revision 1.8
diff -u -p -r1.8 alloc.c
--- alloc.c 21 Jul 2008 17:30:08 -  1.8
+++ alloc.c 30 Aug 2011 18:05:47 -
@@ -62,7 +62,7 @@ alloc(size_t size, Area *ap)
 {
struct link *l;
 
-   l = malloc(sizeof(struct link) + size);
+   l = calloc(1, sizeof(struct link) + size);
if (l == NULL)
internal_errorf(1, unable to allocate memory);
l-next = ap-freelist;
Index: history.c
===
RCS file: /cvs/src/bin/ksh/history.c,v
retrieving revision 1.39
diff -u -p -r1.39 history.c
--- history.c   19 May 2010 17:36:08 -  1.39
+++ history.c   1 Sep 2011 20:14:44 -
@@ -11,8 +11,7 @@
  * a)  the original in-memory history  mechanism
  * b)  a more complicated mechanism done by  p...@hillside.co.uk
  * that more closely follows the real ksh way of doing
- * things. You need to have the mmap system call for this
- * to work on your system
+ * things.
  */
 
 #include sh.h
@@ -20,21 +19,11 @@
 
 #ifdef HISTORY
 # include sys/file.h
-# include sys/mman.h
 
-/*
- * variables for handling the data file
- */
-static int histfd;
-static int hsize;
-
-static int hist_count_lines(unsigned char *, int);
-static int hist_shrink(unsigned char *, int);
-static unsigned char *hist_skip_back(unsigned char *,int *,int);
-static void histload(Source *, unsigned char *, int);
-static void histinsert(Source *, int, unsigned char *);
-static void writehistfile(int, char *);
-static int sprinkle(int);
+static voidwritehistfile(void);
+static FILE*history_open(void);
+static int history_load(Source *);
+static voidhistory_close(void);
 
 static int hist_execute(char *);
 static int hist_replace(char **, const char *, const char *, int);
@@ -42,11 +31,14 @@ static char   **hist_get(const char *, i
 static char   **hist_get_oldest(void);
 static voidhistbackup(void);
 
+static FILE*histfd;
 static char   **current;   /* current position in history[] */
 static char*hname; /* current name of history file */
 static int hstarted;   /* set after hist_init() called */
-static Source  *hist_source;
+static Source  *hist_source;
+static uint32_tline_co;
 
+static struct stat last_sb;
 
 int
 c_fc(char **wp)
@@ -529,15 +521,10 @@ sethistfile(const char *name)
/* if the name is the same as the name we have */
if (hname  strcmp(hname, name) == 0)
return;
-
/*
 * its a new name - possibly
 */
-   if (histfd) {
-   /* yes the file is open */
-   (void) close(histfd);
-   histfd = 0;
-   hsize = 0;
+   if (hname) {
afree(hname, APERM);
hname = NULL;
/* let's reset the history */
@@ -545,6 +532,9 @@ sethistfile(const char *name)
hist_source-line = 0;
}
 
+   if (histfd)
+   history_close();
+
hist_init(hist_source);
 }
 
@@ -561,6 +551,27 @@ init_histvec(void)
}
 }
 
+static void
+history_lock(void)
+{
+   while (flock(fileno(histfd), LOCK_EX) != 0) {
+   if (errno == EINTR || errno == EAGAIN)
+   continue;
+   else
+   break;
+   }
+}
+
+static void
+history_unlock(void)
+{
+   while (flock(fileno(histfd), LOCK_UN) != 0) {
+   if (errno == EINTR || errno == EAGAIN)
+   continue;
+   else
+   break;
+   }
+}
 
 /*
  * Routines added by Peter Collinson BSDI(Europe)/Hillside Systems to
@@ -577,18 +588,29 @@ init_histvec(void)
 void
 histsave(int lno, const char *cmd, int dowrite)
 {
-   char **hp;
-   char *c, *cp;
+   char**hp;
+   char*c, *cp;
+   struct stat sb;
+
+   if (dowrite  histfd) {
+   history_lock();
+   if (fstat(fileno(histfd), sb) != -1) {
+   if (timespeccmp(sb.st_mtim, last_sb.st_mtim, ==))
+   ; /* file is unchanged */
+   else {
+   /* reset history */
+   histptr = history - 1;
+   hist_source-line = 0;
+   history_load(hist_source);
+   }
+   }
+   }
 
c = str_save(cmd, APERM);
if ((cp =

arp(8) verb omission

2011-09-01 Thread Brynet 
Index: arp.8
===
RCS file: /cvs/src/usr.sbin/arp/arp.8,v
retrieving revision 1.29
diff -u -p -u -r1.29 arp.8
--- src/usr.sbin/arp/arp.8  13 Mar 2011 21:24:20 -  1.29
+++ src/usr.sbin/arp/arp.8  1 Sep 2011 20:38:31 -
@@ -65,7 +65,7 @@ may be specified by name or by number,
 using Internet dot notation.
 .Pp
 .Nm
-can also used to send Wake on LAN (WoL) frames over a local
+can also be used to send Wake on LAN (WoL) frames over a local
 Ethernet network to one or more hosts using their link layer (hardware)
 addresses.
 WoL functionality is generally enabled in a machine's BIOS

Re: arp(8) verb omission

2011-09-01 Thread Jason McIntyre 
On Thu, Sep 01, 2011 at 04:46:54PM -0400, Brynet wrote:
 Index: arp.8
 ===
 RCS file: /cvs/src/usr.sbin/arp/arp.8,v
 retrieving revision 1.29
 diff -u -p -u -r1.29 arp.8
 --- src/usr.sbin/arp/arp.813 Mar 2011 21:24:20 -  1.29
 +++ src/usr.sbin/arp/arp.81 Sep 2011 20:38:31 -
 @@ -65,7 +65,7 @@ may be specified by name or by number,
  using Internet dot notation.
  .Pp
  .Nm
 -can also used to send Wake on LAN (WoL) frames over a local
 +can also be used to send Wake on LAN (WoL) frames over a local
  Ethernet network to one or more hosts using their link layer (hardware)
  addresses.
  WoL functionality is generally enabled in a machine's BIOS
 

fixed, thanks.
jmc

Re: ksh history corruption

2011-09-01 Thread Amit Kulkarni 
FWIW, I like this... makes it better for me to atleast grab some stuff
and copy over ***if*** it corrupts again.

thanks

On Thu, Sep 1, 2011 at 3:22 PM, Marco Peereboom sl...@peereboom.us wrote:
 Alright this diff keeps the file open and appends lines to HISTFILE.  It
 only rewrites HISTFILE at 125% of HISTSIZE.  Does fancy locking and
 deals with signals too.  So unless someone finds some bugs I'll consider
 this version final.

 Yes, no on moving ksh history to text?

 Other comments?

 Index: alloc.c
 ===
 RCS file: /cvs/src/bin/ksh/alloc.c,v
 retrieving revision 1.8
 diff -u -p -r1.8 alloc.c
 --- alloc.c 21 Jul 2008 17:30:08 -  1.8
 +++ alloc.c 30 Aug 2011 18:05:47 -
 @@ -62,7 +62,7 @@ alloc(size_t size, Area *ap)
  {
struct link *l;

 -   l = malloc(sizeof(struct link) + size);
 +   l = calloc(1, sizeof(struct link) + size);
if (l == NULL)
internal_errorf(1, unable to allocate memory);
l-next = ap-freelist;
 Index: history.c
 ===
 RCS file: /cvs/src/bin/ksh/history.c,v
 retrieving revision 1.39
 diff -u -p -r1.39 history.c
 --- history.c   19 May 2010 17:36:08 -  1.39
 +++ history.c   1 Sep 2011 20:14:44 -
 @@ -11,8 +11,7 @@
  * a)  the original in-memory history  mechanism
  * b)  a more complicated mechanism done by  p...@hillside.co.uk
  * that more closely follows the real ksh way of doing
 - * things. You need to have the mmap system call for this
 - * to work on your system
 + * things.
  */

  #include sh.h
 @@ -20,21 +19,11 @@

  #ifdef HISTORY
  # include sys/file.h
 -# include sys/mman.h

 -/*
 - * variables for handling the data file
 - */
 -static int histfd;
 -static int hsize;
 -
 -static int hist_count_lines(unsigned char *, int);
 -static int hist_shrink(unsigned char *, int);
 -static unsigned char *hist_skip_back(unsigned char *,int *,int);
 -static void histload(Source *, unsigned char *, int);
 -static void histinsert(Source *, int, unsigned char *);
 -static void writehistfile(int, char *);
 -static int sprinkle(int);
 +static voidwritehistfile(void);
 +static FILE*history_open(void);
 +static int history_load(Source *);
 +static voidhistory_close(void);

  static int hist_execute(char *);
  static int hist_replace(char **, const char *, const char *, int);
 @@ -42,11 +31,14 @@ static char   **hist_get(const char *, i
  static char   **hist_get_oldest(void);
  static voidhistbackup(void);

 +static FILE*histfd;
  static char   **current;   /* current position in history[] */
  static char*hname; /* current name of history file */
  static int hstarted;   /* set after hist_init() called */
 -static Source  *hist_source;
 +static Source  *hist_source;
 +static uint32_tline_co;

 +static struct stat last_sb;

  int
  c_fc(char **wp)
 @@ -529,15 +521,10 @@ sethistfile(const char *name)
/* if the name is the same as the name we have */
if (hname  strcmp(hname, name) == 0)
return;
 -
/*
 * its a new name - possibly
 */
 -   if (histfd) {
 -   /* yes the file is open */
 -   (void) close(histfd);
 -   histfd = 0;
 -   hsize = 0;
 +   if (hname) {
afree(hname, APERM);
hname = NULL;
/* let's reset the history */
 @@ -545,6 +532,9 @@ sethistfile(const char *name)
hist_source-line = 0;
}

 +   if (histfd)
 +   history_close();
 +
hist_init(hist_source);
  }

 @@ -561,6 +551,27 @@ init_histvec(void)
}
  }

 +static void
 +history_lock(void)
 +{
 +   while (flock(fileno(histfd), LOCK_EX) != 0) {
 +   if (errno == EINTR || errno == EAGAIN)
 +   continue;
 +   else
 +   break;
 +   }
 +}
 +
 +static void
 +history_unlock(void)
 +{
 +   while (flock(fileno(histfd), LOCK_UN) != 0) {
 +   if (errno == EINTR || errno == EAGAIN)
 +   continue;
 +   else
 +   break;
 +   }
 +}

  /*
  * Routines added by Peter Collinson BSDI(Europe)/Hillside Systems to
 @@ -577,18 +588,29 @@ init_histvec(void)
  void
  histsave(int lno, const char *cmd, int dowrite)
  {
 -   char **hp;
 -   char *c, *cp;
 +   char**hp;
 +   char*c, *cp;
 +   struct stat sb;
 +
 +   if (dowrite  histfd) {
 +   history_lock();
 +   if (fstat(fileno(histfd), sb) != -1) {
 +   if (timespeccmp(sb.st_mtim, last_sb.st_mtim, ==))
 +   ; /* file is unchanged */
 +   else {
 +

ksh wish

2011-09-01 Thread Damien Miller 
Hi,

While people are excited about hacking on ksh(1) - let me add my wish:
unrestricted multibyte character binding so I can have ctrl-left_arrow
(^[[1;5D on my terminal) bound to backward-word and so forth.

Last time I checked the code for bind could only handle a couple of
characters after ^[

-d