Re: httpd crashes when fetching a hidden file located on a CD

[David Gwynne]

> On 11 Dec 2015, at 9:23 PM, Ted Unangst  wrote:
> 
> Ted Unangst wrote:
>> Jonathan Gray wrote:
 
 There's one thing to add though, it looks like it happens for any file on
 cd9660, not just dotfiles.
>>> 
>>> It is worth pointing out that httpd has had trouble serving files off
>>> specific filesystems in the past due to kqueue issues.
>>> 
>>> cd9660_vops does not currently set .vop_kqfilter, does anything change
>>> if you set EVENT_NOKQUEUE before running httpd?
>> 
>> this maybe adds kqueue to cd9660.
> 
> We've confirmed this diff fixes the problem. However, there seems to be a
> larger problem that httpd/libevent cannot gracefully handle the condition
> where kevent returns an error. We are doomed to see this problem repeat if
> that is not addressed.

on one hand i agree with you, but on the other i wonder why httpd thinks 
setting events up on files is useful.

Re: httpd crashes when fetching a hidden file located on a CD

[Reyk Floeter]

On Sat, Dec 12, 2015 at 08:09:44AM -0700, Theo de Raadt wrote:
> > on one hand i agree with you, but on the other i wonder why httpd thinks
> > setting events up on files is useful.=
> 
> I wondered this too.  And since this is libevent, and poll/select cannot
> do anything like that, what is the goal?
> 

httpd is designed to use libevent's event buffers - not really for
poll/select - to fill the buffers as long as data is available and to
write it out as soon as possible; with throttling and wartermarks
involved.  It pretty much works like relayd, and intentionally uses
the same relaying between fds (in this case between file/fastcgi and
tcp/tls connections).  Honestly, I didn't think of somebody serving
files from iso9660 filesystems, but there are always surprises ;)

Reyk

Re: httpd crashes when fetching a hidden file located on a CD

[Ted Unangst]

Ted Unangst wrote:
> Jonathan Gray wrote:
> > > 
> > > There's one thing to add though, it looks like it happens for any file on
> > > cd9660, not just dotfiles.
> > 
> > It is worth pointing out that httpd has had trouble serving files off
> > specific filesystems in the past due to kqueue issues.
> > 
> > cd9660_vops does not currently set .vop_kqfilter, does anything change
> > if you set EVENT_NOKQUEUE before running httpd?
> 
> this maybe adds kqueue to cd9660.

We've confirmed this diff fixes the problem. However, there seems to be a
larger problem that httpd/libevent cannot gracefully handle the condition
where kevent returns an error. We are doomed to see this problem repeat if
that is not addressed.

Re: httpd crashes when fetching a hidden file located on a CD

[Ted Unangst]

Jonathan Gray wrote:
> > 
> > There's one thing to add though, it looks like it happens for any file on
> > cd9660, not just dotfiles.
> 
> It is worth pointing out that httpd has had trouble serving files off
> specific filesystems in the past due to kqueue issues.
> 
> cd9660_vops does not currently set .vop_kqfilter, does anything change
> if you set EVENT_NOKQUEUE before running httpd?

this maybe adds kqueue to cd9660.


Index: cd9660_vnops.c
===
RCS file: /cvs/src/sys/isofs/cd9660/cd9660_vnops.c,v
retrieving revision 1.72
diff -u -p -r1.72 cd9660_vnops.c
--- cd9660_vnops.c  17 Apr 2015 04:43:20 -  1.72
+++ cd9660_vnops.c  8 Dec 2015 15:24:38 -
@@ -65,6 +65,9 @@
 #include 
 #include 
 
+int cd9660_kqfilter(void *v);
+
+
 /*
  * Structure for reading directories
  */
@@ -841,6 +844,7 @@ struct vops cd9660_vops = {
.vop_write  = cd9660_write,
.vop_ioctl  = cd9660_ioctl,
.vop_poll   = cd9660_poll,
+   .vop_kqfilter   = cd9660_kqfilter,
.vop_revoke = cd9660_revoke,
.vop_fsync  = cd9660_fsync,
.vop_remove = cd9660_remove,
@@ -947,3 +951,103 @@ struct vops cd9660_fifovops = {
.vop_advlock= fifo_advlock,
 };
 #endif /* FIFO */
+
+void filt_cd9660detach(struct knote *kn);
+int filt_cd9660read(struct knote *kn, long hint);
+int filt_cd9660write(struct knote *kn, long hint);
+int filt_cd9660vnode(struct knote *kn, long hint);
+
+struct filterops cd9660read_filtops = 
+   { 1, NULL, filt_cd9660detach, filt_cd9660read };
+struct filterops cd9660write_filtops = 
+   { 1, NULL, filt_cd9660detach, filt_cd9660write };
+struct filterops cd9660vnode_filtops = 
+   { 1, NULL, filt_cd9660detach, filt_cd9660vnode };
+
+int
+cd9660_kqfilter(void *v)
+{
+   struct vop_kqfilter_args *ap = v;
+   struct vnode *vp = ap->a_vp;
+   struct knote *kn = ap->a_kn;
+
+   switch (kn->kn_filter) {
+   case EVFILT_READ:
+   kn->kn_fop = _filtops;
+   break;
+   case EVFILT_WRITE:
+   kn->kn_fop = _filtops;
+   break;
+   case EVFILT_VNODE:
+   kn->kn_fop = _filtops;
+   break;
+   default:
+   return (EINVAL);
+   }
+
+   kn->kn_hook = (caddr_t)vp;
+
+   SLIST_INSERT_HEAD(>v_selectinfo.si_note, kn, kn_selnext);
+
+   return (0);
+}
+
+void
+filt_cd9660detach(struct knote *kn)
+{
+   struct vnode *vp = (struct vnode *)kn->kn_hook;
+
+   SLIST_REMOVE(>v_selectinfo.si_note, kn, knote, kn_selnext);
+}
+
+int
+filt_cd9660read(struct knote *kn, long hint)
+{
+   struct vnode *vp = (struct vnode *)kn->kn_hook;
+   struct iso_node *node = VTOI(vp);
+
+   /*
+* filesystem is gone, so set the EOF flag and schedule 
+* the knote for deletion.
+*/
+   if (hint == NOTE_REVOKE) {
+   kn->kn_flags |= (EV_EOF | EV_ONESHOT);
+   return (1);
+   }
+
+   kn->kn_data = node->i_size - kn->kn_fp->f_offset;
+   if (kn->kn_data == 0 && kn->kn_sfflags & NOTE_EOF) {
+   kn->kn_fflags |= NOTE_EOF;
+   return (1);
+   }
+
+   return (kn->kn_data != 0);
+}
+
+int
+filt_cd9660write(struct knote *kn, long hint)
+{
+   /*
+* filesystem is gone, so set the EOF flag and schedule 
+* the knote for deletion.
+*/
+   if (hint == NOTE_REVOKE) {
+   kn->kn_flags |= (EV_EOF | EV_ONESHOT);
+   return (1);
+   }
+
+   kn->kn_data = 0;
+   return (1);
+}
+
+int
+filt_cd9660vnode(struct knote *kn, long hint)
+{
+   if (kn->kn_sfflags & hint)
+   kn->kn_fflags |= hint;
+   if (hint == NOTE_REVOKE) {
+   kn->kn_flags |= EV_EOF;
+   return (1);
+   }
+   return (kn->kn_fflags != 0);
+}

Re: httpd crashes when fetching a hidden file located on a CD

[Theo de Raadt]

> Stuart Henderson wrote:
> >  26600 httpdCALL  issetugid()
> >  26600 httpdRET   issetugid 0
> >  26600 httpdCALL  open(0x7f7d82a0,0)
> >  26600 httpdNAMI  "/usr/share/zoneinfo/GMT"
> >  26600 httpdRET   open -1 errno 2 No such file or directory
> >  26600 httpdCALL  issetugid()
> >  26600 httpdRET   issetugid 0
> >  26600 httpdCALL  open(0x7f7d81f0,0)
> >  26600 httpdNAMI  "/usr/share/zoneinfo/posixrules"
> >  26600 httpdRET   open -1 errno 2 No such file or directory
> 
> unrelated to the problem at hand, but httpd is trying to access timezone files
> in the chroot.

feels like a missing tzset before jailing...

Re: httpd crashes when fetching a hidden file located on a CD

[Sevan / Venture37]

Thanks so much for the patch, It's resolved the issue.


Sevan

Re: httpd crashes when fetching a hidden file located on a CD

[Theo de Raadt]

This bug report totally sucks.

Have you ever heard of ktrace, and if you have, why did you not try
to reproduce it?

You want us to reproduce it?  Why?

> Hi,
> I ran across an issue with httpd(8) on 5.8-RELEASE & -CURRENT (2/12/2015
> snapshot) where fetching a .hidden file located on a CD through httpd
> results in httpd crashing (no core file or error message logged).
> 
> To reproduce, mount CD in a location which is served by httpd. eg CentOS
> minimal install iso[1] has a hidden file in the root called .treeinfo
> 
> Try to fetch http://myweb/.treeinfo
> 
> Of course this is not a common scenario found in production, I happened
> to run into it whist taking a shortcut to save time & disk space by
> mounting the CentOS iso on a virtualbox guest which was running OpenBSD.
> 
> 
> Sevan
> [1]
> http://isoredirect.centos.org/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1503-01.iso
> 

httpd crashes when fetching a hidden file located on a CD

[Sevan / Venture37]

Hi,
I ran across an issue with httpd(8) on 5.8-RELEASE & -CURRENT (2/12/2015
snapshot) where fetching a .hidden file located on a CD through httpd
results in httpd crashing (no core file or error message logged).

To reproduce, mount CD in a location which is served by httpd. eg CentOS
minimal install iso[1] has a hidden file in the root called .treeinfo

Try to fetch http://myweb/.treeinfo

Of course this is not a common scenario found in production, I happened
to run into it whist taking a shortcut to save time & disk space by
mounting the CentOS iso on a virtualbox guest which was running OpenBSD.


Sevan
[1]
http://isoredirect.centos.org/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1503-01.iso

Re: httpd crashes when fetching a hidden file located on a CD

[Jonathan Gray]

On Tue, Dec 08, 2015 at 12:31:09AM +, Stuart Henderson wrote:
> On 2015/12/07 15:44, Theo de Raadt wrote:
> > This bug report totally sucks.
> > 
> > Have you ever heard of ktrace, and if you have, why did you not try
> > to reproduce it?
> > 
> > You want us to reproduce it?  Why?
> > 
> > > Hi,
> > > I ran across an issue with httpd(8) on 5.8-RELEASE & -CURRENT (2/12/2015
> > > snapshot) where fetching a .hidden file located on a CD through httpd
> > > results in httpd crashing (no core file or error message logged).
> > > 
> > > To reproduce, mount CD in a location which is served by httpd. eg CentOS
> > > minimal install iso[1] has a hidden file in the root called .treeinfo
> > > 
> > > Try to fetch http://myweb/.treeinfo
> > > 
> > > Of course this is not a common scenario found in production, I happened
> > > to run into it whist taking a shortcut to save time & disk space by
> > > mounting the CentOS iso on a virtualbox guest which was running OpenBSD.
> > > 
> > > 
> > > Sevan
> > > [1]
> > > http://isoredirect.centos.org/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1503-01.iso
> > > 
> > 
> 
> It's not that bad a report, it has everything necessary to reproduce,
> tested on release and -current so it's obviously not pledge related.
> 
> There's one thing to add though, it looks like it happens for any file on
> cd9660, not just dotfiles.

It is worth pointing out that httpd has had trouble serving files off
specific filesystems in the past due to kqueue issues.

cd9660_vops does not currently set .vop_kqfilter, does anything change
if you set EVENT_NOKQUEUE before running httpd?

Re: httpd crashes when fetching a hidden file located on a CD

[Stuart Henderson]

On 2015/12/07 15:44, Theo de Raadt wrote:
> This bug report totally sucks.
> 
> Have you ever heard of ktrace, and if you have, why did you not try
> to reproduce it?
> 
> You want us to reproduce it?  Why?
> 
> > Hi,
> > I ran across an issue with httpd(8) on 5.8-RELEASE & -CURRENT (2/12/2015
> > snapshot) where fetching a .hidden file located on a CD through httpd
> > results in httpd crashing (no core file or error message logged).
> > 
> > To reproduce, mount CD in a location which is served by httpd. eg CentOS
> > minimal install iso[1] has a hidden file in the root called .treeinfo
> > 
> > Try to fetch http://myweb/.treeinfo
> > 
> > Of course this is not a common scenario found in production, I happened
> > to run into it whist taking a shortcut to save time & disk space by
> > mounting the CentOS iso on a virtualbox guest which was running OpenBSD.
> > 
> > 
> > Sevan
> > [1]
> > http://isoredirect.centos.org/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1503-01.iso
> > 
> 

It's not that bad a report, it has everything necessary to reproduce,
tested on release and -current so it's obviously not pledge related.

There's one thing to add though, it looks like it happens for any file on
cd9660, not just dotfiles.

Here's ktrace, not that it seems particularly useful.

$ sudo kdump
 26600  EMUL  "native"
 26600 httpdRET   kevent 1
 26600 httpdCALL  clock_gettime(CLOCK_MONOTONIC,0x7f7d9460)
 26600 httpdSTRU  struct timespec { 190190<"Jan  3 05:49:50 
1970">.634312834 }
 26600 httpdRET   clock_gettime 0
 26600 httpdCALL  kbind(0x7f7d9208,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  getdtablecount()
 26600 httpdRET   getdtablecount 7
 26600 httpdCALL  kbind(0x7f7d9208,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  getrlimit(RLIMIT_NOFILE,0x7f7d9260)
 26600 httpdSTRU  struct rlimit { cur=6000, max=6000 }
 26600 httpdRET   getrlimit 0
 26600 httpdCALL  kbind(0x7f7d9208,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  
accept4(3,0x7f7d9330,0x7f7d9444,0x4000)
 26600 httpdSTRU  struct sockaddr { AF_INET, 127.0.0.1:16821 }
 26600 httpdRET   accept4 5
 26600 httpdCALL  kbind(0x7f7d9258,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  getpid()
 26600 httpdRET   getpid 26600/0x67e8
 26600 httpdCALL  kbind(0x7f7d9258,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  getsockname(5,0xda798556020,0x7f7d9444)
 26600 httpdSTRU  struct sockaddr { AF_INET, 127.0.0.1:8223 }
 26600 httpdRET   getsockname 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  clock_gettime(CLOCK_MONOTONIC,0x7f7d92f0)
 26600 httpdSTRU  struct timespec { 190190<"Jan  3 05:49:50 
1970">.634549181 }
 26600 httpdRET   clock_gettime 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  
getsockopt(5,SOL_SOCKET,SO_SNDBUF,0xda7985562d8,0x7f7d92fc)
 26600 httpdRET   getsockopt 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d91d8,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d91d8,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d91d8,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9208,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d91e8,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9228,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d91c8,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9158,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kevent(9,0xda79855d000,3,0xda7a840b000,64,0x7f7d9430)
 26600 httpdSTRU  struct timespec { 600 }
 26600 httpdRET   kevent 2
 26600 httpdCALL  clock_gettime(CLOCK_MONOTONIC,0x7f7d9460)
 26600 httpdSTRU  struct timespec { 190190<"Jan  3 05:49:50 
1970">.634743483 }
 26600 httpdRET   clock_gettime 0
 26600 httpdCALL  kbind(0x7f7d9398,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  kbind(0x7f7d9358,0x18,0xd6de5e5b8cf9d5cf)
 26600 httpdRET   kbind 0
 26600 httpdCALL  ioctl(5,FIONREAD,0x7f7d942c)
 26600 httpdRET