Hi,

Trying ipfw_nat under VIMAGE kernel I got this panic on the module load:

Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address   = 0x4
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc09f098e
stack pointer           = 0x28:0xf563b944
frame pointer           = 0x28:0xf563b998
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 4264 (kldload)

witness_checkorder(c6d5e91c,9,ca0ac2e3,223,0,...) at witness_checkorder+0x6e
_rw_wlock(c6d5e91c,ca0ac2e3,223,0,c0e8f795,...) at _rw_wlock+0x82
ipfw_nat_modevent(c98a48c0,0,0,75,0,...) at ipfw_nat_modevent+0x41
module_register_init(ca0ad508,0,c0e8d834,e6,0,...) at module_register_init+0xa7
linker_load_module(0,f563bc18,c0e8d834,3fc,f563bc28,...) at 
linker_load_module+0xa05
kern_kldload(c86835c0,c72d3400,f563bc40,0,c8d0d000,...) at kern_kldload+0x133
kldload(c86835c0,f563bcec,c09e8940,c86835c0,0,...) at kldload+0x74
syscallenter(c86835c0,f563bce4,c0ce05dd,c1022150,0,...) at syscallenter+0x263
syscall(f563bd28) at syscall+0x34
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (304, FreeBSD ELF32, kldload), eip = 0x280da00b, esp = 0xbfbfe79c, 
ebp = 0xbfbfec88 -

It crashed on acessing data from virtualized global variable V_layer3_chain in
ipfw_nat_modevent(). V_layer3_chain is defined in ipfw module and it turns out
that &V_layer3_chain returns wrong location from anywhere but ipfw.ko.

May be this is a known issue, but I have not found info about this, so below
are details of investigation why this happens.

Virtualized global variables are defined using the VNET_DEFINE() macro, which
places them in the 'set_vnet' linker set (in the base kernel or in
module). This is used to

1) copy these "default" values to each virtual network stack instance when 
created;

2) act as unique global names by which the variable can be referred to. The
location of a per-virtual instance variable is calculated at run-time like in
the example below for layer3_chain variable in the default vnet (vnet0):

vnet0->vnet_data_base + (uintptr_t) & vnet_entry_layer3_chain           (1)

For modules the thing is more complicated. When a module is loaded its global
variables from 'set_vnet' linker set are copied to the kernel 'set_vnet', and
for module to be able to access them the linker reallocates all references
accordingly (kern/link_elf.c:elf_relocaddr()):

        if (x >= ef->vnet_start && x < ef->vnet_stop)
                return ((x - ef->vnet_start) + ef->vnet_base);

So from inside the module the access to its virtualized variables works, but
from the outside we get wrong location using calculation like above (1),
because &vnet_entry_layer3_chain returns address of the variable in the
module's 'set_vnet'.

The workaround is to compile such modules into the kernel or use a hack I have
done for ipfw_nat -- add the function to ipfw module which returns the
location of virtualized layer3_chain variable and use this location instead of
V_layer3_chain macro (see the attached patch).

But I suppose the problem is not a new and there might be better approach
already invented to deal with this?

-- 
Mikolaj Golub
Index: sys/netinet/ipfw/ip_fw_private.h
===================================================================
--- sys/netinet/ipfw/ip_fw_private.h	(revision 221673)
+++ sys/netinet/ipfw/ip_fw_private.h	(working copy)
@@ -201,6 +201,8 @@ VNET_DECLARE(int, fw_verbose);
 
 VNET_DECLARE(struct ip_fw_chain, layer3_chain);
 #define	V_layer3_chain		VNET(layer3_chain)
+void* vnet_entry_addr_layer3_chain(void);
+#define V_addr_layer3_chain	(vnet_entry_addr_layer3_chain())
 
 VNET_DECLARE(u_int32_t, set_disable);
 #define	V_set_disable		VNET(set_disable)
Index: sys/netinet/ipfw/ip_fw_nat.c
===================================================================
--- sys/netinet/ipfw/ip_fw_nat.c	(revision 221673)
+++ sys/netinet/ipfw/ip_fw_nat.c	(working copy)
@@ -62,7 +62,7 @@ ifaddr_change(void *arg __unused, struct ifnet *if
 	struct ifaddr *ifa;
 	struct ip_fw_chain *chain;
 
-	chain = &V_layer3_chain;
+	chain = V_addr_layer3_chain;
 	IPFW_WLOCK(chain);
 	/* Check every nat entry... */
 	LIST_FOREACH(ptr, &chain->nat, _next) {
@@ -345,7 +345,7 @@ ipfw_nat_cfg(struct sockopt *sopt)
 {
 	struct cfg_nat *cfg, *ptr;
 	char *buf;
-	struct ip_fw_chain *chain = &V_layer3_chain;
+	struct ip_fw_chain *chain = V_addr_layer3_chain;
 	size_t len;
 	int gencnt, error = 0;
 
@@ -421,7 +421,7 @@ static int
 ipfw_nat_del(struct sockopt *sopt)
 {
 	struct cfg_nat *ptr;
-	struct ip_fw_chain *chain = &V_layer3_chain;
+	struct ip_fw_chain *chain = V_addr_layer3_chain;
 	int i;
 
 	sooptcopyin(sopt, &i, sizeof i, sizeof i);
@@ -444,7 +444,7 @@ ipfw_nat_del(struct sockopt *sopt)
 static int
 ipfw_nat_get_cfg(struct sockopt *sopt)
 {
-	struct ip_fw_chain *chain = &V_layer3_chain;
+	struct ip_fw_chain *chain = V_addr_layer3_chain;
 	struct cfg_nat *n;
 	struct cfg_redir *r;
 	struct cfg_spool *s;
@@ -509,7 +509,7 @@ ipfw_nat_get_log(struct sockopt *sopt)
 	int i, size;
 	struct ip_fw_chain *chain;
 
-	chain = &V_layer3_chain;
+	chain = V_addr_layer3_chain;
 
 	IPFW_RLOCK(chain);
 	/* one pass to count, one to copy the data */
@@ -543,8 +543,11 @@ ipfw_nat_get_log(struct sockopt *sopt)
 static void
 ipfw_nat_init(void)
 {
+	struct ip_fw_chain *chain;
 
-	IPFW_WLOCK(&V_layer3_chain);
+	chain = V_addr_layer3_chain;
+
+	IPFW_WLOCK(chain);
 	/* init ipfw hooks */
 	ipfw_nat_ptr = ipfw_nat;
 	lookup_nat_ptr = lookup_nat;
@@ -552,7 +555,7 @@ ipfw_nat_init(void)
 	ipfw_nat_del_ptr = ipfw_nat_del;
 	ipfw_nat_get_cfg_ptr = ipfw_nat_get_cfg;
 	ipfw_nat_get_log_ptr = ipfw_nat_get_log;
-	IPFW_WUNLOCK(&V_layer3_chain);
+	IPFW_WUNLOCK(chain);
 	V_ifaddr_event_tag = EVENTHANDLER_REGISTER(
 	    ifaddr_event, ifaddr_change,
 	    NULL, EVENTHANDLER_PRI_ANY);
@@ -564,7 +567,7 @@ ipfw_nat_destroy(void)
 	struct cfg_nat *ptr, *ptr_temp;
 	struct ip_fw_chain *chain;
 
-	chain = &V_layer3_chain;
+	chain = V_addr_layer3_chain;
 	IPFW_WLOCK(chain);
 	LIST_FOREACH_SAFE(ptr, &chain->nat, _next, ptr_temp) {
 		LIST_REMOVE(ptr, _next);
Index: sys/netinet/ipfw/ip_fw2.c
===================================================================
--- sys/netinet/ipfw/ip_fw2.c	(revision 221673)
+++ sys/netinet/ipfw/ip_fw2.c	(working copy)
@@ -134,6 +134,11 @@ VNET_DEFINE(int, verbose_limit);
 
 /* layer3_chain contains the list of rules for layer 3 */
 VNET_DEFINE(struct ip_fw_chain, layer3_chain);
+void*
+vnet_entry_addr_layer3_chain(void)
+{
+	return &V_layer3_chain;
+}
 
 ipfw_nat_t *ipfw_nat_ptr = NULL;
 struct cfg_nat *(*lookup_nat_ptr)(struct nat_list *, int);
_______________________________________________
freebsd-virtualization@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-virtualization
To unsubscribe, send any mail to 
"freebsd-virtualization-unsubscr...@freebsd.org"

Reply via email to