Trying ipfw_nat under VIMAGE kernel I got this panic on the module load:

Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address   = 0x4
fault code              = supervisor read, page not present
instruction pointer     = 0x20:0xc09f098e
stack pointer           = 0x28:0xf563b944
frame pointer           = 0x28:0xf563b998
code segment            = base 0x0, limit 0xfffff, type 0x1b
                        = DPL 0, pres 1, def32 1, gran 1
processor eflags        = interrupt enabled, resume, IOPL = 0
current process         = 4264 (kldload)

witness_checkorder(c6d5e91c,9,ca0ac2e3,223,0,...) at witness_checkorder+0x6e
_rw_wlock(c6d5e91c,ca0ac2e3,223,0,c0e8f795,...) at _rw_wlock+0x82
ipfw_nat_modevent(c98a48c0,0,0,75,0,...) at ipfw_nat_modevent+0x41
module_register_init(ca0ad508,0,c0e8d834,e6,0,...) at module_register_init+0xa7
linker_load_module(0,f563bc18,c0e8d834,3fc,f563bc28,...) at 
kern_kldload(c86835c0,c72d3400,f563bc40,0,c8d0d000,...) at kern_kldload+0x133
kldload(c86835c0,f563bcec,c09e8940,c86835c0,0,...) at kldload+0x74
syscallenter(c86835c0,f563bce4,c0ce05dd,c1022150,0,...) at syscallenter+0x263
syscall(f563bd28) at syscall+0x34
Xint0x80_syscall() at Xint0x80_syscall+0x21
--- syscall (304, FreeBSD ELF32, kldload), eip = 0x280da00b, esp = 0xbfbfe79c, 
ebp = 0xbfbfec88 -

It crashed on acessing data from virtualized global variable V_layer3_chain in
ipfw_nat_modevent(). V_layer3_chain is defined in ipfw module and it turns out
that &V_layer3_chain returns wrong location from anywhere but ipfw.ko.

May be this is a known issue, but I have not found info about this, so below
are details of investigation why this happens.

Virtualized global variables are defined using the VNET_DEFINE() macro, which
places them in the 'set_vnet' linker set (in the base kernel or in
module). This is used to

1) copy these "default" values to each virtual network stack instance when 

2) act as unique global names by which the variable can be referred to. The
location of a per-virtual instance variable is calculated at run-time like in
the example below for layer3_chain variable in the default vnet (vnet0):

vnet0->vnet_data_base + (uintptr_t) & vnet_entry_layer3_chain           (1)

For modules the thing is more complicated. When a module is loaded its global
variables from 'set_vnet' linker set are copied to the kernel 'set_vnet', and
for module to be able to access them the linker reallocates all references
accordingly (kern/link_elf.c:elf_relocaddr()):

        if (x >= ef->vnet_start && x < ef->vnet_stop)
                return ((x - ef->vnet_start) + ef->vnet_base);

So from inside the module the access to its virtualized variables works, but
from the outside we get wrong location using calculation like above (1),
because &vnet_entry_layer3_chain returns address of the variable in the
module's 'set_vnet'.

The workaround is to compile such modules into the kernel or use a hack I have
done for ipfw_nat -- add the function to ipfw module which returns the
location of virtualized layer3_chain variable and use this location instead of
V_layer3_chain macro (see the attached patch).

But I suppose the problem is not a new and there might be better approach
already invented to deal with this?

Mikolaj Golub
Index: sys/netinet/ipfw/ip_fw_private.h
--- sys/netinet/ipfw/ip_fw_private.h	(revision 221673)
+++ sys/netinet/ipfw/ip_fw_private.h	(working copy)
@@ -201,6 +201,8 @@ VNET_DECLARE(int, fw_verbose);
 VNET_DECLARE(struct ip_fw_chain, layer3_chain);
 #define	V_layer3_chain		VNET(layer3_chain)
+void* vnet_entry_addr_layer3_chain(void);
+#define V_addr_layer3_chain	(vnet_entry_addr_layer3_chain())
 VNET_DECLARE(u_int32_t, set_disable);
 #define	V_set_disable		VNET(set_disable)
Index: sys/netinet/ipfw/ip_fw_nat.c
--- sys/netinet/ipfw/ip_fw_nat.c	(revision 221673)
+++ sys/netinet/ipfw/ip_fw_nat.c	(working copy)
@@ -62,7 +62,7 @@ ifaddr_change(void *arg __unused, struct ifnet *if
 	struct ifaddr *ifa;
 	struct ip_fw_chain *chain;
-	chain = &V_layer3_chain;
+	chain = V_addr_layer3_chain;
 	/* Check every nat entry... */
 	LIST_FOREACH(ptr, &chain->nat, _next) {
@@ -345,7 +345,7 @@ ipfw_nat_cfg(struct sockopt *sopt)
 	struct cfg_nat *cfg, *ptr;
 	char *buf;
-	struct ip_fw_chain *chain = &V_layer3_chain;
+	struct ip_fw_chain *chain = V_addr_layer3_chain;
 	size_t len;
 	int gencnt, error = 0;
@@ -421,7 +421,7 @@ static int
 ipfw_nat_del(struct sockopt *sopt)
 	struct cfg_nat *ptr;
-	struct ip_fw_chain *chain = &V_layer3_chain;
+	struct ip_fw_chain *chain = V_addr_layer3_chain;
 	int i;
 	sooptcopyin(sopt, &i, sizeof i, sizeof i);
@@ -444,7 +444,7 @@ ipfw_nat_del(struct sockopt *sopt)
 static int
 ipfw_nat_get_cfg(struct sockopt *sopt)
-	struct ip_fw_chain *chain = &V_layer3_chain;
+	struct ip_fw_chain *chain = V_addr_layer3_chain;
 	struct cfg_nat *n;
 	struct cfg_redir *r;
 	struct cfg_spool *s;
@@ -509,7 +509,7 @@ ipfw_nat_get_log(struct sockopt *sopt)
 	int i, size;
 	struct ip_fw_chain *chain;
-	chain = &V_layer3_chain;
+	chain = V_addr_layer3_chain;
 	/* one pass to count, one to copy the data */
@@ -543,8 +543,11 @@ ipfw_nat_get_log(struct sockopt *sopt)
 static void
+	struct ip_fw_chain *chain;
-	IPFW_WLOCK(&V_layer3_chain);
+	chain = V_addr_layer3_chain;
+	IPFW_WLOCK(chain);
 	/* init ipfw hooks */
 	ipfw_nat_ptr = ipfw_nat;
 	lookup_nat_ptr = lookup_nat;
@@ -552,7 +555,7 @@ ipfw_nat_init(void)
 	ipfw_nat_del_ptr = ipfw_nat_del;
 	ipfw_nat_get_cfg_ptr = ipfw_nat_get_cfg;
 	ipfw_nat_get_log_ptr = ipfw_nat_get_log;
-	IPFW_WUNLOCK(&V_layer3_chain);
+	IPFW_WUNLOCK(chain);
 	V_ifaddr_event_tag = EVENTHANDLER_REGISTER(
 	    ifaddr_event, ifaddr_change,
@@ -564,7 +567,7 @@ ipfw_nat_destroy(void)
 	struct cfg_nat *ptr, *ptr_temp;
 	struct ip_fw_chain *chain;
-	chain = &V_layer3_chain;
+	chain = V_addr_layer3_chain;
 	LIST_FOREACH_SAFE(ptr, &chain->nat, _next, ptr_temp) {
 		LIST_REMOVE(ptr, _next);
Index: sys/netinet/ipfw/ip_fw2.c
--- sys/netinet/ipfw/ip_fw2.c	(revision 221673)
+++ sys/netinet/ipfw/ip_fw2.c	(working copy)
@@ -134,6 +134,11 @@ VNET_DEFINE(int, verbose_limit);
 /* layer3_chain contains the list of rules for layer 3 */
 VNET_DEFINE(struct ip_fw_chain, layer3_chain);
+	return &V_layer3_chain;
 ipfw_nat_t *ipfw_nat_ptr = NULL;
 struct cfg_nat *(*lookup_nat_ptr)(struct nat_list *, int);
freebsd-virtualization@freebsd.org mailing list
To unsubscribe, send any mail to 

Reply via email to