Hello list
I am using jail(8) trying to get a functional vimage environment on my
9.1-RELEASE system. My PC only has a single real NIC facing the public
internet.
My goal is to be able to have multiple vimage jails, each with
their own epairXa epairXb and bridgeX where the "X" is the jails JID
number all having their traffic passing through the single rl0 real
interface. The vnet.start script shown below handles this nicely.
The problem is after the first vimage jail is started the rl0 interface
gets marked as busy when the second vimage jail is started.
How do I get all vnet jails to pass through the real rl0 interface?
Thanks for you help
# /root >cat /etc/jail.conf
vimage33 {
host.hostname = "vimage33";
path = "/usr/jails/vimage33";
mount.fstab = "/usr/local/etc/fstab/vimage33";
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.consolelog = "/var/log/vimage33.console.log";
devfs_ruleset = "4";
allow.mount.devfs;
vnet;
exec.poststart="vnet.start vimage33 rl0";
exec.prestop="vnet.stop vimage33";
}
# /root >cat /usr/local/bin/vnet.start
#!/bin/sh
jailname=$1
nicname=$2
jid=`jls -j ${jailname} jid`
if [ "${jid}" -gt "100" ]; then
echo " "
echo "The JID value is greater then 100."
echo "You must shutdown the host and reboot"
echo "to zero out the JID counter and recover"
echo "the lost memory from stopping vimage jails."
echo " "
exit 2
fi
ifconfig bridge${jid} create > /dev/null 2> /dev/null
ifconfig bridge${jid} 10.${jid}.0.1
ifconfig bridge${jid} up
ifconfig epair${jid} create > /dev/null 2> /dev/null
ifconfig bridge${jid} addm ${nicname} addm epair${jid}a
ifconfig epair${jid}a up
ifconfig epair${jid}b vnet ${jid}
jexec ${jailname} ifconfig epair${jid}b 10.${jid}.0.2
jexec ${jailname} route add default 10.${jid}.0.1 > /dev/null 2> /dev/null
jexec ${jailname} ifconfig lo0 127.0.0.1
# Display the hosts network view before starting any vnet jails
# /root >ifconfig
rl0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
options=2008<VLAN_MTU,WOL_MAGIC>
ether 00:0c:6e:09:8b:74
inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
media: Ethernet autoselect (100baseTX <full-duplex>)
status: active
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
inet 127.0.0.1 netmask 0xff000000
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
# Start the first vnet jail
# /root >jail -f /etc/jail.conf -c vimage33
vimage33: created
bridge1: Ethernet address: 02:8f:94:84:0c:02
epair1a: Ethernet address: 02:c0:a4:00:0b:0a
epair1b: Ethernet address: 02:c0:a4:00:0c:0b
# /root >jls
JID IP Address Hostname Path
1 - vimage33 /usr/jails/vimage33
# Lets display the hosts network after the first vnet jail has started
# /root >ifconfig
rl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric
options=2008<VLAN_MTU,WOL_MAGIC>
ether 00:0c:6e:09:8b:74
inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
media: Ethernet autoselect (100baseTX <full-duplex>)
status: active
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
inet 127.0.0.1 netmask 0xff000000
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric
ether 02:8f:94:84:0c:01
inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 9 priority 128 path cost 14183
member: rl0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 5 priority 128 path cost 200000
epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
options=8<VLAN_MTU>
ether 02:c0:a4:00:09:0a
inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
# Login to the vnet jail and display the jails view of the network
# /root >jexec vimage33 tcsh
vimage33 / >ifconfig
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
inet 127.0.0.1 netmask 0xff000000
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair1b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
options=8<VLAN_MTU>
ether 02:c0:a4:00:0a:0b
inet 10.1.0.2 netmask 0xff000000 broadcast 10.255.255.255
inet6 fe80::c0:a4ff:fe00:a0b%epair1b prefixlen 64 scopeid 0x2
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
# Yes the vnet jail can reach the public network
vimage33 / >ping -c 4 8.8.178.135
PING 8.8.178.135 (8.8.178.135): 56 data bytes
64 bytes from 8.8.178.135: icmp_seq=0 ttl=51 time=84.645 ms
64 bytes from 8.8.178.135: icmp_seq=1 ttl=51 time=86.950 ms
64 bytes from 8.8.178.135: icmp_seq=2 ttl=51 time=83.274 ms
64 bytes from 8.8.178.135: icmp_seq=3 ttl=51 time=82.660 ms
--- 8.8.178.135 ping statistics ---
4 packets transmitted, 4 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 82.660/84.382/86.950/1.647 ms
vimage33 / >exit
exit
# Lets start the second vnet jail
# /root >cat /etc/jail.conf.22
vimage22 {
host.hostname = "vimage22";
path = "/usr/jails/vimage22";
mount.fstab = "/usr/local/etc/fstab/vimage22";
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.consolelog = "/var/log/vimage22.console.log";
devfs_ruleset = "4";
allow.mount.devfs;
vnet;
exec.poststart="vnet.start vimage22 rl0";
exec.prestop="vnet.stop vimage22";
}
# /root >jail -f /etc/jail.conf.22 -c vimage22
vimage22: created
# Notice this message about rl0
ifconfig: BRDGADD rl0: Device busy
bridge2: Ethernet address: 02:8f:94:84:0c:02
epair2a: Ethernet address: 02:c0:a4:00:0b:0a
epair2b: Ethernet address: 02:c0:a4:00:0c:0b
# Lets check the hosts view of the network - no rl0 on bridge2
# /root >ifconfig
rl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
options=2008<VLAN_MTU,WOL_MAGIC>
ether 00:0c:6e:09:8b:74
inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
media: Ethernet autoselect (100baseTX <full-duplex>)
status: active
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
inet 127.0.0.1 netmask 0xff000000
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
bridge1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
ether 02:8f:94:84:0c:01
inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 9 priority 128 path cost 14183
member: rl0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 5 priority 128 path cost 200000
epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
options=8<VLAN_MTU>
ether 02:c0:a4:00:09:0a
inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
bridge2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
ether 02:8f:94:84:0c:02
inet 10.2.0.1 netmask 0xff000000 broadcast 10.255.255.255
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
epair2a: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
options=8<VLAN_MTU>
ether 02:c0:a4:00:0b:0a
inet6 fe80::c0:a4ff:fe00:b0a%epair2a prefixlen 64 scopeid 0xb
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
# /root >jls
JID IP Address Hostname Path
1 - vimage33 /usr/jails/vimage33
2 - vimage22 /usr/jails/vimage22
# login to second vnet jail and see if it has public internet connection
# /root >jexec vimage22 tcsh
vimage22 / >ifconfig
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
inet 127.0.0.1 netmask 0xff000000
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair2b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
options=8<VLAN_MTU>
ether 02:c0:a4:00:0c:0b
inet 10.2.0.2 netmask 0xff000000 broadcast 10.255.255.255
inet6 fe80::c0:a4ff:fe00:c0b%epair2b prefixlen 64 scopeid 0x2
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
vimage22 / >ping -c 4 8.8.178.135
PING 8.8.178.135 (8.8.178.135): 56 data bytes
--- 8.8.178.135 ping statistics ---
4 packets transmitted, 0 packets received, 100.0% packet loss
vimage22 / >exit
exit
# Stop the second vnet jail
# /root >jail -f /etc/jail.conf.22 -r vimage22
vimage22: removed
Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
Freed UMA keg was not empty (10 items). Lost 2 pages of memory.
Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
hhook_vnet_uninit: hhook_head type=1, id=1 cleanup required
hhook_vnet_uninit: hhook_head type=1, id=0 cleanup required
# Stop the first vnet jail
# /root >jail -f /etc/jail.conf -r vimage33
vimage33: removed
Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
Freed UMA keg was not empty (10 items). Lost 2 pages of memory.
Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
hhook_vnet_uninit: hhook_head type=1, id=1 cleanup required
hhook_vnet_uninit: hhook_head type=1, id=0 cleanup required
