nice !!!!!! 2016-03-18 11:58 GMT+01:00 martin kolly <martin.ko...@senselan.ch>:
> Hi All > > We are facing the same issue as reported by Milamber (Ticket 9255) > https://issues.apache.org/jira/browse/CLOUDSTACK-9255. When deploying a > couple of VMs or Port Forwarding's the re-deployment of the router with > cleanup fails. > > We found that iptables configuration takes a lot of time, this eventually > leads to a timeout on the management server "Unable to start VM > DomainRouter due to error in finalizeStart, not retrying" > > Environment: > - Cloudstack 4.8 > - KVM (local storage) > - hosts/mgr on Ubuntu 14.04 > > We tested with a simple set of four forwarding rules, here the setup: > > root@r-96-VM:~# cat /etc/cloudstack/forwardingrules.json > { > "185.20.146.56": [ > { > "internal_ip": "10.100.1.95", > "internal_ports": "22:22", > "protocol": "tcp", > "public_ip": "185.20.146.56", > "public_ports": "22:22", > "type": "forward" > } > ], > "185.20.146.79": [ > { > "internal_ip": "10.100.1.42", > "internal_ports": "22:22", > "protocol": "tcp", > "public_ip": "185.20.146.79", > "public_ports": "22:22", > "type": "forward" > }, > { > "internal_ip": "10.100.1.42", > "internal_ports": "8443:8443", > "protocol": "tcp", > "public_ip": "185.20.146.79", > "public_ports": "8443:8443", > "type": "forward" > }, > { > "internal_ip": "10.100.1.42", > "internal_ports": "53:53", > "protocol": "udp", > "public_ip": "185.20.146.79", > "public_ports": "53:53", > "type": "forward" > } > ], > "id": "forwardingrules" > > The definition for every port forwarding seems to take at ~1.5 seconds. > > python /opt/cloud/bin/configure.py.timed > /etc/cloudstack/forwardingrules.json > > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.42:22 > time : 0.000965118408203 > -A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.42:22 > time : 0.395485162735 > -A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 22 -j DNAT > --to-destination 10.100.1.42:22 > time : 0.395533084869 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 22 > time : 1.16180706024 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j > MARK --set-xmark 0x2/0xffffffff > time : 1.16329216957 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 1.16407108307 > -A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 1.53959512711 > ---------------------------------------------- > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j > DNAT --to-destination 10.100.1.42:8443 > time : 0.000781059265137 > -A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 8443 -j > DNAT --to-destination 10.100.1.42:8443 > time : 0.378201007843 > -A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 8443 -j DNAT > --to-destination 10.100.1.42:8443 > time : 0.37822508812 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 8443 > time : 1.14627504349 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j > MARK --set-xmark 0x2/0xffffffff > time : 1.1477329731 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 1.14850592613 > -A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 8443 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 1.52321791649 > ---------------------------------------------- > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j > DNAT --to-destination 10.100.1.42:53 > time : 0.000754117965698 > -A PREROUTING -d 185.20.146.79/32 -i eth0 -p udp -m udp --dport 53 -j > DNAT --to-destination 10.100.1.42:53 > time : 0.383729934692 > -A OUTPUT -d 185.20.146.79/32 -p udp -m udp --dport 53 -j DNAT > --to-destination 10.100.1.42:53 > time : 0.383754968643 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.42/32 -o eth0 -p udp -m udp --dport 53 > time : 1.14376091957 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j > MARK --set-xmark 0x2/0xffffffff > time : 1.14526605606 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 1.14599299431 > -A FORWARD -i eth2 -o eth0 -p udp -m udp --dport 53 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 1.52742600441 > ---------------------------------------------- > -A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.95:22 > time : 0.000700950622559 > -A PREROUTING -d 185.20.146.56/32 -i eth0 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.95:22 > time : 0.382349014282 > -A OUTPUT -d 185.20.146.56/32 -p tcp -m tcp --dport 22 -j DNAT > --to-destination 10.100.1.95:22 > time : 0.382384061813 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.95/32 -o eth0 -p tcp -m tcp --dport 22 > time : 1.1425909996 > -A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j > MARK --set-xmark 0x2/0xffffffff > time : 1.14400196075 > -A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 1.14468812943 > -A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 1.52619600296 > ---------------------------------------------- > > Having a closer look at configure.py how the iptables rules are defined. > We think that it is not efficient to lookup these values for every policy: > > def forward_vr(self, rule): > > fw1 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT > --to-destination %s:%s" % \ > ( > rule['public_ip'], > *self.getDeviceByIp(rule['public_ip']),* > rule['protocol'], > rule['protocol'], > *self.portsToString(rule['public_ports'], ':'),* > rule['internal_ip'], > *self.portsToString(rule['internal_ports'], '-')* > ) > fw2 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT > --to-destination %s:%s" % \ > ( > rule['public_ip'], > * self.getDeviceByIp(rule['internal_ip']),* > rule['protocol'], > rule['protocol'], > * self.portsToString(rule['public_ports'], ':'),* > rule['internal_ip'], > > > * self.portsToString(rule['internal_ports'], '-') ..... * > > Defining these values once at the beginning would be much more efficient, > no ? > > def forward_vr(self, rule): > > * pub_interface = self.getDeviceByIp(rule['public_ip'])* > * int_interface = self.getDeviceByIp(rule['internal_ip'])* > * pub_ports = self.portsToString(rule['public_ports'], ':')* > * int_ports = self.portsToString(rule['internal_ports'], '-')* > > > * int_network = self.getNetworkByIp(rule['internal_ip']) * fw1 = > "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT > --to-destination %s:%s" % \ > ( > rule['public_ip'], > pub_interface, > rule['protocol'], > rule['protocol'], > pub_ports, > rule['internal_ip'], > int_ports > ) > > fw2 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -j DNAT > --to-destination %s:%s" % \ > ( > rule['public_ip'], > int_interface, > rule['protocol'], > rule['protocol'], > pub_ports, > rule['internal_ip'], > int_ports > ) > ..... > > If we run the configure.py with these modifications we have the following: > > root@r-96-VM:~# python /opt/cloud/bin/configure_modified.py > /etc/cloudstack/forwardingrules.json > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.42:22 > time : 0.000349044799805 > -A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.42:22 > time : 0.000686883926392 > -A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 22 -j DNAT > --to-destination 10.100.1.42:22 > time : 0.000943899154663 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 22 > time : 0.00131487846375 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -j > MARK --set-xmark 0x2/0xffffffff > time : 0.00161194801331 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 22 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 0.00186896324158 > -A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 0.00216102600098 > ---------------------------------------------- > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j > DNAT --to-destination 10.100.1.42:8443 > time : 0.000232934951782 > -A PREROUTING -d 185.20.146.79/32 -i eth0 -p tcp -m tcp --dport 8443 -j > DNAT --to-destination 10.100.1.42:8443 > time : 0.000478029251099 > -A OUTPUT -d 185.20.146.79/32 -p tcp -m tcp --dport 8443 -j DNAT > --to-destination 10.100.1.42:8443 > time : 0.00071907043457 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.42/32 -o eth0 -p tcp -m tcp --dport 8443 > time : 0.000991106033325 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -j > MARK --set-xmark 0x2/0xffffffff > time : 0.00136613845825 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p tcp -m tcp --dport 8443 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 0.00174498558044 > -A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 8443 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 0.00219202041626 > ---------------------------------------------- > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j > DNAT --to-destination 10.100.1.42:53 > time : 0.000226974487305 > -A PREROUTING -d 185.20.146.79/32 -i eth0 -p udp -m udp --dport 53 -j > DNAT --to-destination 10.100.1.42:53 > time : 0.000502824783325 > -A OUTPUT -d 185.20.146.79/32 -p udp -m udp --dport 53 -j DNAT > --to-destination 10.100.1.42:53 > time : 0.000762939453125 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.42/32 -o eth0 -p udp -m udp --dport 53 > time : 0.00103092193604 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -j > MARK --set-xmark 0x2/0xffffffff > time : 0.00134587287903 > -A PREROUTING -d 185.20.146.79/32 -i eth2 -p udp -m udp --dport 53 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 0.00158596038818 > -A FORWARD -i eth2 -o eth0 -p udp -m udp --dport 53 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 0.00182485580444 > ---------------------------------------------- > -A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.95:22 > time : 0.000264167785645 > -A PREROUTING -d 185.20.146.56/32 -i eth0 -p tcp -m tcp --dport 22 -j > DNAT --to-destination 10.100.1.95:22 > time : 0.000508069992065 > -A OUTPUT -d 185.20.146.56/32 -p tcp -m tcp --dport 22 -j DNAT > --to-destination 10.100.1.95:22 > time : 0.000750064849854 > -j SNAT --to-source 10.100.1.1 -A POSTROUTING -s 10.100.1.0/24 -d > 10.100.1.95/32 -o eth0 -p tcp -m tcp --dport 22 > time : 0.00102114677429 > -A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -j > MARK --set-xmark 0x2/0xffffffff > time : 0.00138115882874 > -A PREROUTING -d 185.20.146.56/32 -i eth2 -p tcp -m tcp --dport 22 -m > state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask > 0xffffffff > time : 0.00165915489197 > -A FORWARD -i eth2 -o eth0 -p tcp -m tcp --dport 22 -m state --state > NEW,ESTABLISHED -j ACCEPT > Total time for creating Policy : 0.00196814537048 > ---------------------------------------------- > > Location of configure.py: > > https://github.com/apache/cloudstack/blob/master/systemvm/patches/debian/config/opt/cloud/bin/configure.py > > The modified scripts are attached. Thanks for your feedback. > > regards > Martin > >
