Public bug reported: If 2 Tempest tests are run concurrently as follows: Test 1: Any test that uses DHCP and cleans up the DHCP port as part of cleanup. For example: tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_router Test 2: Any test that creates an IPv6 SLAAC subnet tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_64_subnets and this patch has been applied to eliminate DB access deadlocks: https://review.openstack.org/#/c/170690/ Then occasionally Test 2 will fail in subnet create, and the following error can be observed in the neutron server log (q-svc.log):
TRACE neutron.api.v2.resource DBReferenceError: (IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`neutron`.`ipallocations`, CONSTRAINT `ipallocations_ibfk_2` FOREIGN KEY (`port_id`) REFERENCES `ports` (`id`) ON DELETE CASCADE)') 'INSERT INTO ipallocations (port_id, ip_address, subnet_id, network_id) VALUES (%s, %s, %s, %s)' ('dc359c7e-59b1-46d2-966f-194bc7fa0ffb', '2003::f816:3eff:fea6:41d3', 'dc9f1ac8-92c1-4f0e-8b86-524824861fa3', '45e4e8ea-81ed-46ad-a753-cdca392adb2a') Here is a full traceback: 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource Traceback (most recent call last): 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/api/v2/resource.py", line 83, in resource 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource result = method(request=request, **args) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/api/v2/base.py", line 461, in create 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource obj = obj_creator(request.context, **kwargs) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/plugins/ml2/plugin.py", line 794, in create_subnet 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource result, mech_context = self._create_subnet_db(context, subnet) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/plugins/ml2/plugin.py", line 785, in _create_subnet_db 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource result = super(Ml2Plugin, self).create_subnet(context, subnet) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 1349, in create_subnet 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource raise n_exc.BadRequest(resource='subnets', msg=msg) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 1285, in _create_subnet_from_implicit_pool 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource # If this subnet supports auto-addressing, then update any 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 1379, in _add_auto_addrs_on_network_ports 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource except db_exc.DBReferenceError: 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 470, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.rollback() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource compat.reraise(exc_type, exc_value, exc_tb) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 467, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.commit() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 377, in commit 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self._prepare_impl() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 357, in _prepare_impl 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.session.flush() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self._flush(objects) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource transaction.rollback(_capture_exception=True) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource compat.reraise(exc_type, exc_value, exc_tb) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource flush_context.execute() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource rec.execute(self) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource uow 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 65, in save_obj 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource mapper, table, insert) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 570, in _emit_insert_statements 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource execute(statement, multiparams) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource return meth(self, multiparams, params) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 321, in _execute_on_connection 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource return connection._execute_clauseelement(self, multiparams, params) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource compiled_sql, distilled_params 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource context) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource e, statement, parameters, cursor, context) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1156, in _handle_dbapi_exception 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource util.raise_from_cause(newraise, exc_info) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource reraise(type(exception), exception, tb=exc_tb) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource context) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource cursor.execute(statement, parameters) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.errorhandler(self, exc, value) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource raise errorclass, errorvalue 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource DBReferenceError: (IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`neutron`.`ipallocations`, CONSTRAINT `ipallocations_ibfk_2` FOREIGN KEY (`port_id`) REFERENCES `ports` (`id`) ON DELETE CASCADE)') 'INSERT INTO ipallocations (port_id, ip_address, subnet_id, network_id) VALUES (%s, %s, %s, %s)' ('dc359c7e-59b1-46d2-966f-194bc7fa0ffb', '2003::f816:3eff:fea6:41d3', 'dc9f1ac8-92c1-4f0e-8b86-524824861fa3', '45e4e8ea-81ed-46ad-a753-cdca392adb2a') ** Affects: neutron Importance: Undecided Status: New -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to neutron. https://bugs.launchpad.net/bugs/1441382 Title: IPv6 SLAAC subnet Tempest tests fail due to IntegrityError Status in OpenStack Neutron (virtual network service): New Bug description: If 2 Tempest tests are run concurrently as follows: Test 1: Any test that uses DHCP and cleans up the DHCP port as part of cleanup. For example: tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_router Test 2: Any test that creates an IPv6 SLAAC subnet tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_64_subnets and this patch has been applied to eliminate DB access deadlocks: https://review.openstack.org/#/c/170690/ Then occasionally Test 2 will fail in subnet create, and the following error can be observed in the neutron server log (q-svc.log): TRACE neutron.api.v2.resource DBReferenceError: (IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`neutron`.`ipallocations`, CONSTRAINT `ipallocations_ibfk_2` FOREIGN KEY (`port_id`) REFERENCES `ports` (`id`) ON DELETE CASCADE)') 'INSERT INTO ipallocations (port_id, ip_address, subnet_id, network_id) VALUES (%s, %s, %s, %s)' ('dc359c7e-59b1-46d2-966f- 194bc7fa0ffb', '2003::f816:3eff:fea6:41d3', 'dc9f1ac8-92c1-4f0e- 8b86-524824861fa3', '45e4e8ea-81ed-46ad-a753-cdca392adb2a') Here is a full traceback: 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource Traceback (most recent call last): 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/api/v2/resource.py", line 83, in resource 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource result = method(request=request, **args) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/api/v2/base.py", line 461, in create 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource obj = obj_creator(request.context, **kwargs) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/plugins/ml2/plugin.py", line 794, in create_subnet 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource result, mech_context = self._create_subnet_db(context, subnet) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/plugins/ml2/plugin.py", line 785, in _create_subnet_db 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource result = super(Ml2Plugin, self).create_subnet(context, subnet) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 1349, in create_subnet 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource raise n_exc.BadRequest(resource='subnets', msg=msg) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 1285, in _create_subnet_from_implicit_pool 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource # If this subnet supports auto-addressing, then update any 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/opt/stack/neutron/neutron/db/db_base_plugin_v2.py", line 1379, in _add_auto_addrs_on_network_ports 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource except db_exc.DBReferenceError: 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 470, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.rollback() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource compat.reraise(exc_type, exc_value, exc_tb) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 467, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.commit() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 377, in commit 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self._prepare_impl() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 357, in _prepare_impl 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.session.flush() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1919, in flush 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self._flush(objects) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2037, in _flush 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource transaction.rollback(_capture_exception=True) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__ 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource compat.reraise(exc_type, exc_value, exc_tb) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2001, in _flush 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource flush_context.execute() 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 372, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource rec.execute(self) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 526, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource uow 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 65, in save_obj 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource mapper, table, insert) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 570, in _emit_insert_statements 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource execute(statement, multiparams) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 729, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource return meth(self, multiparams, params) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 321, in _execute_on_connection 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource return connection._execute_clauseelement(self, multiparams, params) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 826, in _execute_clauseelement 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource compiled_sql, distilled_params 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 958, in _execute_context 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource context) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/oslo_db/sqlalchemy/compat/handle_error.py", line 261, in _handle_dbapi_exception 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource e, statement, parameters, cursor, context) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1156, in _handle_dbapi_exception 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource util.raise_from_cause(newraise, exc_info) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource reraise(type(exception), exception, tb=exc_tb) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 951, in _execute_context 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource context) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 436, in do_execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource cursor.execute(statement, parameters) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource self.errorhandler(self, exc, value) 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource raise errorclass, errorvalue 2015-04-07 16:39:16.026 TRACE neutron.api.v2.resource DBReferenceError: (IntegrityError) (1452, 'Cannot add or update a child row: a foreign key constraint fails (`neutron`.`ipallocations`, CONSTRAINT `ipallocations_ibfk_2` FOREIGN KEY (`port_id`) REFERENCES `ports` (`id`) ON DELETE CASCADE)') 'INSERT INTO ipallocations (port_id, ip_address, subnet_id, network_id) VALUES (%s, %s, %s, %s)' ('dc359c7e-59b1-46d2-966f-194bc7fa0ffb', '2003::f816:3eff:fea6:41d3', 'dc9f1ac8-92c1-4f0e-8b86-524824861fa3', '45e4e8ea-81ed-46ad-a753-cdca392adb2a') To manage notifications about this bug go to: https://bugs.launchpad.net/neutron/+bug/1441382/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp