Can you try setting the max connections for SQL to the number of API workers? http://docs.openstack.org/trunk/config-reference/content/networking-options-db.html On Oct 22, 2014 4:31 AM, "James Page" <[email protected]> wrote:
> Public bug reported: > > OpenStack Juno release, Ubuntu 14.04 using Cloud Archive; under > relatively high instance creation concurrency (150), neutron starts to > throw some errors: > > 2014-10-21 16:40:44.124 16312 ERROR oslo.messaging._drivers.common > [req-8e3ebbdb-bc01-439d-af86-655176f206a6 ] ['Traceback (most recent call > last):\n', ' File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 134, in _dispatch_and_reply\n incoming.message))\n', ' File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 177, in _dispatch\n return self._do_dispatch(endpoint, method, ctxt, > args)\n', ' File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 123, in _do_dispatch\n result = getattr(endpoint, method)(ctxt, > **new_args)\n', ' File > "/usr/lib/python2.7/dist-packages/neutron/api/rpc/handlers/securitygroups_rpc.py", > line 74, in security_group_info_for_devices\n ports = > self._get_devices_info(devices_info)\n', ' File > "/usr/lib/python2.7/dist-packages/neutron/api/rpc/handlers/securitygroups_rpc.py", > line 41, in _get_devices_info\n port = > self.plugin.get_port_from_device(device)\n', ' File > "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line > 1161, in get_port_from_device\n port = db.get_port_and_sgs(port_id)\n', > ' File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/db.py", line > 222, in get_port_and_sgs\n port_and_sgs = query.all()\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2300, in > all\n return list(self)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2412, in > __iter__\n return self._execute_and_instances(context)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2425, in > _execute_and_instances\n close_with_result=True)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2416, in > _connection_from_session\n **kw)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 854, in > connection\n close_with_result=close_with_result)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 858, in > _connection_for_bind\n return > self.transaction._connection_for_bind(engine)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 322, in > _connection_for_bind\n conn = bind.contextual_connect()\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1799, in > contextual_connect\n self.pool.connect(),\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 338, in > connect\n return _ConnectionFairy._checkout(self)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 641, in > _checkout\n fairy = _ConnectionRecord.checkout(pool)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 440, in > checkout\n rec = pool._do_get()\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 957, in > _do_get\n (self.size(), self.overflow(), self._timeout))\n', > 'TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection > timed out, timeout 10\n'] > 2014-10-21 16:40:44.126 16312 ERROR oslo.messaging.rpc.dispatcher > [req-ea96dc85-dc0f-4ddc-a827-dbc25ab32a03 ] Exception during message > handling: QueuePool limit of size 10 overflow 20 reached, connection timed > out, timeout 10 > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > Traceback (most recent call last): > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 134, in _dispatch_and_reply > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > incoming.message)) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 177, in _dispatch > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self._do_dispatch(endpoint, method, ctxt, args) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 123, in _do_dispatch > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > result = getattr(endpoint, method)(ctxt, **new_args) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 237, in > report_state > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > self.plugin.create_or_update_agent(context, agent_state) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 197, in > create_or_update_agent > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self._create_or_update_agent(context, agent) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 176, in > _create_or_update_agent > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > context, agent['agent_type'], agent['host']) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 153, in > _get_agent_by_type_and_host > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > Agent.host == host).one() > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2369, in > one > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher ret > = list(self) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2412, in > __iter__ > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self._execute_and_instances(context) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2425, in > _execute_and_instances > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > close_with_result=True) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2416, in > _connection_from_session > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher **kw) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 854, in > connection > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > close_with_result=close_with_result) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 858, in > _connection_for_bind > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self.transaction._connection_for_bind(engine) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 322, in > _connection_for_bind > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher conn > = bind.contextual_connect() > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1799, in > contextual_connect > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > self.pool.connect(), > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 338, in connect > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return _ConnectionFairy._checkout(self) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 641, in > _checkout > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > fairy = _ConnectionRecord.checkout(pool) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 440, in checkout > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher rec > = pool._do_get() > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 957, in _do_get > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > (self.size(), self.overflow(), self._timeout)) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection > timed out, timeout 10 > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > > The cloud has three neutron-server instances, each configured with 32 > API workers. > > ** Affects: neutron > Importance: Undecided > Status: New > > ** Affects: neutron (Ubuntu) > Importance: Undecided > Status: New > > > ** Tags: juno scale-testing > > ** Tags added: juno scale-testing > > ** Also affects: neutron > Importance: Undecided > Status: New > > -- > You received this bug notification because you are subscribed to > neutron. > Matching subscriptions: NeutronBugs > https://bugs.launchpad.net/bugs/1384108 > > Title: > Exception during message handling: QueuePool limit of size 10 overflow > 20 reached, connection timed out, timeout 10 > > Status in OpenStack Neutron (virtual network service): > New > Status in “neutron” package in Ubuntu: > New > > Bug description: > OpenStack Juno release, Ubuntu 14.04 using Cloud Archive; under > relatively high instance creation concurrency (150), neutron starts to > throw some errors: > > 2014-10-21 16:40:44.124 16312 ERROR oslo.messaging._drivers.common > [req-8e3ebbdb-bc01-439d-af86-655176f206a6 ] ['Traceback (most recent call > last):\n', ' File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 134, in _dispatch_and_reply\n incoming.message))\n', ' File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 177, in _dispatch\n return self._do_dispatch(endpoint, method, ctxt, > args)\n', ' File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 123, in _do_dispatch\n result = getattr(endpoint, method)(ctxt, > **new_args)\n', ' File > "/usr/lib/python2.7/dist-packages/neutron/api/rpc/handlers/securitygroups_rpc.py", > line 74, in security_group_info_for_devices\n ports = > self._get_devices_info(devices_info)\n', ' File > "/usr/lib/python2.7/dist-packages/neutron/api/rpc/handlers/securitygroups_rpc.py", > line 41, in _get_devices_info\n port = > self.plugin.get_port_from_device(device)\n', ' File > "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/plugin.py", line > 1161, in get_port_from_device\n port = db.get_port_and_sgs(port_id)\n', > ' File "/usr/lib/python2.7/dist-packages/neutron/plugins/ml2/db.py", line > 222, in get_port_and_sgs\n port_and_sgs = query.all()\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2300, in > all\n return list(self)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2412, in > __iter__\n return self._execute_and_instances(context)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2425, in > _execute_and_instances\n close_with_result=True)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2416, in > _connection_from_session\n **kw)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 854, in > connection\n close_with_result=close_with_result)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 858, in > _connection_for_bind\n return > self.transaction._connection_for_bind(engine)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 322, in > _connection_for_bind\n conn = bind.contextual_connect()\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1799, in > contextual_connect\n self.pool.connect(),\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 338, in > connect\n return _ConnectionFairy._checkout(self)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 641, in > _checkout\n fairy = _ConnectionRecord.checkout(pool)\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 440, in > checkout\n rec = pool._do_get()\n', ' File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 957, in > _do_get\n (self.size(), self.overflow(), self._timeout))\n', > 'TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection > timed out, timeout 10\n'] > 2014-10-21 16:40:44.126 16312 ERROR oslo.messaging.rpc.dispatcher > [req-ea96dc85-dc0f-4ddc-a827-dbc25ab32a03 ] Exception during message > handling: QueuePool limit of size 10 overflow 20 reached, connection timed > out, timeout 10 > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > Traceback (most recent call last): > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 134, in _dispatch_and_reply > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > incoming.message)) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 177, in _dispatch > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self._do_dispatch(endpoint, method, ctxt, args) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line > 123, in _do_dispatch > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > result = getattr(endpoint, method)(ctxt, **new_args) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 237, in > report_state > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > self.plugin.create_or_update_agent(context, agent_state) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 197, in > create_or_update_agent > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self._create_or_update_agent(context, agent) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 176, in > _create_or_update_agent > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > context, agent['agent_type'], agent['host']) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/neutron/db/agents_db.py", line 153, in > _get_agent_by_type_and_host > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > Agent.host == host).one() > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2369, in > one > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > ret = list(self) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2412, in > __iter__ > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self._execute_and_instances(context) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2425, in > _execute_and_instances > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > close_with_result=True) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2416, in > _connection_from_session > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > **kw) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 854, in > connection > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > close_with_result=close_with_result) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 858, in > _connection_for_bind > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return self.transaction._connection_for_bind(engine) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 322, in > _connection_for_bind > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > conn = bind.contextual_connect() > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1799, in > contextual_connect > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > self.pool.connect(), > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 338, in connect > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > return _ConnectionFairy._checkout(self) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 641, in > _checkout > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > fairy = _ConnectionRecord.checkout(pool) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 440, in checkout > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > rec = pool._do_get() > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher File > "/usr/lib/python2.7/dist-packages/sqlalchemy/pool.py", line 957, in _do_get > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > (self.size(), self.overflow(), self._timeout)) > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > TimeoutError: QueuePool limit of size 10 overflow 20 reached, connection > timed out, timeout 10 > 2014-10-21 16:40:44.126 16312 TRACE oslo.messaging.rpc.dispatcher > > The cloud has three neutron-server instances, each configured with 32 > API workers. > > To manage notifications about this bug go to: > https://bugs.launchpad.net/neutron/+bug/1384108/+subscriptions > -- You received this bug notification because you are a member of Ubuntu Bugs, which is subscribed to Ubuntu. https://bugs.launchpad.net/bugs/1384108 Title: Exception during message handling: QueuePool limit of size 10 overflow 20 reached, connection timed out, timeout 10 To manage notifications about this bug go to: https://bugs.launchpad.net/neutron/+bug/1384108/+subscriptions -- ubuntu-bugs mailing list [email protected] https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
