Re: [Freeipa-users] concurrent requests to ipalib app giving network error

2016-04-26 Thread Petr Vobornik 
On 04/22/2016 08:44 AM, Martin Basti wrote:
> 
> 
> On 21.04.2016 18:46, Oğuz Yarımtepe wrote:
>> Hi,
>>
>> I have a REST API that is using the ipalib and written with Falcon.
>> Below is the code or you can check it online here: 
>> http://paste.ubuntu.com/15966308/
>>
>> from __future__ import print_function
>> from bson import json_util
>> import json
>> import falcon
>>
>> from ipalib import api as ipaapi
>> from api.utils.utils import parse_json, check_connection
>> from api import settings
>>
>> class Calls(object):
>>
>> #@falcon.before(check_connection)
>> def on_post(self, req, resp):
>>
>> result_json = parse_json(req)
>> command_name = result_json["command_name"]
>> params = result_json["params"]
>>
>> if not hasattr(ipaapi.env, "conf"):
>> #TODO: add kinit oguz for exceptional case
>>  ipaapi.bootstrap_with_global_options(context='satcloud_api')
>> ipaapi.finalize()
>>
>> if ipaapi.env.in_server:
>> ipaapi.Backend.ldap2.connect()
>> else:
>> ipaapi.Backend.rpcclient.connect()
>>
>> #import ipdb
>> #ipdb.set_trace()
>>
>> command=ipaapi.Command
>> command_result=getattr(command,command_name)
>>
>> #resp.set_cookie('api_status_cookie', 'True')
>> if not params:
>> resp.body = json.dumps(command_result())
>> resp.status = falcon.HTTP_200
>> else:
>> if type(params) == dict:
>> arguments = []
>> kwargs = dict()
>> for key, value in params.iteritems():
>> if "arg" in key:
>> arguments.append(value)
>> else:
>> kwargs[key]=value
>> try:
>> #for datetime serialization problems better to use bson
>> dump = command_result(*arguments, **kwargs)
>> resp.body = json.dumps(dump, default=json_util.default)
>> #resp.body = json.dumps(command_result(*arguments, 
>> **kwargs))
>> resp.status = falcon.HTTP_200
>> except UnicodeDecodeError:
>> resp.body = json.dumps(dump, default=json_util.default, 
>> encoding='latin1')
>> resp.status = falcon.HTTP_200
>> except Exception as e:
>> resp.status = falcon.HTTP_BAD_REQUEST
>> resp.body = json.dumps({"description": e.message, 
>> "title": 
>> "Dublicate entry"})
>> #raise falcon.HTTPBadRequest(title="Dublicate 
>> entry",
>> #  description=e,
>> #  href=settings.__docs__)
>> else:
>> dump = command_result(params)
>> resp.body = json.dumps(dump, default=json_util.default)
>> #resp.body = json.dumps(command_result(params))
>> resp.status = falcon.HTTP_200
>>
>>
>> Basically i am making concurrent calls to this rest api and i am getting
>>
>> Network error: http://paste.ubuntu.com/15966347/
>>
>> ipa: INFO: Forwarding 'user_find' to json server 
>> 'https://ipa.foo.com/ipa/json'
>> ipa: INFO: Forwarding 'netgroup_find' to json server 
>> 'https://ipa.foo.com/ipa/json'
>> [pid: 5450|app: 0|req: 9/14] 10.102.235.77 () {34 vars in 463 bytes} [Thu 
>> Apr 
>> 21 17:43:22 2016] POST /v1/ipa/calls => generated 2324 bytes in 227 msecs 
>> (HTTP/1.1 200) 8 headers in 459 bytes (1 switches on core 0)
>> Traceback (most recent call last):
>>   File "falcon/api.py", line 213, in falcon.api.API.__call__ 
>> (falcon/api.c:2521)
>>   File "falcon/api.py", line 182, in falcon.api.API.__call__ 
>> (falcon/api.c:2118)
>>   File "./api/resources/ipa/calls.py", line 38, in on_post
>> resp.body = json.dumps(command_result())
>>   File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 443, in 
>> __call__
>> ret = self.run(*args, **options)
>>   File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 761, in 
>> run
>> return self.forward(*args, **options)
>>   File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 782, in 
>> forward
>> return self.Backend.rpcclient.forward(self.name , 
>> *args, 
>> **kw)
>>   File "/usr/lib/python2.7/site-packages/ipalib/rpc.py", line 935, in forward
>> raise NetworkError(uri=server, error=e.errmsg)
>> ipalib.errors.NetworkError: cannot connect to 
>> 'https://ipa.foo.com/ipa/json': Internal 
>> Server 
>> Error
>> [pid: 5451|app: 0|req: 3/15] 10.102.235.77 () {34 vars in 463 bytes} [Thu 
>> Apr 
>> 21 17:43:22 2016] POST /v1/ipa/calls => generated 0 bytes in 1421 msecs 
>> (HTTP/1.1 500) 0 headers in 0 bytes (0 switches

Re: [Freeipa-users] concurrent requests to ipalib app giving network error

2016-04-22 Thread Martin Basti 



On 21.04.2016 18:46, Oğuz Yarımtepe wrote:

Hi,

I have a REST API that is using the ipalib and written with Falcon.
Below is the code or you can check it online here: 
http://paste.ubuntu.com/15966308/


from __future__ import print_function
from bson import json_util
import json
import falcon

from ipalib import api as ipaapi
from api.utils.utils import parse_json, check_connection
from api import settings

class Calls(object):

#@falcon.before(check_connection)
def on_post(self, req, resp):

result_json = parse_json(req)
command_name = result_json["command_name"]
params = result_json["params"]

if not hasattr(ipaapi.env, "conf"):
#TODO: add kinit oguz for exceptional case
 ipaapi.bootstrap_with_global_options(context='satcloud_api')
ipaapi.finalize()

if ipaapi.env.in_server:
ipaapi.Backend.ldap2.connect()
else:
ipaapi.Backend.rpcclient.connect()

#import ipdb
#ipdb.set_trace()

command=ipaapi.Command
command_result=getattr(command,command_name)

#resp.set_cookie('api_status_cookie', 'True')
if not params:
resp.body = json.dumps(command_result())
resp.status = falcon.HTTP_200
else:
if type(params) == dict:
arguments = []
kwargs = dict()
for key, value in params.iteritems():
if "arg" in key:
arguments.append(value)
else:
kwargs[key]=value
try:
#for datetime serialization problems better to use 
bson

dump = command_result(*arguments, **kwargs)
resp.body = json.dumps(dump, 
default=json_util.default)
#resp.body = json.dumps(command_result(*arguments, 
**kwargs))

resp.status = falcon.HTTP_200
except UnicodeDecodeError:
resp.body = json.dumps(dump, 
default=json_util.default, encoding='latin1')

resp.status = falcon.HTTP_200
except Exception as e:
resp.status = falcon.HTTP_BAD_REQUEST
resp.body = json.dumps({"description": e.message, 
"title": "Dublicate entry"})
#raise 
falcon.HTTPBadRequest(title="Dublicate entry",

#  description=e,
#  href=settings.__docs__)
else:
dump = command_result(params)
resp.body = json.dumps(dump, default=json_util.default)
#resp.body = json.dumps(command_result(params))
resp.status = falcon.HTTP_200


Basically i am making concurrent calls to this rest api and i am getting

Network error: http://paste.ubuntu.com/15966347/

ipa: INFO: Forwarding 'user_find' to json server 
'https://ipa.foo.com/ipa/json'
ipa: INFO: Forwarding 'netgroup_find' to json server 
'https://ipa.foo.com/ipa/json'
[pid: 5450|app: 0|req: 9/14] 10.102.235.77 () {34 vars in 463 bytes} 
[Thu Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 2324 bytes 
in 227 msecs (HTTP/1.1 200) 8 headers in 459 bytes (1 switches on core 0)

Traceback (most recent call last):
  File "falcon/api.py", line 213, in falcon.api.API.__call__ 
(falcon/api.c:2521)
  File "falcon/api.py", line 182, in falcon.api.API.__call__ 
(falcon/api.c:2118)

  File "./api/resources/ipa/calls.py", line 38, in on_post
resp.body = json.dumps(command_result())
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 
443, in __call__

ret = self.run(*args, **options)
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 
761, in run

return self.forward(*args, **options)
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 
782, in forward
return self.Backend.rpcclient.forward(self.name 
, *args, **kw)
  File "/usr/lib/python2.7/site-packages/ipalib/rpc.py", line 935, in 
forward

raise NetworkError(uri=server, error=e.errmsg)
ipalib.errors.NetworkError: cannot connect to 
'https://ipa.foo.com/ipa/json': Internal Server Error
[pid: 5451|app: 0|req: 3/15] 10.102.235.77 () {34 vars in 463 bytes} 
[Thu Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 0 bytes in 
1421 msecs (HTTP/1.1 500) 0 headers in 0 bytes (0 switches on core 0)



This is how a concurrent request is being sent:
#!/usr/bin/env python

from multiprocessing import Process, Pool
import time
import urllib2

def millis():
  return int(round(time.time() * 1000))

def http_get(url):
  start_time = millis()
  request = urllib2.Request(url, headers={"Content-Type": 
"application/json", "Origin": "http://ipa.foo.com;, "Authorization": 
"{'token':

[Freeipa-users] concurrent requests to ipalib app giving network error

2016-04-21 Thread Oğuz Yarımtepe 
Hi,

I have a REST API that is using the ipalib and written with Falcon.
Below is the code or you can check it online here:
http://paste.ubuntu.com/15966308/

from __future__ import print_function
from bson import json_util
import json
import falcon

from ipalib import api as ipaapi
from api.utils.utils import parse_json, check_connection
from api import settings

class Calls(object):

#@falcon.before(check_connection)
def on_post(self, req, resp):

result_json = parse_json(req)
command_name = result_json["command_name"]
params = result_json["params"]

if not hasattr(ipaapi.env, "conf"):
#TODO: add kinit oguz for exceptional case
ipaapi.bootstrap_with_global_options(context='satcloud_api')
ipaapi.finalize()

if ipaapi.env.in_server:
ipaapi.Backend.ldap2.connect()
else:
ipaapi.Backend.rpcclient.connect()

#import ipdb
#ipdb.set_trace()

command=ipaapi.Command
command_result=getattr(command,command_name)

#resp.set_cookie('api_status_cookie', 'True')
if not params:
resp.body = json.dumps(command_result())
resp.status = falcon.HTTP_200
else:
if type(params) == dict:
arguments = []
kwargs = dict()
for key, value in params.iteritems():
if "arg" in key:
arguments.append(value)
else:
kwargs[key]=value
try:
#for datetime serialization problems better to use bson
dump = command_result(*arguments, **kwargs)
resp.body = json.dumps(dump, default=json_util.default)
#resp.body = json.dumps(command_result(*arguments,
**kwargs))
resp.status = falcon.HTTP_200
except UnicodeDecodeError:
resp.body = json.dumps(dump, default=json_util.default,
encoding='latin1')
resp.status = falcon.HTTP_200
except Exception as e:
resp.status = falcon.HTTP_BAD_REQUEST
resp.body = json.dumps({"description": e.message,
"title": "Dublicate entry"})
#raise falcon.HTTPBadRequest(title="Dublicate
entry",
#description=e,
#href=settings.__docs__)
else:
dump = command_result(params)
resp.body = json.dumps(dump, default=json_util.default)
#resp.body = json.dumps(command_result(params))
resp.status = falcon.HTTP_200


Basically i am making concurrent calls to this rest api and i am getting

Network error: http://paste.ubuntu.com/15966347/

ipa: INFO: Forwarding 'user_find' to json server '
https://ipa.foo.com/ipa/json'
ipa: INFO: Forwarding 'netgroup_find' to json server '
https://ipa.foo.com/ipa/json'
[pid: 5450|app: 0|req: 9/14] 10.102.235.77 () {34 vars in 463 bytes} [Thu
Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 2324 bytes in 227
msecs (HTTP/1.1 200) 8 headers in 459 bytes (1 switches on core 0)
Traceback (most recent call last):
  File "falcon/api.py", line 213, in falcon.api.API.__call__
(falcon/api.c:2521)
  File "falcon/api.py", line 182, in falcon.api.API.__call__
(falcon/api.c:2118)
  File "./api/resources/ipa/calls.py", line 38, in on_post
resp.body = json.dumps(command_result())
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 443, in
__call__
ret = self.run(*args, **options)
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 761, in
run
return self.forward(*args, **options)
  File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 782, in
forward
return self.Backend.rpcclient.forward(self.name, *args, **kw)
  File "/usr/lib/python2.7/site-packages/ipalib/rpc.py", line 935, in
forward
raise NetworkError(uri=server, error=e.errmsg)
ipalib.errors.NetworkError: cannot connect to 'https://ipa.foo.com/ipa/json':
Internal Server Error
[pid: 5451|app: 0|req: 3/15] 10.102.235.77 () {34 vars in 463 bytes} [Thu
Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 0 bytes in 1421 msecs
(HTTP/1.1 500) 0 headers in 0 bytes (0 switches on core 0)


This is how a concurrent request is being sent:
#!/usr/bin/env python

from multiprocessing import Process, Pool
import time
import urllib2

def millis():
  return int(round(time.time() * 1000))

def http_get(url):
  start_time = millis()
  request = urllib2.Request(url, headers={"Content-Type":
"application/json", "Origin": "http://ipa.foo.com;, "Authorization":
"{'token':
'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzcnYiOiJpcGEuc2F0Y2xvdWQuY29tLnRyIiwic3ViIjoiMGU1ZGZkNDc3N2I2NmNhOTU3ZTc4ZmJhZjMxNjYxMmEifQ.cr8cNy7zgQkY-q7UUyTCNPCjGlmz-LCCzUYSUV9P694'}"})