On 21.04.2016 18:46, Oğuz Yarımtepe wrote:
Hi,

I have a REST API that is using the ipalib and written with Falcon.
Below is the code or you can check it online here: http://paste.ubuntu.com/15966308/

from __future__ import print_function
from bson import json_util
import json
import falcon

from ipalib import api as ipaapi
from api.utils.utils import parse_json, check_connection
from api import settings

class Calls(object):

    #@falcon.before(check_connection)
    def on_post(self, req, resp):

        result_json = parse_json(req)
        command_name = result_json["command_name"]
        params = result_json["params"]

        if not hasattr(ipaapi.env, "conf"):
            #TODO: add kinit oguz for exceptional case
 ipaapi.bootstrap_with_global_options(context='satcloud_api')
            ipaapi.finalize()

            if ipaapi.env.in_server:
                ipaapi.Backend.ldap2.connect()
            else:
                ipaapi.Backend.rpcclient.connect()

        #import ipdb
        #ipdb.set_trace()

        command=ipaapi.Command
        command_result=getattr(command,command_name)

        #resp.set_cookie('api_status_cookie', 'True')
        if not params:
            resp.body = json.dumps(command_result())
            resp.status = falcon.HTTP_200
        else:
            if type(params) == dict:
                arguments = []
                kwargs = dict()
                for key, value in params.iteritems():
                    if "arg" in key:
                        arguments.append(value)
                    else:
                        kwargs[key]=value
                try:
#for datetime serialization problems better to use bson
                    dump = command_result(*arguments, **kwargs)
resp.body = json.dumps(dump, default=json_util.default) #resp.body = json.dumps(command_result(*arguments, **kwargs))
                    resp.status = falcon.HTTP_200
                except UnicodeDecodeError:
resp.body = json.dumps(dump, default=json_util.default, encoding='latin1')
                    resp.status = falcon.HTTP_200
                except Exception as e:
                    resp.status = falcon.HTTP_BAD_REQUEST
resp.body = json.dumps({"description": e.message, "title": "Dublicate entry"}) #raise falcon.HTTPBadRequest(title="Dublicate entry",
                                        #  description=e,
                                        #  href=settings.__docs__)
            else:
                dump = command_result(params)
                resp.body = json.dumps(dump, default=json_util.default)
                #resp.body = json.dumps(command_result(params))
                resp.status = falcon.HTTP_200


Basically i am making concurrent calls to this rest api and i am getting

Network error: http://paste.ubuntu.com/15966347/

ipa: INFO: Forwarding 'user_find' to json server 'https://ipa.foo.com/ipa/json' ipa: INFO: Forwarding 'netgroup_find' to json server 'https://ipa.foo.com/ipa/json' [pid: 5450|app: 0|req: 9/14] 10.102.235.77 () {34 vars in 463 bytes} [Thu Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 2324 bytes in 227 msecs (HTTP/1.1 200) 8 headers in 459 bytes (1 switches on core 0)
Traceback (most recent call last):
File "falcon/api.py", line 213, in falcon.api.API.__call__ (falcon/api.c:2521) File "falcon/api.py", line 182, in falcon.api.API.__call__ (falcon/api.c:2118)
  File "./api/resources/ipa/calls.py", line 38, in on_post
    resp.body = json.dumps(command_result())
File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 443, in __call__
    ret = self.run(*args, **options)
File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 761, in run
    return self.forward(*args, **options)
File "/usr/lib/python2.7/site-packages/ipalib/frontend.py", line 782, in forward return self.Backend.rpcclient.forward(self.name <http://self.name>, *args, **kw) File "/usr/lib/python2.7/site-packages/ipalib/rpc.py", line 935, in forward
    raise NetworkError(uri=server, error=e.errmsg)
ipalib.errors.NetworkError: cannot connect to 'https://ipa.foo.com/ipa/json': Internal Server Error [pid: 5451|app: 0|req: 3/15] 10.102.235.77 () {34 vars in 463 bytes} [Thu Apr 21 17:43:22 2016] POST /v1/ipa/calls => generated 0 bytes in 1421 msecs (HTTP/1.1 500) 0 headers in 0 bytes (0 switches on core 0)


This is how a concurrent request is being sent:
#!/usr/bin/env python

from multiprocessing import Process, Pool
import time
import urllib2

def millis():
  return int(round(time.time() * 1000))

def http_get(url):
  start_time = millis()
request = urllib2.Request(url, headers={"Content-Type": "application/json", "Origin": "http://ipa.foo.com";, "Authorization": "{'token': 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzcnYiOiJpcGEuc2F0Y2xvdWQuY29tLnRyIiwic3ViIjoiMGU1ZGZkNDc3N2I2NmNhOTU3ZTc4ZmJhZjMxNjYxMmEifQ.cr8cNy7zgQkY-q7UUyTCNPCjGlmz-LCCzUYSUV9P694'}"}) result = {"url": url, "data": urllib2.urlopen(request, timeout=10).read()[:100]} #result = {"url": url, "data": urllib2.urlopen(request, timeout=5).read()}
  print url + " took " + str(millis() - start_time) + " ms"
  return result


urls = ['http://api.foo.com:8888/v1/users', 'http://api.foo.com:8888/v1/organizations']

pool = Pool(processes=2)

start_time = millis()
results = pool.map(http_get, urls)

print "\nTotal took " + str(millis() - start_time) + " ms\n"

for result in results:
  print result

I am confused about the reason of the error. Any idea?


--
Oğuz Yarımtepe
http://about.me/oguzy




Hello, could you check /var/logs/httpd/error_log if there is any info about Internal server error?

It looks like there is no session cookie set (but not sure). IMO because the parallel processing you may need to use local instances of API instead the global one for each thread/process.

From top of my head:

api = create_api(mode=None)
api.bootstrap(<optionshere>)
api.finalize()


But I'm not sure what is the exact problem, you need try :)

Martin


-- 
Manage your subscription for the Freeipa-users mailing list:
https://www.redhat.com/mailman/listinfo/freeipa-users
Go to http://freeipa.org for more info on the project

Reply via email to