Hello
I am testing web2py with uWSGI and some request have random timeout (after
IO Heavy operation is started , but it is spawned using subprocess.Popen())
And then progress is selected from DB Every 5 Seconds (Progress Percent
Insert,Commit to db is only done at each 5% intervals)
At the IO Heavy times , the progress status is Timedout! even tho I set
timeouts to really huge 6000 secs , it still timeout.
Also it seems that all uWSGI worker processes are running at the same core
as extraction process is running . how can i make it run in different cores
(to make use of multple cores?)
Here is my uWSGI config:
./uwsgi/uwsgi --http :8000 --chdir ./web2py/ --python-path ./web2py/
--module uwsgihandler \
--socket /tmp/web2py.sock -p 12 --harakiri-verbose --socket-timeout 6000
--harakiri-verbose 6000
Here is the controller looks like :
@service.jsonrpc
def import_new(case_id,source_path,extracted_path,
custodian,batch_id,dedupe_scope,dedupe_fields):
try:
#print 'CID: %s, SP: %s, EP: %s, Cu: %s, BID: %s'% (case_id,
# source_path,
# extracted_path,
# custodian,
# batch_id)
extracted_path = os.path.join(extracted_path,batch_id,"completed")
query = db(db.casedb.id == case_id).select(db.casedb.name).last()
db_name = query.name
casesdb = caseutils.setupTBL(db_name, request.folder + 'databases')
#print dedupe_mode
import_id=casesdb.import_data.insert(custodian=custodian,
case_id=case_id,
batch_hash=batch_id,
extracted_path = extracted_path,
dedupe_scope=dedupe_scope,
dedupe_mode=dedupe_fields)
#print 'IMPORT_ID',import_id
session.import_id=import_id
_start_processing(case_id,extracted_path,request.folder +
"databases",import_id,custodian, dedupe_scope, dedupe_fields)
#THIS ONE is IO HEAVY but it is started as subprocess.Popen without waiting
for it to be done.
except:
print 'Err#',traceback.format_exc()
raise
return dict(case_id=case_id,extracted_path=extracted_path,
import_id=import_id)
def
_start_processing(case_id,extracted_path,db_path,import_id,custodian,dedupe_scope,dedupe_fields):
import shlex
cmd = "python ../processes/extractor_process.py %s '%s' '%s' %s '%s'
'%s' '%s'" %
(case_id,extracted_path,db_path,import_id,custodian,dedupe_scope,dedupe_fields)
#print 'CMD:',cmd
#tmp1 = shlex.split(cmd)
tmp1 = [ x.replace("\0", "") for x in shlex.split(cmd) ]
#print 'TMP1', tmp1
subprocess.Popen(tmp1) #Start of the IO Heavy
process, i know it is not safe to do that way but we will add security
optimizations later.
return True
@service.jsonrpc
def get_extraction_progress(import_id, db_name): #EXTRACTION
PROGRESS! Timeout Occours here When uWSGI is used ! Never occours before
using web2py Rocket
casesdb = caseutils.setupTBL(db_name, request.folder + 'databases')
rows = casesdb(casesdb.progress_data.import_id ==
import_id).select().last()
if rows:
return dict(extract_count = rows.extract_count,
extract_file = rows.extract_file,
extract_start = str(rows.extract_start),
extract_end = str(rows.extract_end))
else:
return dict(extract_count = 0,
extract_file = "",
extract_start = "None",
extract_end = "None")
@service.jsonrpc
def get_parsing_progress(import_id, db_name): #PARSER PROGRESS! Timeout
Occours here When uWSGI is used ! Never occours before using web2py Rocket
casesdb = caseutils.setupTBL(db_name, request.folder + 'databases')
rows = casesdb(casesdb.progress_data.import_id ==
import_id).select().last()
if rows:
return dict(progress_percent = rows.progress_percent,
processed_files = rows.processed_files,
total_files = rows.total_files,
process_start = str(rows.process_start),
process_end = str(rows.process_end))
else:
return dict(progress_percent = 0,
processed_files = "None",
total_files = 0,
process_start = "None",
process_end = "None")
