Sorry for late reply and thank you so much, Niphlod! I am still playing with Scheduler, so I didn't know how I should use it.
One more question, I am testing the scheduler to write a csv file from reading JSON formatted data, The scheduler stops at one point, I get "STOPPED" on the terminal I am using csvkit library and here's the code. def json2csv(datatype): turl = '' filetype='' turl = 'http://www.source.com/'+datatype filetype=str('cdef-'+datatype+'.csv') #response.headers['Content-Type'] = contenttype('.csv') #disposition = 'attachment; filename='+filetype+'output.csv' #response.headers['Content-disposition'] = disposition inputfile = urllib2.urlopen(turl) outputfile = convert.json2csv(inputfile,key='value') myFile = open('/home/www-data/web2py/applications/cdef/static/cdef/'+filetype,'w') myFile.write(outputfile) myFile.close() db.data_dw.update_or_insert(db.data_dw.datatype==datatype,datatype=datatype,filename=filetype,fundtype='cdef',uploaddate=request.now) db.commit() return True On Tuesday, November 24, 2015 at 9:52:30 AM UTC-5, Niphlod wrote: > > please remove any print statement .... and .... why are you importing > modules inside the function ? > > On Tuesday, November 24, 2015 at 1:15:15 PM UTC+1, kenny c wrote: >> >> Just tested with psycopg and same error. >> ===== >> ERROR:web2py.scheduler.postit#20382:Error retrieving status >> Exception in thread Thread-1: >> Traceback (most recent call last): >> File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner >> self.run() >> File "/home/www-data/web2py/gluon/scheduler.py", line 442, in run >> self.send_heartbeat(counter) >> File "/home/www-data/web2py/gluon/scheduler.py", line 1041, in >> send_heartbeat >> db.rollback() >> File "/home/www-data/web2py/gluon/packages/dal/pydal/base.py", line >> 937, in rollback >> self._adapter.rollback() >> File "/home/www-data/web2py/gluon/packages/dal/pydal/adapters/base.py", >> line 1332, in rollback >> return self.connection.rollback() >> InterfaceError: connection already closed >> >> Traceback (most recent call last): >> File "/home/www-data/web2py/gluon/shell.py", line 271, in run >> exec(python_code, _env) >> File "<string>", line 1, in <module> >> File "/home/www-data/web2py/gluon/scheduler.py", line 730, in loop >> self.wrapped_report_task(task, self.async(task)) >> File "/home/www-data/web2py/gluon/scheduler.py", line 879, in >> wrapped_report_task >> db.rollback() >> File "/home/www-data/web2py/gluon/packages/dal/pydal/base.py", line >> 937, in rollback >> self._adapter.rollback() >> File "/home/www-data/web2py/gluon/packages/dal/pydal/adapters/base.py", >> line 1332, in rollback >> return self.connection.rollback() >> InterfaceError: connection already closed >> ===== >> >> function crawls data from five RSS feed links that can have more than one >> page, due to this connection issue, I defined istart=0 and iend=1 to read >> only one page at a time >> >> def feeddata(istart,iend): >> import feedparser >> import datetime >> import ciso8601 >> import random >> for i in range(istart,iend): >> print i >> for a in feed_list: >> e = feedparser.parse(a+str(i)) >> for row in e.entries: >> dtime = float(row.a)/1000.0 >> d = >> datetime.datetime.fromtimestamp(dtime).strftime('%Y-%m-%d %H:%M:%S') >> dt_endtime = ciso8601.parse_datetime_unaware(d) >> img_link = BeautifulSoup(row.f).find('img')['src'] >> if row.rx_bidcount > 0: >> db.abpost.update_or_insert(db.abpost.post_id == >> row.id, >> post_id = row.id, >> item_id = str(row.id).split('/')[5].split('?')[0], >> published = row.published, >> a = row.a, >> b = row.b, >> c = row.c, >> d = float(row.d)/100, >> e = row.e, >> f = f, >> g = row.g, >> h = row.h, >> img_link = img_link, >> upvote = random.randint(10, 1000), >> downvote = random.randint(1, 5), >> title = row.title, >> title_detail = row.title_detail) >> db.commit() >> return "Done" >> >> On Monday, November 23, 2015 at 6:11:04 PM UTC-5, Niphlod wrote: >>> >>> not to point fingers around, but can you try installing psycopg2 ? it'll >>> override pg8000 that showed here and there some obscure bugs: pscopg2 is >>> certainly more solid. If the same happens with psycopg2, can we see the >>> actual function queued (or a function that replicates this behaviour) ? >>> >>> >>> BTW: send_heartbeat() shouldn't timeout that easily: it's called every 3 >>> seconds so AT LEAST THAT connection should never be dropped. >>> >>> On Sunday, November 22, 2015 at 5:00:31 AM UTC+1, kenny c wrote: >>>> >>>> I think there is timeout issue on the scheduler and I am not sure which >>>> one it is.. >>>> >>>> Seems to me that the Scheduler stops if it takes longer than 5 minutes >>>> in regardless of the value of timeout saved in the task row. >>>> >>>> On Saturday, November 21, 2015 at 8:12:05 PM UTC-5, kenny c wrote: >>>>> >>>>> HI, I am not sure what causes this issue.. >>>>> >>>>> I am running Web2py with Postgresql, I am getting this exception in >>>>> thread while running a scheduler. and I get InterfaceError("Connection is >>>>> closed") . >>>>> >>>>> I am wondering if Postgresql has timeout setting or scheduler timeout? >>>>> (I made sure that timeout is long enough to handle all the jobs) >>>>> >>>>> Any help would be appreciated. Thank you and please see the log below. >>>>> >>>>> ERROR:web2py.scheduler.example#6580:Error retrieving status >>>>> Exception in thread Thread-1: >>>>> Traceback (most recent call last): >>>>> File "/usr/lib/python2.7/threading.py", line 810, in >>>>> __bootstrap_inner >>>>> self.run() >>>>> File "/home/www-data/web2py/gluon/scheduler.py", line 442, in run >>>>> self.send_heartbeat(counter) >>>>> File "/home/www-data/web2py/gluon/scheduler.py", line 1041, in >>>>> send_heartbeat >>>>> db.rollback() >>>>> File "/home/www-data/web2py/gluon/packages/dal/pydal/base.py", line >>>>> 937, in rollback >>>>> self._adapter.rollback() >>>>> File >>>>> "/home/www-data/web2py/gluon/packages/dal/pydal/adapters/base.py", line >>>>> 1332, in rollback >>>>> return self.connection.rollback() >>>>> File "/home/www-data/web2py/gluon/contrib/pg8000/core.py", line >>>>> 1429, in rollback >>>>> self.execute(self._cursor, "rollback", None) >>>>> File "/home/www-data/web2py/gluon/contrib/pg8000/core.py", line >>>>> 1622, in execute >>>>> raise InterfaceError("connection is closed") >>>>> InterfaceError: connection is closed >>>>> >>>>> Traceback (most recent call last): >>>>> File "/home/www-data/web2py/gluon/shell.py", line 271, in run >>>>> exec(python_code, _env) >>>>> File "<string>", line 1, in <module> >>>>> File "/home/www-data/web2py/gluon/scheduler.py", line 730, in loop >>>>> self.wrapped_report_task(task, self.async(task)) >>>>> File "/home/www-data/web2py/gluon/scheduler.py", line 879, in >>>>> wrapped_report_task >>>>> db.rollback() >>>>> File "/home/www-data/web2py/gluon/packages/dal/pydal/base.py", line >>>>> 937, in rollback >>>>> self._adapter.rollback() >>>>> File >>>>> "/home/www-data/web2py/gluon/packages/dal/pydal/adapters/base.py", line >>>>> 1332, in rollback >>>>> return self.connection.rollback() >>>>> File "/home/www-data/web2py/gluon/contrib/pg8000/core.py", line >>>>> 1429, in rollback >>>>> self.execute(self._cursor, "rollback", None) >>>>> File "/home/www-data/web2py/gluon/contrib/pg8000/core.py", line >>>>> 1622, in execute >>>>> raise InterfaceError("connection is closed") >>>>> InterfaceError: connection is closed >>>>> >>>> -- Resources: - http://web2py.com - http://web2py.com/book (Documentation) - http://github.com/web2py/web2py (Source code) - https://code.google.com/p/web2py/issues/list (Report Issues) --- You received this message because you are subscribed to the Google Groups "web2py-users" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/d/optout.
