Hi there, I am new to web2py and am trying to initiate a scraper script I have written in python when opening a view in web2py. The script extracts information from our internal intranet site and stores the data in a sqlite database. I have used beautifulsoup for the parsing. Any ideas how I can best implement this?
many thanks! John Enter codimport sqlite3 from bs4 import BeautifulSoup from urllib2 import urlopen #Py 3: use urllib.request from datetime import datetime, timedelta myAddress = "http://xx.xx.xxx.58/xxx/schedule/CRY.html"; htmlPage = urlopen(myAddress) htmlText = htmlPage.read() #Py 3: decode mySoup = BeautifulSoup(htmlText) table = mySoup.find("table", width="100%") with sqlite3.connect('estrack_cs.db') as connection: c = connection.cursor() c.execute("DROP TABLE IF EXISTS nextpasses") c.execute("CREATE TABLE nextpasses(DOY INT,STATION TEXT,BOT DATETIME, EOT DATETIME)") for row in table.find_all('tr'): col = row.find_all('td') if col[0].string.strip()<>"StrtS DATE": StrtDate = col[0].string.strip() DOY = col[1].string.strip() STATION = col[2].string.strip() StrtS = datetime.strptime(col[0].string.strip() + " " + col[3].string.strip(),'%Y-%m-%d %H:%M:%S') BOT = datetime.strptime(col[0].string.strip() + " " + col[4].string.strip(),'%Y-%m-%d %H:%M:%S') EOT = datetime.strptime(col[0].string.strip() + " " + col[5].string.strip(),'%Y-%m-%d %H:%M:%S') StopS = col[6].string.strip() record = (DOY,STATION,BOT.strftime('%Y-%m-%d %H:%M:%S'),EOT.strftime('%Y-%m-%d %H:%M:%S')) if DOY[3:4]=="-" and (StrtS > BOT) : print "StrtD:" + StrtS.strftime('%Y-%m-%d %H:%M:%S') + "; BOT:" + BOT.strftime('%%Y-%m-%d %H:%M:%S') new_DOY = DOY[4:] new_BOT = BOT + timedelta(days=1) new_EOT = EOT + timedelta(days=1) record = (new_DOY,STATION,new_BOT.strftime('%Y-%m-%d %H:%M:%S'),new_EOT.strftime('%Y-%m-%d %H:%M:%S')) elif DOY[3:4]=="-" and (BOT > EOT) : new_EOT = EOT + timedelta(days=1) record = (DOY,STATION,BOT.strftime('%Y-%m-%d %H:%M:%S'),new_EOT.strftime('%Y-%m-%d %H:%M:%S')) c.execute("INSERT INTO nextpasses(DOY,STATION,BOT,EOT) VALUES(?,?,?,?);", record) -- --- You received this message because you are subscribed to the Google Groups "web2py-users" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. For more options, visit https://groups.google.com/groups/opt_out.
