The branch, dharma has been updated
via 2753c8bd6f0f114518cc773cdd2813c5e2468b35 (commit)
from 4873b555c6bce15d4f4b995d4542ebb23d7b36db (commit)
- Log -----------------------------------------------------------------
http://xbmc.git.sourceforge.net/git/gitweb.cgi?p=xbmc/scripts;a=commit;h=2753c8bd6f0f114518cc773cdd2813c5e2468b35
commit 2753c8bd6f0f114518cc773cdd2813c5e2468b35
Author: amet <[email protected]>
Date: Tue Mar 13 22:32:11 2012 +0400
[script.game.whatthemovie] -v0.9.3
0.9.3
- Fix white shot (website changes)
0.9.2
- Multithreading background scraping of shots
- Now showing overall-score in addition to featurefilms-score
- now going back and forward through already scraped shots with 'back' and
'random'
- Code cleanups and improvements
- Removed the need for mechanize-lib
diff --git a/script.game.whatthemovie/addon.xml
b/script.game.whatthemovie/addon.xml
index 872c413..72d78ca 100644
--- a/script.game.whatthemovie/addon.xml
+++ b/script.game.whatthemovie/addon.xml
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<addon id="script.game.whatthemovie"
name="WhatTheMovie"
- version="0.9.1"
+ version="0.9.3"
provider-name="sphere, tadi, skiller2k1">
<requires>
<import addon="xbmc.python" version="1.0"/>
diff --git a/script.game.whatthemovie/changelog.txt
b/script.game.whatthemovie/changelog.txt
index e083085..16506ca 100644
--- a/script.game.whatthemovie/changelog.txt
+++ b/script.game.whatthemovie/changelog.txt
@@ -1,3 +1,13 @@
+0.9.3
+- Fix white shot (website changes)
+
+0.9.2
+- Multithreading background scraping of shots
+- Now showing overall-score in addition to featurefilms-score
+- now going back and forward through already scraped shots with 'back' and
'random'
+- Code cleanups and improvements
+- Removed the need for mechanize-lib
+
0.9.1
- Bugfix (error if player has exactly 1 point)
diff --git a/script.game.whatthemovie/resources/language/English/strings.xml
b/script.game.whatthemovie/resources/language/English/strings.xml
index 323b24c..2721585 100644
--- a/script.game.whatthemovie/resources/language/English/strings.xml
+++ b/script.game.whatthemovie/resources/language/English/strings.xml
@@ -25,7 +25,7 @@ Misc = 3300-3349
<string id="3102">Wrong: [B]%s[/B]</string>
<string id="3103">Not logged in</string>
<string id="3104">Logged in as user: [B]%s[/B]</string>
- <string id="3105">Your score: [B]%s[/B]</string>
+ <string id="3105">Your score: [B]%s[/B] ([B]%s[/B])</string>
<string id="3106">Solved [B][COLOR=ff008B00]%s[/COLOR][/B] times, first by
[B]%s[/B]</string>
<string id="3107">Not solved yet</string>
<string id="3108">Shot ID: [B]%s[/B]</string>
diff --git a/script.game.whatthemovie/resources/language/German/strings.xml
b/script.game.whatthemovie/resources/language/German/strings.xml
index 1bfb497..7055e68 100644
--- a/script.game.whatthemovie/resources/language/German/strings.xml
+++ b/script.game.whatthemovie/resources/language/German/strings.xml
@@ -25,7 +25,7 @@ Misc = 3300-3349
<string id="3102">Falsch: [B]%s[/B]</string>
<string id="3103">Nicht eingeloggt</string>
<string id="3104">Eingeloggt als: [B]%s[/B]</string>
- <string id="3105">Deine Punkte: [B]%s[/B]</string>
+ <string id="3105">Deine Punkte: [B]%s[/B] ([B]%s[/B])</string>
<string id="3106">[B][COLOR=ff008B00]%s[/COLOR][/B] mal gelöst. Erster:
[B]%s[/B]</string>
<string id="3107">Bisher ungelöst</string>
<string id="3108">Shot ID: [B]%s[/B]</string>
diff --git a/script.game.whatthemovie/resources/lib/gui.py
b/script.game.whatthemovie/resources/lib/gui.py
index 0c9a300..3d658dd 100644
--- a/script.game.whatthemovie/resources/lib/gui.py
+++ b/script.game.whatthemovie/resources/lib/gui.py
@@ -59,6 +59,7 @@ class GUI(xbmcgui.WindowXMLDialog):
CID_LIST_STARS = 1017
CID_PROGR_OWN_RATING = 1018
CID_GROUP_RATING = 1016
+ CID_LABEL_PRELOADS = 1020
# STRING_IDs
# Messages
@@ -140,6 +141,7 @@ class GUI(xbmcgui.WindowXMLDialog):
self.getString = self.Addon.getLocalizedString
self.getSetting = self.Addon.getSetting
self.setSetting = self.Addon.setSetting
+ self.createPaths()
# get controls
self.button_guess = self.getControl(self.CID_BUTTON_GUESS)
@@ -163,6 +165,7 @@ class GUI(xbmcgui.WindowXMLDialog):
self.group_rating = self.getControl(self.CID_GROUP_RATING)
self.progr_avg_rating = self.getControl(self.CID_PROGR_AVG_RATING)
self.progr_own_rating = self.getControl(self.CID_PROGR_OWN_RATING)
+ self.label_preloads = self.getControl(self.CID_LABEL_PRELOADS)
# set control visibility depending on xbmc-addon settings
self.hideLabels()
@@ -179,9 +182,11 @@ class GUI(xbmcgui.WindowXMLDialog):
user_agent = 'XBMC-ADDON - %s - V%s' % (self.ADDON_ID,
self.ADDON_VERSION)
self.Quiz = whatthemovie.WhatTheMovie(user_agent)
+ self.Quiz.setImagePath(xbmc.translatePath(self.cache_path))
# try to login and get first random shot. If it fails exit
try:
self.login()
+ self.Quiz.start(callback=self.updatePreload)
self.getShot('random')
except Exception, error:
self.errorMessage(self.getString(self.SID_ERROR_LOGIN),
@@ -274,6 +279,7 @@ class GUI(xbmcgui.WindowXMLDialog):
def closeDialog(self):
self.setWTMProperty('main_image', '')
+ self.Quiz.stop()
self.close()
def getShot(self, shot_request):
@@ -282,19 +288,17 @@ class GUI(xbmcgui.WindowXMLDialog):
self.setWTMProperty('busy', 'loading')
# hide label_status
self.setWTMProperty('solved_status', 'inactive')
- # scrape shot and download picture
+ # scrape shot
try:
self.shot = self.Quiz.getShot(shot_request)
shot = self.shot
- image_path = self.downloadPic(shot['image_url'],
- shot['shot_id'])
self.log('Got a shot: %s' % self.shot)
except Exception, error:
self.errorMessage(self.getString(self.SID_ERROR_SHOT),
str(error))
self.setWTMProperty('busy', '')
return
- self._showShotImage(image_path)
+ self._showShotImage(shot['image_url'])
self._showShotType(shot['shot_type'])
self._showShotPostedBy(shot['posted_by'])
self._showShotSolvedStatus(shot['solved'])
@@ -479,8 +483,14 @@ class GUI(xbmcgui.WindowXMLDialog):
self.setWTMProperty('sotd', '')
def _showUserScore(self, score):
- score_string = self.getString(self.SID_YOUR_SCORE) % str(score)
- self.label_score.setLabel(score_string)
+ ff_score_label = (self.getString(self.SID_YOUR_SCORE) \
+ % (score['ff_score'], score['all_score']))
+ self.label_score.setLabel(ff_score_label)
+
+ def updatePreload(self, num_preloads):
+ # fixme(anyone): This needs a better place ;-)
+ label = 'Preloads : %s' % num_preloads
+ self.label_preloads.setLabel(label)
def rateShot(self, shot_id, own_rating):
if self.logged_in:
@@ -544,7 +554,12 @@ class GUI(xbmcgui.WindowXMLDialog):
# clear solved_status
self.setWTMProperty('solved_status', 'inactive')
guess = keyboard.getText().decode('utf8')
- gives_point = (self.shot['gives_point']) # call by value forced
+ # We need to avoid pythons call by reference with gives_point.
+ # If answer is right the api will turn the values from gives_point
+ # to false to avoid double scoring. Now we use var "gave_point"
+ # which wont be automatically updated.
+ gave_point = {'ff': (self.shot['gives_point']['ff']),
+ 'all': (self.shot['gives_point']['all'])}
self.log('Try to check the title: %s' % guess)
# enter checking status
self.image_solution.setColorDiffuse('FFFFFF00')
@@ -562,7 +577,7 @@ class GUI(xbmcgui.WindowXMLDialog):
# call answerRight or answerWrong
if solution['is_right']:
self.answerRight(solution['title_year'],
- gives_point)
+ gave_point)
else:
self.answerWrong(guess)
@@ -572,12 +587,16 @@ class GUI(xbmcgui.WindowXMLDialog):
self.setWTMProperty('solved_status', 'correct')
self.image_solution.setColorDiffuse('FF00FF00')
# if this shot gives points, do so
- if gives_point:
- self.score += 1
+ if gives_point['ff'] or gives_point['all']:
+ if gives_point['all']:
+ self.score['all_score'] += 1
+ message = self.getString(self.SID_ANSWER_RIGHT) % title_year
+ if gives_point['ff']:
+ self.score['ff_score'] += 1
+ message = self.getString(self.SID_ANSWER_RIGHT_POINT) %
title_year
self._showUserScore(self.score)
- message = self.getString(self.SID_ANSWER_RIGHT_POINT) % title_year
else:
- message = self.getString(self.SID_ANSWER_RIGHT) % title_year
+ message = 'FIXME: Is this possible?'
self.label_message.setLabel(message)
# if user wants auto_jump, do so
if self.getSetting('auto_jump_enabled') == 'true':
@@ -601,15 +620,15 @@ class GUI(xbmcgui.WindowXMLDialog):
self.image_solution.setColorDiffuse('FFFF0000')
def login(self):
- self.score = 0
+ self.score = {'ff_score': 0,
+ 'all_score': 0}
self.logged_in = False
label = self.getString(self.SID_NOT_LOGGED_IN)
# if login is enabeld start loop until
# self.logged_in is true or user disables login
if self.getSetting('login') == 'true':
- cookie_dir = self.Addon.getAddonInfo('profile')
- self.checkCreatePath(cookie_dir)
- cookie_file = xbmc.translatePath('%s/cookie.txt' % cookie_dir)
+ cookie_file = xbmc.translatePath('%s/cookie.txt'
+ % self.profile_path)
# try to login until self.logged_in becomes True
while not self.logged_in:
if self.getSetting('login') == 'false':
@@ -631,7 +650,7 @@ class GUI(xbmcgui.WindowXMLDialog):
self.log('Login successfull via: %s' % self.logged_in)
# login successfully
label = self.getString(self.SID_LOGGED_IN_AS) % user
- self.score = int(self.Quiz.getScore(user)['ff_score'])
+ self.score = self.Quiz.getScore(user)
self.setRandomOptions()
self.label_loginstate.setLabel(label)
self._showUserScore(self.score)
@@ -658,18 +677,11 @@ class GUI(xbmcgui.WindowXMLDialog):
self.Quiz.setRandomOptions(options)
self.setSetting('already_sent_options', current_options)
- def downloadPic(self, image_url, shot_id):
- subst_image_url = 'http://static.whatthemovie.com/images/substitute'
- if not image_url.startswith(subst_image_url):
- cache_dir = ('special://profile/addon_data/%s/cache'
- % self.ADDON_ID)
- self.checkCreatePath(cache_dir)
- image_path = xbmc.translatePath('%s/%s.jpg' % (cache_dir, shot_id))
- if not os.path.isfile(image_path):
- self.Quiz.downloadFile(image_url, image_path)
- else:
- image_path = image_url
- return image_path
+ def createPaths(self):
+ self.profile_path = self.Addon.getAddonInfo('profile')
+ self.cache_path = self.profile_path + '/cache/'
+ self.checkCreatePath(self.profile_path)
+ self.checkCreatePath(self.cache_path)
def checkCreatePath(self, path):
result = False
diff --git a/script.game.whatthemovie/resources/lib/whatthemovie.py
b/script.game.whatthemovie/resources/lib/whatthemovie.py
index fa8526d..3608c5d 100644
--- a/script.game.whatthemovie/resources/lib/whatthemovie.py
+++ b/script.game.whatthemovie/resources/lib/whatthemovie.py
@@ -19,9 +19,12 @@
#
#
-import mechanize
import re
-from urllib import urlencode
+import urllib
+import urllib2
+import cookielib
+import threading
+import Queue
from BeautifulSoup import BeautifulSoup
@@ -35,7 +38,8 @@ class WhatTheMovie(object):
'self_posted': False,
'bookmarked': False,
'favourite': False,
- 'gives_point': True,
+ 'gives_point': {'ff': False,
+ 'all': False},
'already_solved': False,
'tags': [u'tag1'],
'posted_by': u'Hagentinho',
@@ -67,13 +71,15 @@ class WhatTheMovie(object):
def __init__(self, user_agent):
# Get browser stuff
- self.cookies = mechanize.LWPCookieJar()
- self.browser = mechanize.Browser()
- self.browser.set_cookiejar(self.cookies)
- self.browser.addheaders = [('user-agent', user_agent)]
+ self.cookies = cookielib.LWPCookieJar()
+ processor = urllib2.HTTPCookieProcessor(self.cookies)
+ self.opener = urllib2.build_opener(processor)
+ self.opener.addheaders = [('user-agent', user_agent)]
# Set empty returns
self.shot = dict()
self.last_shots = list()
+ self.image_download_path = None
+ self.running = False
def login(self, user, password, cookie_path):
logged_in = False
@@ -85,7 +91,7 @@ class WhatTheMovie(object):
except IOError:
cookie_found = False
if cookie_found:
- logged_in_user = self._getUsername(retrieve=True)
+ logged_in_user = self._getUsername()
if logged_in_user == user:
logged_in = 'cookie'
if not logged_in:
@@ -93,20 +99,41 @@ class WhatTheMovie(object):
data_dict = dict()
data_dict['name'] = user
data_dict['upassword'] = password
- data = urlencode(data_dict)
- req = mechanize.Request(login_url, data)
- self.browser.open(req)
+ data = urllib.urlencode(data_dict)
+ html = self.opener.open(login_url, data).read()
logged_in_user = self._getUsername()
if logged_in_user:
logged_in = 'auth'
self.cookies.save(cookie_path)
return logged_in
- def _getUsername(self, retrieve=False):
+ def setImagePath(self, image_download_path):
+ self.image_download_path = image_download_path
+
+ def start(self, num_workers=3, num_init_jobs=3, callback=None):
+ self.callback = callback
+ self.num_workers = num_workers
+ self.num_init_jobs = num_init_jobs
+ self.workers = [self.Scraper(self.opener, self.image_download_path,
+ self.callback) for i in range(self.num_workers)]
+ for worker in self.workers:
+ worker.start()
+ for job in range(self.num_init_jobs):
+ self.Scraper.jobs.put('random')
+ self.running = True
+
+ def stop(self):
+ self.Scraper.exit_requested = True
+ for i in range(self.num_workers):
+ self.Scraper.jobs.put('exit')
+ for worker in self.workers:
+ worker.join()
+ self.running = False
+
+ def _getUsername(self, html=None):
# only retrieve if there is no previous retrieve which we can use
- if retrieve:
- self.browser.open(self.MAIN_URL)
- html = self.browser.response().read()
+ if not html:
+ html = self.opener.open(self.MAIN_URL).read()
tree = BeautifulSoup(html)
section = tree.find('li', attrs={'class': 'secondary_nav',
'style': 'margin-left: 0'})
@@ -124,234 +151,76 @@ class WhatTheMovie(object):
def _sendAjaxReq(self, url, data_dict=None):
if data_dict:
- post_data = urlencode(data_dict)
+ post_data = urllib.urlencode(data_dict)
else:
post_data = ' '
- req = mechanize.Request(url, post_data)
+ req = urllib2.Request(url, post_data)
req.add_header('Accept', 'text/javascript, */*')
req.add_header('Content-Type',
'application/x-www-form-urlencoded; charset=UTF-8')
req.add_header('X-Requested-With', 'XMLHttpRequest')
- self.browser.open(req)
- response = self.browser.response().read()
+ response = self.opener.open(req).read()
response_c = response.replace('&', '&').decode('unicode-escape')
return response_c
def getShot(self, shot_request):
+ if not self.running:
+ self.start()
if self.OFFLINE_DEBUG:
return self.OFFLINE_SHOT
if shot_request == 'back':
if self.last_shots:
+ self.Scraper.next_shots_lock.acquire()
+ self.Scraper.next_shots.insert(0, self.shot)
+ if self.callback:
+ self.callback(len(self.Scraper.next_shots))
+ self.Scraper.next_shots_lock.release()
self.shot = self.last_shots.pop()
else:
if self.shot: # if there is already a shot - put it in list
self.last_shots.append(self.shot)
if shot_request.isdigit() or shot_request == 'random':
- self.shot = self.scrapeShot(shot_request)
+ pass
elif shot_request in self.shot['nav'].keys():
# fixme(sphere): replace with better logic
- # if there is no shot_request in dict
+ # if there is no matching val in nav
if not self.shot['nav'][shot_request]:
# check if it is a unsolved request and try without
if (shot_request[-9:] == '_unsolved'
and self.shot['nav'][shot_request[:-9]]):
request = shot_request[:-9]
- resolved_shot_request = self.shot['nav'][request]
- else:
- # else fallback to random
- resolved_shot_request = 'random'
+ shot_request = self.shot['nav'][request]
else:
- resolved_shot_request = self.shot['nav'][shot_request]
- self.shot = self.scrapeShot(resolved_shot_request)
- self.shot['requested_as'] = shot_request
- return self.shot
-
- def scrapeShot(self, shot_request):
- self.shot = dict()
- shot_url = '%s/shot/%s' % (self.MAIN_URL, shot_request)
- self.browser.open(shot_url)
- html = self.browser.response().read()
- tree = BeautifulSoup(html)
- # id
- shot_id = tree.find('li', attrs={'class': 'number'}).string.strip()
- # prev/next
- nav = dict()
- section = tree.find('ul', attrs={'id': 'nav_shots'}).findAll('li')
- nav_types = ((0, 'first'), (1, 'prev'), (2, 'prev_unsolved'),
- (4, 'next_unsolved'), (5, 'next'), (6, 'last'))
- for i, nav_type in nav_types:
- if section[i].a:
- nav[nav_type] = section[i].a['href'][6:]
- else:
- nav[nav_type] = None
- # image url
- image_url = tree.find('img', alt='guess this movie snapshot')['src']
- # languages
- lang_list = dict()
- lang_list['main'] = list()
- lang_list['hidden'] = list()
- section = tree.find('ul', attrs={'class': 'language_flags'})
- langs_main = section.findAll(lambda tag: len(tag.attrs) == 0)
- for lang in langs_main:
- if lang.img:
- lang_list['main'].append(lang.img['src'][-6:-4])
- langs_hidden = section.findAll('li',
- attrs={'class': 'hidden_languages'})
- for lang in langs_hidden:
- if lang.img:
- lang_list['hidden'].append(lang.img['src'][-6:-4])
- lang_list['all'] = lang_list['main'] + lang_list['hidden']
- # date
- shot_date = None
- section = tree.find('ul', attrs={'class': 'nav_date'})
- if section:
- r = ('<a href="/overview/(?P<year>[0-9]+)/'
- '(?P<month>[0-9]+)/(?P<day>[0-9]+)">')
- date_match = re.search(r, unicode(section))
- if date_match:
- date_dict = date_match.groupdict()
- if date_dict:
- shot_date = (int(date_dict['year']),
- int(date_dict['month']),
- int(date_dict['day']))
- # posted by
- sections = tree.find('ul',
- attrs={'class': 'nav_shotinfo'}).findAll('li')
- if sections[0].a:
- posted_by = sections[0].a.string
- else:
- posted_by = None
- # solved
- solved = dict()
- try:
- solved_string, solved_count = sections[1].string[8:].split()
- if solved_string == 'solved':
- solved['status'] = True
- solved['count'] = int(solved_count.strip('()'))
- except:
- solved['status'] = False
- solved['count'] = 0
- try:
- solved['first_by'] = sections[2].a.string
- except:
- solved['first_by'] = None
- # already solved + own_shot
- already_solved = False
- self_posted = False
- js_list = tree.findAll('script',
- attrs={'type': 'text/javascript'},
- text=re.compile('guess_problem'))
- if js_list:
- message = str(js_list)
- if re.search('already solved', message):
- already_solved = True
- elif re.search('You posted this', message):
- self_posted = True
- # voting
- voting = dict()
- section = tree.find('script',
- attrs={'type': 'text/javascript'},
- text=re.compile('tt_shot_rating_stars'))
- r = ('<strong>(?P<overall_rating>[0-9.]+|hidden)</strong> '
- '\((?P<votes>[0-9]+) votes\)'
- '(<br>Your rating: <strong>(?P<own_rating>[0-9.]+)</strong>)?')
- if section:
- voting = re.search(r, section).groupdict()
- # tags
- tags = list()
- tags_list = tree.find('ul', attrs={'id':
- 'shot_tag_list'}).findAll('li')
- for tag in tags_list:
- if tag.a:
- tags.append(tag.a.string)
- # shot_type
- section = tree.find('h2', attrs={'class':
- 'topbar_title'}).string.strip()
- shot_type = 0 # Unknown
- if section == 'New Submissions':
- shot_type = 1
- elif section == 'Feature Films':
- shot_type = 2
- elif section == 'The Archive':
- shot_type = 3
- elif section == 'Rejected Snapshots':
- shot_type = 4
- elif section == 'The Vault':
- shot_type = 5
- elif section == 'Deleted':
- shot_type = 6
- # gives_point
- gives_point = False
- if shot_type == 2 and not already_solved and not self_posted:
- gives_point = True
- # bookmarked
- if tree.find('li', attrs={'id': 'watchbutton'}):
- bookmark_link = tree.find('li', attrs={'id': 'watchbutton'}).a
- try:
- if bookmark_link['class'] == 'active':
- bookmarked = True
- except KeyError:
- bookmarked = False
- else:
- bookmarked = None # Not logged in
- # favourite
- if tree.find('li', attrs={'class': 'love'}):
- favourite_link = tree.find('li', attrs={'class': 'love'}).a
- try:
- if favourite_link['class'] == 'active':
- favourite = True
- except KeyError:
- favourite = False
- else:
- favourite = None # Not logged in
- # Snapshot of the Day
- sotd = False
- if tree.find('div', attrs={'class': 'sotd_banner'}):
- sotd = True
- # Solvable
- solvable = False
- section = tree.find('li', attrs={'id': 'solutionbutton'})
- if section is not None:
- try:
- if section.a['class'] == 'inactive':
- solvable = False
- except KeyError:
- solvable = True
- # redirected
- redirected = False
- section = tree.find('div', attrs={'class':
- re.compile('flash_message')})
- if section:
- message = section.string
- redirected = 1
- if re.search('you are not allowed', message):
- redirected = 2
- if re.search('No such snapshot', message):
- redirected = 3
- # create return dict
- self.shot['shot_id'] = shot_id
- self.shot['image_url'] = image_url
- self.shot['lang_list'] = lang_list
- self.shot['posted_by'] = posted_by
- self.shot['solved'] = solved
- self.shot['date'] = shot_date
- self.shot['already_solved'] = already_solved
- self.shot['self_posted'] = self_posted
- self.shot['voting'] = voting
- self.shot['tags'] = tags
- self.shot['shot_type'] = shot_type
- self.shot['gives_point'] = gives_point
- self.shot['nav'] = nav
- self.shot['bookmarked'] = bookmarked
- self.shot['favourite'] = favourite
- self.shot['sotd'] = sotd
- self.shot['solvable'] = solvable
- self.shot['redirected'] = redirected
+ shot_request = self.shot['nav'][shot_request]
+ if len(self.Scraper.next_shots) - 1 < self.num_init_jobs:
+ self.Scraper.jobs.put(shot_request)
+ # delete actual shot because we want a new one
+ self.shot = None
+ self.Scraper.new_shot_condition.acquire()
+ # as long as we dont have a shot which we want (ex. 'random')
+ while not self.shot:
+ # lock the list of preloaded shots
+ self.Scraper.next_shots_lock.acquire()
+ # search in already preloaded shots for one we want
+ for i, shot in enumerate(self.Scraper.next_shots):
+ # if this is a shot we want
+ if shot['requested_as'] == shot_request:
+ # save the shot we want and delete from list
+ self.shot = self.Scraper.next_shots.pop(i)
+ if self.callback:
+ self.callback(len(self.Scraper.next_shots))
+ # stop searching in the list of preloaded shots
+ break
+ # relase the lock - new shots can now be inserted from workers
+ self.Scraper.next_shots_lock.release()
+ # if our search was successfull leave the waiting state
+ if self.shot:
+ break
+ # there was no shot we want - wait for a new
+ self.Scraper.new_shot_condition.wait()
+ self.Scraper.new_shot_condition.release()
return self.shot
- def downloadFile(self, url, local_path):
- self.browser.retrieve(url, local_path, )
-
def guessShot(self, shot_id, title_guess):
if self.OFFLINE_DEBUG:
if title_guess.lower() == self.OFFLINE_ANSWER['title'].lower():
@@ -370,8 +239,8 @@ class WhatTheMovie(object):
if self.shot['shot_id'] == shot_id:
if not self.shot['already_solved']:
self.shot['already_solved'] = True
- if self.shot['gives_point']:
- self.shot['gives_point'] = False
+ self.shot['gives_point'] = {'ff': False,
+ 'all': False}
return answer
def rateShot(self, shot_id, user_rate, rerated='false'):
@@ -424,17 +293,254 @@ class WhatTheMovie(object):
return solved_title
def getScore(self, username):
+ score = {'ff_score': 0,
+ 'all_score': 0}
if self.OFFLINE_DEBUG:
- score = {'ff_score': 0,
- 'all_score': 0}
return score
- score = 0
profile_url = '%s/user/%s/' % (self.MAIN_URL, username)
- self.browser.open(profile_url)
- html = self.browser.response().read()
+ html = self.opener.open(profile_url).read()
tree = BeautifulSoup(html)
box = tree.find('div', attrs={'class': 'box_white'})
r = ('>(?P<ff_score>[0-9]+) Feature Film.*'
'>(?P<all_score>[0-9]+) Snapshot')
- score = re.search(r, str(box.p)).groupdict()
+ if re.search(r, str(box.p)):
+ score_dict = re.search(r, str(box.p)).groupdict()
+ score = {'ff_score': int(score_dict['ff_score']),
+ 'all_score': int(score_dict['all_score'])}
return score
+
+ class Scraper(threading.Thread):
+
+ jobs = Queue.Queue()
+ next_shots = list()
+ next_shots_lock = threading.Lock()
+ new_shot_condition = threading.Condition()
+ exit_requested = False
+
+ def __init__(self, opener, image_download_path, callback=None):
+ self.opener = opener
+ self.callback = callback
+ self.image_download_path = image_download_path
+ threading.Thread.__init__(self)
+
+ def run(self):
+ while not self.exit_requested:
+ job = WhatTheMovie.Scraper.jobs.get()
+ if job == 'exit':
+ break
+ # scrape the shot - this will take some time
+ shot = self.scrapeShot(job)
+ # lock the list of shots
+ WhatTheMovie.Scraper.next_shots_lock.acquire()
+ # save the shot
+ WhatTheMovie.Scraper.next_shots.append(shot)
+ # tell that a new shot was inserted
+ WhatTheMovie.Scraper.new_shot_condition.acquire()
+ WhatTheMovie.Scraper.new_shot_condition.notify()
+ WhatTheMovie.Scraper.new_shot_condition.release()
+ if self.callback:
+ self.callback(len(WhatTheMovie.Scraper.next_shots))
+ WhatTheMovie.Scraper.next_shots_lock.release()
+ try: # python >= 2.5 needed for task_done
+ WhatTheMovie.Scraper.jobs.task_done()
+ except AttributeError:
+ pass
+
+ def scrapeShot(self, shot_request):
+ self.shot = dict()
+ shot_url = '%s/shot/%s' % (WhatTheMovie.MAIN_URL, shot_request)
+ html = self.opener.open(shot_url).read()
+ tree = BeautifulSoup(html)
+ # id
+ shot_id = tree.find('li', attrs={'class': 'number'}).string.strip()
+ # prev/next
+ nav = dict()
+ section = tree.find('ul', attrs={'id': 'nav_shots'}).findAll('li')
+ nav_types = ((0, 'first'), (1, 'prev'), (2, 'prev_unsolved'),
+ (4, 'next_unsolved'), (5, 'next'), (6, 'last'))
+ for i, nav_type in nav_types:
+ if section[i].a:
+ nav[nav_type] = section[i].a['href'][6:]
+ else:
+ nav[nav_type] = None
+ # image url
+ section = tree.find('style', {'type': 'text/css'})
+ image_url = ''
+ if section:
+ r_img = re.compile('background-image:url\("(.+?)"\)')
+ m_img = re.search(r_img, section.string)
+ if m_img:
+ image_url = WhatTheMovie.MAIN_URL + m_img.group(1)
+ subst_image_url = 'http://static.whatthemovie.com/images/subst'
+ if self.image_download_path:
+ if not image_url.startswith(subst_image_url):
+ local_image_file = '%s%s.jpg' % (self.image_download_path,
+ shot_id)
+ urllib.urlretrieve(image_url, local_image_file, )
+ image_url = local_image_file
+ # languages
+ lang_list = dict()
+ lang_list['main'] = list()
+ lang_list['hidden'] = list()
+ section = tree.find('ul', attrs={'class': 'language_flags'})
+ langs_main = section.findAll(lambda tag: len(tag.attrs) == 0)
+ for lang in langs_main:
+ if lang.img:
+ lang_list['main'].append(lang.img['src'][-6:-4])
+ langs_hidden = section.findAll('li',
+ attrs={'class': 'hidden_languages'})
+ for lang in langs_hidden:
+ if lang.img:
+ lang_list['hidden'].append(lang.img['src'][-6:-4])
+ lang_list['all'] = lang_list['main'] + lang_list['hidden']
+ # date
+ shot_date = None
+ section = tree.find('ul', attrs={'class': 'nav_date'})
+ if section:
+ r = ('<a href="/overview/(?P<year>[0-9]+)/'
+ '(?P<month>[0-9]+)/(?P<day>[0-9]+)">')
+ date_match = re.search(r, unicode(section))
+ if date_match:
+ date_dict = date_match.groupdict()
+ if date_dict:
+ shot_date = (int(date_dict['year']),
+ int(date_dict['month']),
+ int(date_dict['day']))
+ # posted by
+ sections = tree.find('ul',
+ attrs={'class': 'nav_shotinfo'}).findAll('li')
+ if sections[0].a:
+ posted_by = sections[0].a.string
+ else:
+ posted_by = None
+ # solved
+ solved = dict()
+ try:
+ solved_string, solved_count = sections[1].string[8:].split()
+ if solved_string == 'solved':
+ solved['status'] = True
+ solved['count'] = int(solved_count.strip('()'))
+ except:
+ solved['status'] = False
+ solved['count'] = 0
+ try:
+ solved['first_by'] = sections[2].a.string
+ except:
+ solved['first_by'] = None
+ # already solved + own_shot
+ already_solved = False
+ self_posted = False
+ js_list = tree.findAll('script',
+ attrs={'type': 'text/javascript'},
+ text=re.compile('guess_problem'))
+ if js_list:
+ message = str(js_list)
+ if re.search('already solved', message):
+ already_solved = True
+ elif re.search('You posted this', message):
+ self_posted = True
+ # voting
+ voting = dict()
+ section = tree.find('script',
+ attrs={'type': 'text/javascript'},
+ text=re.compile('tt_shot_rating_stars'))
+ r = ('<strong>(?P<overall_rating>[0-9.]+|hidden)</strong> '
+ '\((?P<votes>[0-9]+) votes\)'
+ '(<br>Your rating: <strong>(?P<own_rating>[0-9.]+)</strong)?')
+ if section:
+ voting = re.search(r, section).groupdict()
+ # tags
+ tags = list()
+ tags_list = tree.find('ul', attrs={'id':
+ 'shot_tag_list'}).findAll('li')
+ for tag in tags_list:
+ if tag.a:
+ tags.append(tag.a.string)
+ # shot_type
+ section = tree.find('h2', attrs={'class':
+ 'topbar_title'}).string.strip()
+ shot_type = 0 # Unknown
+ if section == 'New Submissions':
+ shot_type = 1
+ elif section == 'Feature Films':
+ shot_type = 2
+ elif section == 'The Archive':
+ shot_type = 3
+ elif section == 'Rejected Snapshots':
+ shot_type = 4
+ elif section == 'The Vault':
+ shot_type = 5
+ elif section == 'Deleted':
+ shot_type = 6
+ # gives_point
+ gives_point = {'ff': False,
+ 'all': False}
+ if not already_solved and not self_posted:
+ gives_point['all'] = True
+ if shot_type == 2:
+ gives_point['ff'] = True
+ # bookmarked
+ if tree.find('li', attrs={'id': 'watchbutton'}):
+ bookmark_link = tree.find('li', attrs={'id': 'watchbutton'}).a
+ try:
+ if bookmark_link['class'] == 'active':
+ bookmarked = True
+ except KeyError:
+ bookmarked = False
+ else:
+ bookmarked = None # Not logged in
+ # favourite
+ if tree.find('li', attrs={'class': 'love'}):
+ favourite_link = tree.find('li', attrs={'class': 'love'}).a
+ try:
+ if favourite_link['class'] == 'active':
+ favourite = True
+ except KeyError:
+ favourite = False
+ else:
+ favourite = None # Not logged in
+ # Snapshot of the Day
+ sotd = False
+ if tree.find('div', attrs={'class': 'sotd_banner'}):
+ sotd = True
+ # Solvable
+ solvable = False
+ section = tree.find('li', attrs={'id': 'solutionbutton'})
+ if section is not None:
+ try:
+ if section.a['class'] == 'inactive':
+ solvable = False
+ except KeyError:
+ solvable = True
+ # redirected
+ redirected = False
+ section = tree.find('div', attrs={'class':
+ re.compile('flash_message')})
+ if section:
+ message = section.string
+ redirected = 1
+ if re.search('you are not allowed', message):
+ redirected = 2
+ if re.search('No such snapshot', message):
+ redirected = 3
+ # create return dict
+ self.shot['shot_id'] = shot_id
+ self.shot['image_url'] = image_url
+ self.shot['lang_list'] = lang_list
+ self.shot['posted_by'] = posted_by
+ self.shot['solved'] = solved
+ self.shot['date'] = shot_date
+ self.shot['already_solved'] = already_solved
+ self.shot['self_posted'] = self_posted
+ self.shot['voting'] = voting
+ self.shot['tags'] = tags
+ self.shot['shot_type'] = shot_type
+ self.shot['gives_point'] = gives_point
+ self.shot['nav'] = nav
+ self.shot['bookmarked'] = bookmarked
+ self.shot['favourite'] = favourite
+ self.shot['sotd'] = sotd
+ self.shot['solvable'] = solvable
+ self.shot['redirected'] = redirected
+ self.shot['requested_as'] = shot_request
+ return self.shot
diff --git
a/script.game.whatthemovie/resources/skins/default/720p/script-WhatTheMovie-main.xml
b/script.game.whatthemovie/resources/skins/default/720p/script-WhatTheMovie-main.xml
index 75c55da..3db33b1 100644
---
a/script.game.whatthemovie/resources/skins/default/720p/script-WhatTheMovie-main.xml
+++
b/script.game.whatthemovie/resources/skins/default/720p/script-WhatTheMovie-main.xml
@@ -215,7 +215,7 @@
<ondown>3003</ondown>
</control>
</control>
- <control type="label" id="1001">
+ <control type="label" id="1001">
<description>Login state label</description>
<posx>20</posx>
<posy>12</posy>
@@ -226,6 +226,17 @@
<shadowcolor>black</shadowcolor>
<visible>true</visible>
</control>
+ <control type="label" id="1020">
+ <description>Preload label</description>
+ <posx>300</posx>
+ <posy>12</posy>
+ <width>450</width>
+ <align>left</align>
+ <font>font12</font>
+ <textcolor>white</textcolor>
+ <shadowcolor>black</shadowcolor>
+ <visible>true</visible>
+ </control>
<control type="label" id="1004">
<description>Posted by label</description>
<posx>280</posx>
-----------------------------------------------------------------------
Summary of changes:
script.game.whatthemovie/addon.xml | 2 +-
script.game.whatthemovie/changelog.txt | 10 +
.../resources/language/English/strings.xml | 2 +-
.../resources/language/German/strings.xml | 2 +-
script.game.whatthemovie/resources/lib/gui.py | 70 +-
.../resources/lib/mechanize/__init__.py | 211 --
.../resources/lib/mechanize/_auth.py | 68 -
.../resources/lib/mechanize/_beautifulsoup.py | 1077 -------
.../resources/lib/mechanize/_clientcookie.py | 1725 ----------
.../resources/lib/mechanize/_debug.py | 28 -
.../resources/lib/mechanize/_firefox3cookiejar.py | 248 --
.../resources/lib/mechanize/_form.py | 3280 --------------------
.../resources/lib/mechanize/_gzip.py | 105 -
.../resources/lib/mechanize/_headersutil.py | 241 --
.../resources/lib/mechanize/_html.py | 629 ----
.../resources/lib/mechanize/_http.py | 447 ---
.../resources/lib/mechanize/_lwpcookiejar.py | 185 --
.../resources/lib/mechanize/_markupbase.py | 393 ---
.../resources/lib/mechanize/_mechanize.py | 669 ----
.../resources/lib/mechanize/_mozillacookiejar.py | 161 -
.../resources/lib/mechanize/_msiecookiejar.py | 388 ---
.../resources/lib/mechanize/_opener.py | 442 ---
.../resources/lib/mechanize/_pullparser.py | 391 ---
.../resources/lib/mechanize/_request.py | 40 -
.../resources/lib/mechanize/_response.py | 525 ----
.../resources/lib/mechanize/_rfc3986.py | 245 --
.../resources/lib/mechanize/_sgmllib_copy.py | 559 ----
.../resources/lib/mechanize/_sockettimeout.py | 6 -
.../resources/lib/mechanize/_testcase.py | 162 -
.../resources/lib/mechanize/_urllib2.py | 50 -
.../resources/lib/mechanize/_urllib2_fork.py | 1414 ---------
.../resources/lib/mechanize/_useragent.py | 367 ---
.../resources/lib/mechanize/_util.py | 305 --
.../resources/lib/mechanize/_version.py | 2 -
.../resources/lib/whatthemovie.py | 550 ++--
.../default/720p/script-WhatTheMovie-main.xml | 13 +-
36 files changed, 394 insertions(+), 14618 deletions(-)
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/__init__.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_auth.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_beautifulsoup.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_clientcookie.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_debug.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_firefox3cookiejar.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_form.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_gzip.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_headersutil.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_html.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_http.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_lwpcookiejar.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_markupbase.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_mechanize.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_mozillacookiejar.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_msiecookiejar.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_opener.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_pullparser.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_request.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_response.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_rfc3986.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_sgmllib_copy.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_sockettimeout.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_testcase.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_urllib2.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_urllib2_fork.py
delete mode 100644
script.game.whatthemovie/resources/lib/mechanize/_useragent.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_util.py
delete mode 100644 script.game.whatthemovie/resources/lib/mechanize/_version.py
hooks/post-receive
--
Scripts
------------------------------------------------------------------------------
Keep Your Developer Skills Current with LearnDevNow!
The most comprehensive online learning library for Microsoft developers
is just $99.99! Visual Studio, SharePoint, SQL - plus HTML5, CSS3, MVC3,
Metro Style Apps, more. Free future releases when you subscribe now!
http://p.sf.net/sfu/learndevnow-d2d
_______________________________________________
Xbmc-addons mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/xbmc-addons
