On Jul 02, gourgi <[EMAIL PROTECTED]> wrote:
> > It's also possible that something has changed in videoDB: my local
> > copy can be a bit old: I'll update from the CVS later.
> most likely yes , something has changed
> my videodb version is 3.0.1
Noticed; I've fixed it in the CVS and in the attached vdbpy_add.py
script.
As usual, let me know if it works.
Thank you very much for your reports!
--
Davide Alberani <[EMAIL PROTECTED]> [PGP KeyID: 0x465BFD47]
http://erlug.linux.it/~da/
#!/usr/bin/env python
"""vdbpy_add.py
Insert or update videoDB entries with data from IMDb.
"""
import os, sys, getopt, urllib, ConfigParser
from types import UnicodeType, ListType, TupleType
try:
import MySQLdb
import _mysql_exceptions
except ImportError:
print 'Unable to import the MySQLdb module!'
sys.exit(1)
try:
from imdb import IMDb, IMDbError
except ImportError:
print 'Unable to import the IMDbPY package!'
sys.exit(1)
help = """vdbpy_add.py usage:
%s [-o] [-u vdbID] [-f file] imdbID [key1=value1, key2=value2, ...]
-o override values.
-u vdbID updates the videoDB database entry with vdbID ID.
-f file select an alternate configuration file.
-t don't modify the database.
-h show this help and exit.
Example:
vdbpy_add.py 0112573 diskid="0361" subtitle="cuore impavido" seen=1
""" % sys.argv[0]
# Manage arguments list.
try:
optlist, args = getopt.getopt(sys.argv[1:], 'u:f:oth', ['help'])
except getopt.error, e:
print 'Troubles with arguments.'
print HELP
sys.exit(2)
if len(args) < 1:
print 'You must supply at least the imdbID!'
print help
sys.exit(2)
# If set, we have to update the entry with the UPDATE_ID id.
UPDATE = 0
UPDATE_ID = None
# The name of the configuration file.
CONF_FILE = ''
TEST = 0
# Configure the IMDb instance.
IMDB_AS = 'http'
IMDB_PARAMS = []
IMDB_KWDS = {}
# For the userseen table.
VDB_USER = None
VDB_UID = None
USER_SEEN = None
confset = { # Parameters to connect to the database.
'db_server': '',
'db_password': '',
'db_database': '',
'db_user': '',
'db_charset': 'latin1',
# If set, data from the config file override the data
# already in the database.
'override': 0,
# Fetch images?
'getCover': 1,
'getHeadShots': 0}
for opt in optlist:
if opt[0] == '-u':
UPDATE = 1
UPDATE_ID = opt[1]
elif opt[0] == '-f':
CONF_FILE = opt[1]
elif opt[0] == '-t':
TEST = 1
print 'Test mode. Do not modify the database.'
elif opt[0] in ('-h', '--help'):
print help
sys.exit(0)
# The imdb ID.
imdbID = args[0]
# Everything else on the command line.
args = args[1:]
if UPDATE:
print 'Updating database entry "%s" with movie "%s".' % \
(UPDATE_ID, imdbID)
else:
print 'Inserting movie "%s".' % imdbID
# Read and parse the configuration file.
conffile = ConfigParser.ConfigParser()
if not CONF_FILE:
# ./vdbpyrc
CONF_FILE = ['vdbpyrc']
# /etc/vdbpyrc
CONF_FILE.append(os.path.join(os.sep, 'etc', 'vdbpyrc'))
if os.name != 'posix':
# C:\\Python24\etc\imdbpyweb.conf
CONF_FILE.append(os.path.join(sys.prefix, 'etc', 'vdbpyrc'))
# ~/.vdbpyrc
CONF_FILE.append(os.path.join(os.path.expanduser('~'), '.vdbpyrc'))
try: conffile.read(CONF_FILE)
except ConfigParser.Error, e:
print 'Unable to parse configuration file "%s".' % CONF_FILE
print str(e)
sys.exit(3)
if not conffile.has_section('config'):
print 'Section [config] not present in the configuration file.'
sys.exit(3)
for key in ('db_server', 'db_user', 'db_password', 'db_database'):
if not conffile.has_option('config', key):
print 'Required option "%s" not present in the [config] section.'
sys.exit(3)
confset[key] = conffile.get('config', key)
# Special cases (boolean value), and the switch in the command line.
for key in ('override', 'getCover', 'getHeadShots'):
if not conffile.has_option('config', key):
print 'Required option "%s" not present in the [config] section.'
sys.exit(3)
confset[key] = conffile.getboolean('config', key)
if [x for x in optlist if x[0] == '-o']: confset['override'] = 1
if conffile.has_option('config', 'imdbpy_as'):
the_as = conffile.get('config', 'imdbpy_as')
if the_as: IMDB_AS = the_as
if conffile.has_option('config', 'imdbpy_params'):
the_params = conffile.get('config', 'imdbpy_params')
if the_params:
for x in the_params.split(','):
if x.find('=') != -1:
k, v = x.split('=')
k, v = k.strip(), v.strip()
if k and v: IMDB_KWDS[k] = v
else:
x = x.strip()
if x: IMDB_PARAMS.append(x)
if conffile.has_option('config', 'vdb_user'):
VDB_USER = conffile.get('config', 'vdb_user') or None
# ------- CONNECT TO THE DATABASE.
def _(s):
if isinstance(s, (ListType, TupleType)):
s = [_(i) for i in s]
return s
print 'Connecting to the VideoDB database... ',
sys.stdout.flush()
try:
db = MySQLdb.connect(db=confset['db_database'], host=confset['db_server'],
user=confset['db_user'], passwd=confset['db_password'],
charset=confset['db_charset'])
curs = db.cursor()
except MySQLdb.Error, e:
print '\nUnable to connect to the database.'
print str(e)
sys.exit(4)
##try: curs.execute('SET NAMES "latin1";')
##except _mysql_exceptions.MySQLError: pass
print 'done!'
# Collect information about the videodata table.
print 'Collecting information about the VideoDB database... ',
sys.stdout.flush()
curs.execute('SHOW COLUMNS FROM videodata;')
columns = [col[0] for col in curs.fetchall()]
# Columns in the videogenre table.
curs.execute('SHOW COLUMNS FROM videogenre;')
vg_cols = [col[0] for col in curs.fetchall()]
if 'video_id' in vg_cols: vg_cols[:] = ['video_id', 'genre_id']
else: vg_cols[:] = ['id', 'gid']
curs.execute('SHOW COLUMNS FROM userseen;')
us_cols = [col[0] for col in curs.fetchall()]
if 'user_id' in us_cols: us_cols[:] = ['user_id', 'video_id']
else: us_cols[:] = ['user', 'video_id']
# The videodata_set dictionary, with every info about the movie.
if not UPDATE:
# Inserting new data; set everything to None by default.
videodata_set = dict([(col, None) for col in columns])
else:
# Updating an existing entry; collect information already
# in the database.
if not curs.execute("SELECT * from videodata WHERE id = %s;", (UPDATE_ID,)):
print '\nUnable to select movie ID "%s".' % UPDATE_ID
sys.exit(5)
videodata_set = dict([(col, val) for col, val
in zip(columns, curs.fetchone())])
# Remove the 'id' column.
if videodata_set.has_key('id'): del videodata_set['id']
# Get mediatype ids.
curs.execute("SELECT name, id FROM mediatypes;")
mediatypes = dict([(col[0].lower(), col[1]) for col in curs.fetchall()])
# Get genre ids.
curs.execute("SELECT name, id FROM genres;")
gen_dict = dict([(col[0].lower(), col[1]) for col in curs.fetchall()])
# User id.
if VDB_USER is not None:
if curs.execute("SELECT id FROM users WHERE name = %s;", (VDB_USER,)) == 1:
VDB_UID = curs.fetchone()[0]
# There was no "Music" in VideoDB 1.x. XXX: should it be set to 12?
if not gen_dict.has_key('music'): gen_dict['music'] = 21
print 'done!'
# If the configuration file has a [videodata] section,
# use these values as defaults.
if conffile.has_section('videodata'):
updata = [(key, conffile.get('videodata', key))
for key in conffile.options('videodata')
if key in videodata_set]
if not confset['override']:
updata[:] = [x for x in updata if not videodata_set.get(x[0])]
videodata_set.update(dict(updata))
# Userseen.
if conffile.has_option('videodata', 'userseen'):
USER_SEEN = conffile.getboolean('videodata', 'userseen')
# Get the defaults from the userconfig table of the VideoDB database.
if VDB_UID is not None:
for defkey, key in (('mediadefault', 'mediatype'),
('langdefault', 'language')):
if not curs.execute("SELECT value FROM userconfig WHERE opt = %s " +
"AND " + us_cols[0] + "= %s;", (defkey, VDB_UID)):
continue
defval = curs.fetchone()[0]
# These defaults never override values already set.
if defval and not videodata_set.get(key):
videodata_set[key] = defval
# Get the defaults from the config table of the VideoDB database.
for defkey, key in (('mediadefault', 'mediatype'),
('langdefault', 'language')):
curs.execute("SELECT value FROM config WHERE opt = %s;", (defkey,))
defval = curs.fetchone()[0]
# These defaults never override values already set.
if defval and not videodata_set.get(key):
videodata_set[key] = defval
# ------- CONNECT TO IMDB AND GET MOVIE INFORMATION.
try:
imdbconn = IMDb(IMDB_AS, *IMDB_PARAMS, **IMDB_KWDS)
except IMDbError, e:
print '\nError instancing the IMDb class.'
print str(e)
sys.exit(6)
print 'Fetching data from IMDb... ',
sys.stdout.flush()
try:
movie = imdbconn.get_movie(imdbID)
except IMDbError, e:
print '\nError fetching data from IMDb.'
print str(e)
sys.exit(6)
print 'done!'
# ------- COLLECT INFORMATION FROM THE MOVIE OBJECT.
title = _(movie.get('canonical title', ''))
subtitle = ''
genres = _(movie.get('genres', []))
language = _(movie.get('language', []))
year = _(movie.get('year', '????'))
director = movie.get('director', [])
coverURL = _(movie.get('cover url', ''))
actors_list = movie.get('cast', [])
runtime = _(movie.get('runtime', []))
country = _(movie.get('country', []))
plot = _(movie.get('plot', []))
mpaa = _(movie.get('mpaa', ''))
rating = _(movie.get('rating', ''))
kind = _(movie.get('kind', ''))
certificates = _(movie.get('certificates'))
episodes = movie.get('number of episodes')
comment = ''
# Some basic transformation.
if title:
tsplit = title.split(' - ')
if len(tsplit) > 1:
title = tsplit[0]
subtitle = ' - '.join(tsplit[1:])
if language: language = language[0].lower()
if director: director = ', '.join([_(unicode(x)) for x in director[:2]])
actors = ''
tmp_actlist = []
for act in actors_list:
actline = _(act.get('name', u''))
if act.currentRole: actline += '::%s' % _(unicode(act.currentRole))
if act.notes: actline += ' %s' % _(act.notes)
tmp_actlist.append(actline)
if actors_list: actors = '\n'.join(tmp_actlist)
if runtime:
runtime = runtime[0]
runtime = runtime.split()[0]
i = runtime.find(':')
if i != -1: runtime = runtime[i+1:]
##if episodes is None and kind == 'episode':
## imdbconn.update(movie, 'episodes')
## from imdb.helpers import sortedEpisodes
## episodes = len(sortedEpisodes(movie))
if episodes:
try:
episodesnr = int(episodes)
intrunt = int(runtime)
comment = '%s episodes (%s minutes each one)' % (episodes, runtime)
runtime = episodesnr * intrunt
except (ValueError, TypeError, OverflowError): pass
if country: country = ', '.join(country)
if plot:
plot = plot[0]
i = plot.find('::')
if i != -1: plot = plot[i+2:]
bbfc = ''
if certificates:
for cert in certificates:
if cert.find(':') == -1: continue
certp = cert.split(':')
if len(certp) < 2: continue
if certp[0].lower() == 'uk' and certp[1]:
bbfc = certp[1]
# The data from IMDb.
imdbdata = [('language', language), ('title', title), ('year', year),
('director', director), ('imgurl', coverURL), ('actors', actors),
('runtime', runtime), ('country', country), ('plot', plot),
('imdbID', imdbID), ('subtitle', subtitle), ('comment', comment)]
if kind in ('tv series', 'tv mini series', 'episode'):
imdbdata.append(('istv', '1'))
# A since version 3.0 or so, videoDB supports ratings.
try:
curs.execute('SELECT rating from videodata;')
imdbdata.append(('rating', rating))
except _mysql_exceptions:
pass
# Add custom columns.
for cf in ('custom1type', 'custom2type', 'custom3type', 'custom4type'):
curs.execute("SELECT value from config WHERE opt = %s", (cf,))
res = curs.fetchone()
if not (res and res[0]): continue
res = res[0]
cf = cf[:7]
if res == 'mpaa' and mpaa:
imdbdata.append((cf, mpaa))
elif res == 'orgtitle' and title:
imdbdata.append((cf, title))
elif res == 'rating' and rating:
imdbdata.append((cf, rating))
elif res == 'language' and language:
imdbdata.append((cf, language))
elif res == 'bbfc' and bbfc:
imdbdata.append((cf, bbfc))
if not confset['override']:
imdbdata[:] = [x for x in imdbdata if not videodata_set.get(x[0])]
# Update the videodata_set information.
videodata_set.update(dict(imdbdata))
# The owner_id field.
if VDB_UID is not None:
videodata_set['owner_id'] = VDB_UID
# The data from the line arguments (always override).
in_encoding = sys.stdin.encoding or sys.getdefaultencoding()
userdata = []
for arg in args:
kv = arg.split('=')
if len(kv) != 2:
print 'Ignoring argument "%s".' % arg
continue
if kv[0] == 'userseen' and VDB_UID is not None:
try: USER_SEEN = int(kv[1])
except ValueError: pass
continue
if kv[0] not in videodata_set.keys():
print 'Ignoring argument "%s".' % arg
continue
k = unicode(kv[0], in_encoding, 'xmlcharrefreplace')
v = kv[0]
userdata.append((k, v))
# Build the definitive videodata_set dictionary.
videodata_set.update(dict(userdata))
# Convert a string like "dvd+rw" to a numeric value like 10.
if videodata_set.has_key('mediatype'):
mediatype = str(videodata_set['mediatype']).lower()
if mediatype in mediatypes.keys():
videodata_set['mediatype'] = mediatypes[mediatype]
# XXX: how can "genres" be override from the config file or command line?
# Strip every empty value and create the sql string and data set.
sql_set = [(key, value) for key, value in videodata_set.items() if value]
sql_str = ', '.join(['%s = %%s' % x[0] for x in sql_set])
if sql_str: sql_str += ', created = NOW()'
sql_data = [x[1] for x in sql_set]
vdata_sql = ''
if not UPDATE:
print 'Inserting the new movie "%s"... ' % imdbID,
sys.stdout.flush()
vdata_sql = 'INSERT INTO videodata SET %s;' % sql_str
else:
print 'Updating VideoDB id "%s" with data from "%s"... ' % \
(UPDATE_ID, imdbID),
sys.stdout.flush()
vdata_sql = "UPDATE videodata SET %s WHERE id = %s;" % (sql_str, UPDATE_ID)
try:
if not TEST: curs.execute(vdata_sql, sql_data)
except MySQLdb.Error, e:
print '\nError updating the videodata table.'
print str(e)
sys.exit(5)
print 'done!'
if not UPDATE and not TEST:
if hasattr(curs, 'insert_id'):
try:
UPDATE_ID = curs.insert_id()
except MySQLdb.Error:
UPDATE_ID = None
elif hasattr(curs, 'lastrowid'):
UPDATE_ID = curs.lastrowid
else:
UPDATE_ID = None
# ------- INSERT DATA INTO THE videogenre TABLE.
if UPDATE_ID is None:
print 'ERROR: unable to get the id for the last inserted row.'
else:
print 'Updating genres and userseen information... ',
sys.stdout.flush()
for genre in genres:
gid = gen_dict.get(genre.lower())
if not gid:
print '\nunknown genre: "%s".' % genre
continue
if not curs.execute("SELECT * from videogenre WHERE " + vg_cols[0] +
" = %s AND " + vg_cols[1] + " = %s;",
(UPDATE_ID, gid)):
if not TEST:
curs.execute("INSERT INTO videogenre SET " + vg_cols[0] +
" = %s, " + vg_cols[1] + " = %s;", (UPDATE_ID, gid))
if not TEST and VDB_UID is not None and USER_SEEN is not None:
alreadySeen = curs.execute("SELECT * FROM userseen WHERE " +
us_cols[1] + " = %s AND " + us_cols[0] +
" = %s;", (UPDATE_ID, VDB_UID))
if USER_SEEN:
if not alreadySeen:
curs.execute("INSERT INTO userseen SET " +
us_cols[1] + " = %s, " + us_cols[0] + " = %s;",
(UPDATE_ID, VDB_UID))
else:
if alreadySeen:
curs.execute("DELETE FROM userseen WHERE " + us_cols[1] +
" = %s " + "AND " + us_cols[0] + " = %s;",
(UPDATE_ID, VDB_UID))
print 'done!'
# ------- UPDATE THE actors TABLE.
print 'Updating the actors table... ',
sys.stdout.flush()
# XXX: if actors="name1::role1\nname2::role2" is given on the
# command line this is ignored and values from IMDb still used.
for act in actors_list:
# Strip the name.
name = act.get('name', u'')
actURL = act.get('headshot', '')
personID = imdbconn.get_imdbID(act) or ''
# XXX: use only actorid?
curs.execute("SELECT name,imgurl,actorid FROM actors WHERE name = %s;",
(name, ))
res = curs.fetchone()
if TEST: continue
if not res:
curs.execute('INSERT INTO actors SET name = %s, imgurl = %s;',
(act, actURL))
elif len(filter(None, res)) != 3 and res[0] and confset['override']:
curs.execute("""UPDATE actors SET imgurl = %s, actorid = %s
WHERE name = %s;""", (actURL, personID, name))
print 'done!'
if confset['getCover'] and coverURL:
print 'Fetching the movie cover... ',
sys.stdout.flush()
fu = urllib.urlopen(coverURL)
fu.read()
fu.close()
print 'done!'
if confset['getHeadShots']:
print 'Fetching the actors headshots... ',
sys.stdout.flush()
for act in actors_list:
imdbconn.update(act)
hshot = act.get('headshot')
if not hshot: continue
fu = urllib.urlopen(hshot)
fu.read()
fu.close()
print 'done!'
sys.exit(0)
-------------------------------------------------------------------------
Sponsored by: SourceForge.net Community Choice Awards: VOTE NOW!
Studies have shown that voting for your favorite open source project,
along with a healthy diet, reduces your potential for chronic lameness
and boredom. Vote Now at http://www.sourceforge.net/community/cca08
_______________________________________________
Imdbpy-devel mailing list
Imdbpy-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/imdbpy-devel
