Author: duncan Date: Wed Sep 12 14:11:33 2007 New Revision: 9870 Log: [ 1783643 ] Youtube / URL downloader webserver plugin. Update from Don Lock added
Added: branches/rel-1/freevo/src/www/htdocs/downloadurl.py (contents, props changed) branches/rel-1/freevo/src/www/htdocs/youtube-dl (contents, props changed) branches/rel-1/freevo/src/www/htdocs/youtube.js (contents, props changed) Modified: branches/rel-1/freevo/ChangeLog branches/rel-1/freevo/src/www/htdocs/youtube.rpy Modified: branches/rel-1/freevo/ChangeLog ============================================================================== --- branches/rel-1/freevo/ChangeLog (original) +++ branches/rel-1/freevo/ChangeLog Wed Sep 12 14:11:33 2007 @@ -19,8 +19,9 @@ * New Afrikaans translation (F#1790781) * Updated German translation (F#1770195) * New Eject CD-ROM plug-in, adding a menu item to the drive menu (F#1773418) - * New Text Entry and Program Search (F#1768790) - * New youtube plug-in for the webserver (F#1783643) + * New lastfm menu plug-in (F#1792494) + * New text entry and program search (F#1768790) + * New youtube plug-in for the webserver (F#1783643,F#1792819) * Updated lyrics grabber for scrollable lyrics (F#1786313) * Updated the LCD plug-in to allow a 16x4 mode (F#1776450) * Updated alsamixer with event args and synchronous mixer control (F#1767928) Added: branches/rel-1/freevo/src/www/htdocs/downloadurl.py ============================================================================== --- (empty file) +++ branches/rel-1/freevo/src/www/htdocs/downloadurl.py Wed Sep 12 14:11:33 2007 @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +__author__="Andrew Pennebaker ([EMAIL PROTECTED])" +__date__="3 Nov 2005 - 14 Feb 2007" +__copyright__="Copyright 2006 2007 Andrew Pennebaker" +__license__="GPL" +__version__="0.5" +__URL__="http://snippets.dzone.com/posts/show/2887"; + +from urllib import urlopen +import os +import math +import time +import sys +from getopt import getopt + +def optimum_k_exp(num_bytes): + const_1k = 1024 + if num_bytes == 0: + return 0 + return long(math.log(num_bytes, const_1k)) + +def format_bytes(num_bytes): + const_1k = 1024 + try: + exp = optimum_k_exp(num_bytes) + suffix = 'bkMGTPEZY'[exp] + if exp == 0: + return '%s%s' % (num_bytes, suffix) + converted = float(num_bytes) / float(const_1k**exp) + return '%.2f%s' % (converted, suffix) + except IndexError: + sys.exit('Error: internal error formatting number of bytes.') + + +# Calculate ETA and return it in string format as MM:SS +def calc_eta(start, now, total, current): + if current == 0: + return '--:--' + rate = float(current) / (now - start) + eta = long((total - current) / rate) + eta_mins = eta / 60 + eta_secs = eta % 60 + if eta_mins > 99: + return '--:--' + return '%02d:%02d' % (eta_mins, eta_secs) + +# Calculate speed and return it in string format +def calc_speed(start, now, bytes): + if bytes == 0: + return 'N/A b' + return format_bytes(float(bytes) / (now - start)) + +def cond_print(str): + sys.stdout.write(str) + sys.stdout.flush() + +def getURLName(url): + name = url.split("/")[-1] + return name + +def createDownload(url, proxy=None): + instream=urlopen(url, None, proxy) + + filename=instream.info().getheader("Content-Length") + if filename==None: + filename="temp" + + return (instream, filename) + +def download(instream, outstream): + outstream.write(instream.read()) + + outstream.close() + +def usage(): + print "Usage: %s [options] <url1 url2 url3 ...>" % (sys.argv[0]) + print "\n--httpproxy <proxy>" + print "--ftpproxy <proxy>" + print "--gopherproxy <proxy>" + print "\n--help (usage)" + + sys.exit() + +def main(): + systemArgs=sys.argv[1:] # ignore program name + start_time = time.time() + + urls=[] + proxies={} + + optlist=[] + args=[] + + try: + optlist, args=getopt(systemArgs, None, ["url=", "httpproxy=", "ftpproxy=", "gopherproxy=", "help"]) + except Exception, e: + usage() + + if len(args)<1: + usage() + + for option, value in optlist: + if option=="--help": + usage() + + elif option=="--httpproxy": + proxies["http"]=value + elif option=="--ftpproxy": + proxies["ftp"]=value + elif options=="--gopherproxy": + proxies["gopher"]=value + + urls=args + + for url in urls: + + outfile = getURLName(url) + print outfile + fileName= outfile.split(os.sep)[-1] + fName = fileName + fileName= "partfile" + fileName + print "fName - " + fName + print "fileName - " + fileName + +# try: + + outfile=open(fileName, "wb") + url, length=createDownload(url, proxies) + if not length: + length="?" + + print "Downloading %s (%s bytes) ..." % (url.url, length) + if length!="?": + length=float(length) + bytesRead=0.0 + + for line in url: + bytesRead+=len(line) + + if length!="?": + status = "%s: %.02f/%.02f kb (%d%%)" % ( + fileName, + bytesRead/1024.0, + length/1024.0, + 100*bytesRead/length + ) + percent = (100*bytesRead/length) + percent_str = '%.1f' % percent + counter = format_bytes(bytesRead) + video_len_str = format_bytes(length) + speed = calc_speed(start_time,time.time(),bytesRead) + eta = calc_eta(start_time, time.time(), length, bytesRead) + cond_print('\rRetrieving video data: %5s%% (%8s of %s) at %s ETA %s ' % (percent_str, counter, video_len_str,speed,eta)) +# print status + + outfile.write(line) + + url.close() + outfile.close() + +# except Exception, e: +# print "Error downloading %s: %s" % (url, e) + os.rename(fileName,fName) + + + +if __name__=="__main__": + main() Added: branches/rel-1/freevo/src/www/htdocs/youtube-dl ============================================================================== --- (empty file) +++ branches/rel-1/freevo/src/www/htdocs/youtube-dl Wed Sep 12 14:11:33 2007 @@ -0,0 +1,405 @@ +#!/usr/bin/env python +# +# Copyright (c) 2006-2007 Ricardo Garcia Gonzalez +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name(s) of the above copyright +# holders shall not be used in advertising or otherwise to promote the +# sale, use or other dealings in this Software without prior written +# authorization. +# +import getpass +import httplib +import math +import netrc +import optparse +import os +import re +import socket +import string +import sys +import time +import urllib2 + +# Global constants +const_video_url_str = 'http://www.youtube.com/watch?v=%s' +const_video_url_re = re.compile(r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?(?:.+&)?v=))?([0-9A-Za-z_-]+)(?(1)[&/].*)?$') +const_login_url_str = 'http://www.youtube.com/login?next=/watch%%3Fv%%3D%s' +const_login_post_str = 'current_form=loginForm&next=%%2Fwatch%%3Fv%%3D%s&username=%s&password=%s&action_login=Log+In' +const_age_url_str = 'http://www.youtube.com/verify_age?next_url=/watch%%3Fv%%3D%s' +const_age_post_str = 'next_url=%%2Fwatch%%3Fv%%3D%s&action_confirm=Confirm' +const_url_t_param_re = re.compile(r'[,{]t:\'([^\']*)\'') +const_video_url_real_str = 'http://www.youtube.com/get_video?video_id=%s&t=%s' +const_video_title_re = re.compile(r'<title>YouTube - ([^<]*)</title>', re.M | re.I) +const_1k = 1024 +const_initial_block_size = 10 * const_1k + +# Print error message, followed by standard advice information, and then exit +def error_advice_exit(error_text): + sys.stderr.write('Error: %s.\n' % error_text) + sys.stderr.write('Try again several times. It may be a temporary problem.\n') + sys.stderr.write('Other typical problems:\n\n') + sys.stderr.write('* Video no longer exists.\n') + sys.stderr.write('* Video requires age confirmation but you did not provide an account.\n') + sys.stderr.write('* You provided the account data, but it is not valid.\n') + sys.stderr.write('* The connection was cut suddenly for some reason.\n') + sys.stderr.write('* YouTube changed their system, and the program no longer works.\n') + sys.stderr.write('\nTry to confirm you are able to view the video using a web browser.\n') + sys.stderr.write('Use the same video URL and account information, if needed, with this program.\n') + sys.stderr.write('When using a proxy, make sure http_proxy has http://host:port format.\n') + sys.stderr.write('Try again several times and contact me if the problem persists.\n') + sys.exit('\n') + +# Wrapper to create custom requests with typical headers +def request_create(url, data=None): + retval = urllib2.Request(url) + if data is not None: + retval.add_data(data) + # Try to mimic Firefox, at least a little bit + retval.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0') + retval.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7') + retval.add_header('Accept', 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5') + retval.add_header('Accept-Language', 'en-us,en;q=0.5') + return retval + +# Perform a request, process headers and return response +def perform_request(url, data=None): + request = request_create(url, data) + response = urllib2.urlopen(request) + return response + +# Conditional print +def cond_print(str): + global cmdl_opts + if not (cmdl_opts.quiet or cmdl_opts.get_url): + sys.stdout.write(str) + sys.stdout.flush() + +# Title string normalization +def title_string_norm(title): + title = ''.join((x in string.ascii_letters or x in string.digits) and x or ' ' for x in title) + title = '_'.join(title.split()) + title = title.lower() + return title + +# Title string minimal transformation +def title_string_touch(title): + return title.replace(os.sep, '%') + +# Generic download step +def download_step(return_data_flag, step_title, step_error, url, post_data=None): + try: + cond_print('%s... ' % step_title) + data = perform_request(url, post_data).read() + cond_print('done.\n') + if return_data_flag: + return data + return None + + except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error): + cond_print('failed.\n') + error_advice_exit(step_error) + + except KeyboardInterrupt: + sys.exit('\n') + +# Generic extract step +def extract_step(step_title, step_error, regexp, data): + try: + cond_print('%s... ' % step_title) + match = regexp.search(data) + + if match is None: + cond_print('failed.\n') + error_advice_exit(step_error) + + extracted_data = match.group(1) + cond_print('done.\n') + return extracted_data + + except KeyboardInterrupt: + sys.exit('\n') + +# Calculate new block size based on previous block size +def new_block_size(before, after, bytes): + new_min = max(bytes / 2.0, 1.0) + new_max = max(bytes * 2.0, 1.0) + dif = after - before + if dif < 0.0001: + return int(new_max) + rate = bytes / dif + if rate > new_max: + return int(new_max) + if rate < new_min: + return int(new_min) + return int(rate) + +# Get optimum 1k exponent to represent a number of bytes +def optimum_k_exp(num_bytes): + global const_1k + if num_bytes == 0: + return 0 + return long(math.log(num_bytes, const_1k)) + +# Get optimum representation of number of bytes +def format_bytes(num_bytes): + global const_1k + try: + exp = optimum_k_exp(num_bytes) + suffix = 'bkMGTPEZY'[exp] + if exp == 0: + return '%s%s' % (num_bytes, suffix) + converted = float(num_bytes) / float(const_1k**exp) + return '%.2f%s' % (converted, suffix) + except IndexError: + sys.exit('Error: internal error formatting number of bytes.') + +# Calculate ETA and return it in string format as MM:SS +def calc_eta(start, now, total, current): + if current == 0: + return '--:--' + rate = float(current) / (now - start) + eta = long((total - current) / rate) + eta_mins = eta / 60 + eta_secs = eta % 60 + if eta_mins > 99: + return '--:--' + return '%02d:%02d' % (eta_mins, eta_secs) + +# Calculate speed and return it in string format +def calc_speed(start, now, bytes): + if bytes == 0: + return 'N/A b' + return format_bytes(float(bytes) / (now - start)) + +# Create the command line options parser and parse command line +cmdl_usage = 'usage: %prog [options] video_url' +cmdl_version = '2007.08.24' +cmdl_parser = optparse.OptionParser(usage=cmdl_usage, version=cmdl_version, conflict_handler='resolve') +cmdl_parser.add_option('-h', '--help', action='help', help='print this help text and exit') +cmdl_parser.add_option('-v', '--version', action='version', help='print program version and exit') +cmdl_parser.add_option('-u', '--username', dest='username', metavar='USERNAME', help='account username') +cmdl_parser.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password') +cmdl_parser.add_option('-o', '--output', dest='outfile', metavar='FILE', help='output video file name') +cmdl_parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode') +cmdl_parser.add_option('-s', '--simulate', action='store_true', dest='simulate', help='do not download video') +cmdl_parser.add_option('-t', '--title', action='store_true', dest='use_title', help='use title in file name') +cmdl_parser.add_option('-l', '--literal', action='store_true', dest='use_literal', help='use literal title in file name') +cmdl_parser.add_option('-n', '--netrc', action='store_true', dest='use_netrc', help='use .netrc authentication data') +cmdl_parser.add_option('-g', '--get-url', action='store_true', dest='get_url', help='print final video URL only') +cmdl_parser.add_option('-2', '--title-too', action='store_true', dest='get_title', help='used with -g, print title too') +cmdl_parser.add_option('-L','--logfile', action='store_true', dest='log_file', help='used to redirect output messages') + +(cmdl_opts, cmdl_args) = cmdl_parser.parse_args() + +# Get video URL +if len(cmdl_args) != 1: + cmdl_parser.print_help() + sys.exit('\n') +video_url_cmdl = cmdl_args[0] + +# Verify video URL format and convert to "standard" format +video_url_mo = const_video_url_re.match(video_url_cmdl) +if video_url_mo is None: + sys.exit('Error: URL does not seem to be a youtube video URL. If it is, report a bug.') +video_url_id = video_url_mo.group(2) +video_url = const_video_url_str % video_url_id + +# Check conflicting options +if cmdl_opts.outfile is not None and (cmdl_opts.simulate or cmdl_opts.get_url): + sys.stderr.write('Warning: video file name given but will not be used.\n') + +if cmdl_opts.outfile is not None and (cmdl_opts.use_title or cmdl_opts.use_literal): + sys.exit('Error: using the video title conflicts with using a given file name.') + +if cmdl_opts.use_netrc and cmdl_opts.password is not None: + sys.exit('Error: using netrc conflicts with giving command line password.') + +if cmdl_opts.use_title and cmdl_opts.use_literal: + sys.exit('Error: cannot use title and literal title at the same time.') + +if cmdl_opts.quiet and cmdl_opts.get_url: + sys.exit('Error: cannot be quiet and print final URL at the same time.') + +# Incorrect option formatting +if cmdl_opts.username is None and cmdl_opts.password is not None: + sys.exit('Error: password give but username is missing.') + +if cmdl_opts.get_url is None and cmdl_opts.get_title is not None: + sys.exit('Error: getting title requires getting URL.') + +# Get account information if any +account_username = None +account_password = None + +if cmdl_opts.use_netrc: + try: + info = netrc.netrc().authenticators('youtube') + if info is None: + sys.exit('Error: no authenticators for machine youtube.') + netrc_username = info[0] + netrc_password = info[2] + except IOError: + sys.exit('Error: unable to read .netrc file.') + except netrc.NetrcParseError: + sys.exit('Error: unable to parse .netrc file.') + +if cmdl_opts.password is not None: + account_username = cmdl_opts.username + account_password = cmdl_opts.password +else: + if cmdl_opts.username is not None and cmdl_opts.use_netrc: + if cmdl_opts.username != netrc_username: + sys.exit('Error: conflicting username from .netrc and command line options.') + account_username = cmdl_opts.username + account_password = netrc_password + elif cmdl_opts.username is not None: + account_username = cmdl_opts.username + account_password = getpass.getpass('Type YouTube password and press return: ') + elif cmdl_opts.use_netrc: + if len(netrc_username) == 0: + sys.exit('Error: empty username in .netrc file.') + account_username = netrc_username + account_password = netrc_password + +# Get output file name +if cmdl_opts.outfile is None: + video_filename = '%s.flv' % video_url_id +else: + video_filename = cmdl_opts.outfile + +# Check name +if not video_filename.lower().endswith('.flv'): + sys.stderr.write('Warning: video file name does not end in .flv\n') + +# Test writable file +if not (cmdl_opts.simulate or cmdl_opts.get_url): + try: + disk_test = open(video_filename, 'wb') + disk_test.close() + + except (OSError, IOError): + sys.exit('Error: unable to open %s for writing.' % video_filename) + +# Install cookie and proxy handlers +urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler())) +urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor())) + +# Log in and confirm age if needed +if account_username is not None: + url = const_login_url_str % video_url_id + post = const_login_post_str % (video_url_id, account_username, account_password) + download_step(False, 'Logging in', 'unable to log in', url, post) + + url = const_age_url_str % video_url_id + post = const_age_post_str % video_url_id + download_step(False, 'Confirming age', 'unable to confirm age', url, post) + +# Retrieve video webpage +video_webpage = download_step(True, 'Retrieving video webpage', 'unable to retrieve video webpage', video_url) + +# Extract video title if needed +if cmdl_opts.use_title or cmdl_opts.use_literal or cmdl_opts.get_title: + video_title = extract_step('Extracting video title', 'unable to extract video title', const_video_title_re, video_webpage) + +# Extract needed video URL parameters +video_url_t_param = extract_step('Extracting URL "t" parameter', 'unable to extract URL "t" parameter', const_url_t_param_re, video_webpage) +video_url_real = const_video_url_real_str % (video_url_id, video_url_t_param) + +# Retrieve video data +try: + video_data = perform_request(video_url_real) + cond_print('Video data found at %s\n' % video_data.geturl()) + + if cmdl_opts.get_title: + print video_title + + if cmdl_opts.get_url: + print video_data.geturl() + + if cmdl_opts.simulate or cmdl_opts.get_url: + sys.exit() + + video_file = open(video_filename, 'wb') + try: + video_len = long(video_data.info()['Content-length']) + video_len_str = format_bytes(video_len) + except KeyError: + video_len = None + video_len_str = 'N/A' + + byte_counter = 0 + block_size = const_initial_block_size + start_time = time.time() + while True: + if video_len is not None: + percent = float(byte_counter) / float(video_len) * 100.0 + percent_str = '%.1f' % percent + eta_str = calc_eta(start_time, time.time(), video_len, byte_counter) + else: + percent_str = '---.-' + eta_str = '--:--' + counter = format_bytes(byte_counter) + speed_str = calc_speed(start_time, time.time(), byte_counter) + cond_print('\rRetrieving video data: %5s%% (%8s of %s) at %8s/s ETA %s ' % (percent_str, counter, video_len_str, speed_str, eta_str)) + + before = time.time() + video_block = video_data.read(block_size) + after = time.time() + dl_bytes = len(video_block) + if dl_bytes == 0: + break + byte_counter += dl_bytes + video_file.write(video_block) + block_size = new_block_size(before, after, dl_bytes) + + if video_len is not None and byte_counter != video_len: + error_advice_exit('server did not send the expected ammount of data') + + video_file.close() + cond_print('done.\n') + cond_print('Video data saved to %s\n' % video_filename) + +except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error): + cond_print('failed.\n') + error_advice_exit('unable to download video data') + +except KeyboardInterrupt: + sys.exit('\n') + +# Rename video file if needed +if cmdl_opts.use_title or cmdl_opts.use_literal: + try: + if cmdl_opts.use_title: + prefix = title_string_norm(video_title) + else: + prefix = title_string_touch(video_title) + final_filename = '%s-%s.flv' % (prefix, video_url_id) + os.rename(video_filename, final_filename) + cond_print('Video file renamed to %s\n' % final_filename) + + except OSError: + sys.stderr.write('Warning: unable to rename file.\n') + + except KeyboardInterrupt: + sys.exit('\n') + +# Finish +sys.exit() Added: branches/rel-1/freevo/src/www/htdocs/youtube.js ============================================================================== --- (empty file) +++ branches/rel-1/freevo/src/www/htdocs/youtube.js Wed Sep 12 14:11:33 2007 @@ -0,0 +1,187 @@ +//enter refresh time in "minutes:seconds" Minutes should range from 0 to inifinity. Seconds should range from 0 to 59' +//var limit="0:10" +if (document.images){ + var parselimit=5 +} + +function beginrefresh(){ + if (!document.images) + return + if (parselimit==1) { + document.refreshForm.visited.value = "-1"; + makeRequest('youtube.rpy?xml=1'); + parselimit=5 + setTimeout("beginrefresh()",1000) + } + else{ + parselimit-=1 + document.refreshForm.visited.value = parselimit + curmin=Math.floor(parselimit/60) + cursec=parselimit%60 + if (curmin!=0) + curtime=curmin+" minutes and "+cursec+" seconds left until page refresh!" + else + curtime=cursec+" seconds left until page refresh!" + window.status=curtime + setTimeout("beginrefresh()",1000) + } +} +window.onload=beginrefresh + +function makeRequest(url) { + var httpRequest; + + if (window.XMLHttpRequest) { // Mozilla, Safari, ... + httpRequest = new XMLHttpRequest(); + if (httpRequest.overrideMimeType) { + httpRequest.overrideMimeType('text/xml'); + // See note below about this line + } + } + else if (window.ActiveXObject) { // IE + try { + httpRequest = new ActiveXObject("Msxml2.XMLHTTP"); + } + catch (e) { + try { + httpRequest = new ActiveXObject("Microsoft.XMLHTTP"); + } + catch (e) {} + } + } + + if (!httpRequest) { + alert('Giving up :( Cannot create an XMLHTTP instance'); + return false; + } + httpRequest.onreadystatechange = function() { alertContents(httpRequest); }; + httpRequest.open('GET', url, true); + httpRequest.send(''); + +} + +function TableAddRow(file,filesize,percent,amtdone,speed,eta) { + var tbl,lastrow,newrow,cellLeft; + var textNode; + + tbl = document.getElementById("filelist") + lastrow = tbl.rows.length; + newrow = tbl.insertRow(lastrow); + newrow.id = file + newrow.className = "chanrow" + + cellLeft = newrow.insertCell(0); + textNode = document.createTextNode("Delete"); + cellLeft.appendChild(textNode); + cellLeft.className = "basic" + + cellLeft = newrow.insertCell(1); + textNode = document.createTextNode(file); + cellLeft.appendChild(textNode); + cellLeft.className = "basic" + + cellLeft = newrow.insertCell(2); + textNode = document.createTextNode(percent); + cellLeft.appendChild(textNode); + cellLeft.id = file + ".PERCENT" + cellLeft.className = "basic" + + cellLeft = newrow.insertCell(3); + textNode = document.createTextNode(amtdone); + cellLeft.appendChild(textNode); + cellLeft.id = file + ".SOFAR" + cellLeft.className = "basic" + + cellLeft = newrow.insertCell(4); + textNode = document.createTextNode(filesize); + cellLeft.appendChild(textNode); + cellLeft.id = file + ".FILESIZE" + cellLeft.className = "basic" + + cellLeft = newrow.insertCell(5); + textNode = document.createTextNode(speed); + cellLeft.appendChild(textNode); + cellLeft.id = file + ".SPEED" + cellLeft.className = "basic" + + cellLeft = newrow.insertCell(6); + textNode = document.createTextNode(eta); + cellLeft.appendChild(textNode); + cellLeft.id = file + ".ETA" + cellLeft.className = "basic" + + lastrow = tbl.rows.length; +} + +function RemoveRows(lfiles) { + var tbl,r,cnt,rfile,fnd,xfile ; + tbl = document.getElementById("filelist"); + + for (r=1; r < tbl.rows.length; r++) { + rfile = tbl.rows[r].id + fnd = 0 + for (j=0; j < lfiles.childNodes.length; j++) { + cfile = lfiles.childNodes[j] + xfile = cfile.childNodes[0].firstChild.nodeValue; + if (xfile == rfile) { + fnd = 1 + j = lfiles.childNodes.length + } + } + if (fnd == 0) { + fnd = 1; + tbl.deleteRow(r); + //alert("Delete Row - " + rfile); + } + cnt = 0; + } + +} + +function alertContents(httpRequest) { + var j,cellObj; + var cfile,filename,percent,amtdone,filesize,speed,eta; + + if (httpRequest.readyState == 4) { + if (httpRequest.status == 200) { + + var filelist = httpRequest.responseXML.childNodes[0]; + for (j=0; j < filelist.childNodes.length; j++) { + cfile = filelist.childNodes[j] + filename = cfile.childNodes[0].firstChild.nodeValue; + + percent = cfile.childNodes[1].firstChild.nodeValue; + amtdone = cfile.childNodes[2].firstChild.nodeValue; + filesize = cfile.childNodes[3].firstChild.nodeValue; + speed = cfile.childNodes[4].firstChild.nodeValue; + eta = cfile.childNodes[5].firstChild.nodeValue; + + // Check to see if a table row exists for the file. + cellObj = document.getElementById(filename + ".FILESIZE"); + if (cellObj == null) { + TableAddRow(filename,filesize,percent,amtdone,speed,eta) + } + else { + cellObj = document.getElementById(filename + ".FILESIZE"); + cellObj.childNodes[0].nodeValue=filesize; + cellObj = document.getElementById(filename + ".PERCENT"); + cellObj.childNodes[0].nodeValue=percent; + cellObj = document.getElementById(filename + ".SOFAR"); + cellObj.childNodes[0].nodeValue=amtdone; + cellObj = document.getElementById(filename + ".SPEED"); + cellObj.childNodes[0].nodeValue=speed; + cellObj = document.getElementById(filename + ".ETA"); + cellObj.childNodes[0].nodeValue=eta; + } + + } + RemoveRows(filelist); + } else { + alert('There was a problem with the request.'); + } + } +} + + + + Modified: branches/rel-1/freevo/src/www/htdocs/youtube.rpy ============================================================================== --- branches/rel-1/freevo/src/www/htdocs/youtube.rpy (original) +++ branches/rel-1/freevo/src/www/htdocs/youtube.rpy Wed Sep 12 14:11:33 2007 @@ -20,137 +20,287 @@ # # ----------------------------------------------------------------------- - import sys +import time import os import os.path import config +import subprocess +import string +import math + from www.web_types import HTMLResource, FreevoResource from urllib import urlopen from getopt import getopt -def createDownload(url, proxy=None): - instream=urlopen(url, None, proxy) +print "CONFIG=====" +print (config) - filename=instream.info().getheader("Content-Length") - if filename==None: - filename="temp" +def xmlStatus(fstatus): + xmlstatus = "" + xmlstatus += "<PERCENT>" + fstatus['percent'] + "</PERCENT>" + xmlstatus += "<DOWNLOADED>" + fstatus['downloaded'] + "</DOWNLOADED>" + xmlstatus += "<FILESIZE>" + fstatus['filesize'] + "</FILESIZE>" + xmlstatus += "<SPEED>" + fstatus['speed'] + "</SPEED>" + xmlstatus += "<ETA>" + fstatus['eta'] + "</ETA>" + + return xmlstatus + +# Get optimum 1k exponent to represent a number of bytes +def optimum_k_exp(num_bytes): + const_1k = 1024 + if num_bytes == 0: + return 0 + return long(math.log(num_bytes, const_1k)) + + +# Get optimum representation of number of bytes +def format_bytes(num_bytes): + const_1k = 1024 + try: + exp = optimum_k_exp(num_bytes) + suffix = 'bkMGTPEZY'[exp] + if exp == 0: + return '%s%s' % (num_bytes, suffix) + converted = float(num_bytes) / float(const_1k**exp) + return '%.2f%s' % (converted, suffix) + except IndexError: + return "Error" + + +def getStatus(ytfile): + fileStatus = {'percent': '--.-%', 'downloaded': 'done' , 'filesize': 'DONE' ,'speed' : '--', 'eta' : '--:--'} + fileStatus['filesize'] = format_bytes(os.path.getsize(config.YOUTUBE_DIR + ytfile)) + logfile = config.YOUTUBE_DIR + ".tmp/" + os.path.splitext(ytfile)[0] + ".log" + if os.path.exists(logfile): + lfhandle = open(logfile, "r") + llines = lfhandle.readlines() + try : + if len(llines) == 5: + status = llines[4].split("\r")[-1] + status = status.split() + fileStatus['percent'] = status[3] + fileStatus['downloaded'] = status[5] + fileStatus['filesize'] = status[7].strip(")") + fileStatus['speed'] = status[9] + fileStatus['eta'] = status[11] + else: + fileStatus['percent'] = "NA" + fileStatus['downloaded'] = "NA" + fileStatus['speed'] = "NA" + fileStatus['eta'] = "Unknown" + except Exception, e: + fileStatus['percent'] = "NA" + fileStatus['downloaded'] = "NA" + fileStatus['speed'] = "NA" + fileStatus['eta'] = "Unknown" + return fileStatus + + +def getXML(): + + dfiles = '<?xml version="1.0" encoding="ISO-8859-1" ?>' + dfiles += '<FILELIST>' + enablerefresh = False + + filelist = os.listdir(config.YOUTUBE_DIR) + filelist.sort() + for fl in filelist : + if fl != ".tmp" and fl != "folder.fxd" : + # check for log file. + dfiles += '<FILE id="' + fl + '">' + dfiles += '<FILENAME>' + fl + '</FILENAME>' + fstats = getStatus(fl) + dfiles += xmlStatus(fstats) + dfiles += '</FILE>' + dfiles += "</FILELIST>" + retdfile = [] + retdfile.append(dfiles) + retdfile.append(enablerefresh) + return retdfile + +def displayfiles(fvhtml): + fvhtml.res += '' + fvhtml.tableOpen('class="library" id="filelist"') + fvhtml.tableRowOpen('class="chanrow"') + fvhtml.tableCell('Current Downloads','class ="guidehead" colspan="2"') + fvhtml.tableCell('% Done','class ="guidehead" colspan="1"') + fvhtml.tableCell('','class ="guidehead" colspan="1"') + fvhtml.tableCell('Size','class ="guidehead" colspan="1"') + fvhtml.tableCell('Speed','class ="guidehead" colspan="1"') + fvhtml.tableCell('ETA','class ="guidehead" colspan="1"') + fvhtml.tableRowClose() + + enablerefresh = False + + filelist = os.listdir(config.YOUTUBE_DIR) + filelist.sort() + for fl in filelist : + + # check to see if the file is currently being downloaded. + if fl != ".tmp" and fl != "folder.fxd" : + + # check for log file. + flstatus = getStatus(fl) + fvhtml.tableRowOpen('class="chanrow" colspan="1" id="'+fl +'"') + fvhtml.tableCell('<a href="youtube.rpy?Delete=1&file=' + fl + '">DELETE</a>','class="basic" colspan="1"') + fvhtml.tableCell(fl,'class="basic" colspan="1"') + fvhtml.tableCell(flstatus['percent'],'class="basic" colspan="1" id="' + fl + '.PERCENT"') + fvhtml.tableCell(flstatus['downloaded'],'class="basic" colspan="1" id = "' + fl + '.SOFAR"') + fvhtml.tableCell(flstatus['filesize'],'class="basic" colspan="1" id="' + fl + '.FILESIZE"') + fvhtml.tableCell(flstatus['speed'],'class="basic" colspan="1" id="' + fl + '.SPEED"') + fvhtml.tableCell(flstatus['eta'],'class="basic" colspan="1" id="' + fl + '.ETA"') + fvhtml.tableRowClose() + + fvhtml.tableClose() + retdfile = [] + retdfile.append(fvhtml.res) + retdfile.append(enablerefresh) + return retdfile + + +def CleanupLogFiles(): + logfiles = os.listdir(config.YOUTUBE_DIR + ".tmp/") + + for lfile in logfiles: + # Check to see if the movie file exists. + vfile = config.YOUTUBE_DIR + os.path.splitext(lfile)[0] + ".flv" + if not os.path.exists(vfile): + os.remove(config.YOUTUBE_DIR + ".tmp/" + lfile) - return (instream, filename) +def download_youtube(yt_url, yt_out): -def getURLName(url): - directory=os.curdir + stime = time.localtime() + cmd = "python " + config.YOUTUBE_DL + " -t " + yt_url + pwdcur = os.getcwd() + os.chdir(config.YOUTUBE_DIR) - name="%s%s%s" % ( - directory, - os.sep, - url.split("/")[-1] - ) + # get the file name from the url. + logfile = config.YOUTUBE_DIR + ".tmp/" + yt_url.split("=")[-1] + ".log" + ytpid = 0 + lfile = open (logfile, "w") + ytpid = subprocess.Popen((config.YOUTUBE_DL,"-t",yt_url),universal_newlines=True,stdout=lfile).pid + os.chdir(pwdcur) + dlstatus = "<br><br>Starting download of " + yt_url + " <br>" + dlstatus = dlstatus + '<input type="text" name="pid" size="40" value="' + str(ytpid) + '" />' - print "NAME " - print name - return name + return dlstatus -def download_url (dl_url): - print "Download URL - " + dl_url - url = dl_url +def download_url(dl_url, dl_out): - try: - outfile=open(getURLName(url), "wb") - fileName=outfile.name.split(os.sep)[-1] - fileName = config.YOUTUBE_DIR + fileName - print "FILENAME - " + fileName - - url, length=createDownload(url, None) - if not length: - length="?" - print "Downloading %s (%s bytes) ..." % (url.url, length) - - if length!="?": - length=float(length) - bytesRead=0.0 - - for line in url: - bytesRead+=len(line) - outfile.write(line) - - url.close() - outfile.close() - print "Done" + stime = time.localtime() + cmd = "python " + config.DOWNLOAD_DL + " " + dl_url + pwdcur = os.getcwd() + os.chdir(config.YOUTUBE_DIR) - except Exception, e: - return "Error downloading %s: %s" % (dl_url, e) + # get the file name from the url. + logfile = config.YOUTUBE_DIR + ".tmp/partfile" + dl_url.split("/")[-1] + logfile = os.path.splitext(logfile)[0] + ".log" + ytpid = 0 + lfile = open (logfile, "w") + ytpid = subprocess.Popen((config.DOWNLOAD_DL,dl_url),universal_newlines=True,stdout=lfile).pid + os.chdir(pwdcur) + dlstatus = "" - return "Done downloading - " + dl_url + return dlstatus -def download_youtube(yt_url, yt_out): - cmd = "python " + config.YOUTUBE_DL + " -t " + yt_url - pwdcur = os.getcwd() - pwddl = config.YOUTUBE_DIR - os.chdir(pwddl) +def addPageRefresh(): - cnt = 1 - for ln in os.popen(cmd).readline(): - print "COUNTING" - print cnt - cnt = cnt + 1 - print ln - -# child = os.popen(cmd) -# data = child.read() - -# err = child.close() -# if err: -# raise RuntimeError, '%s failed w/ exit code %d' % (command, err) - os.chdir(pwdcur) - return "<br/><br/>Done downloading " + yt_url + prhtml = '<script type="text/JavaScript" src="scripts/youtube.js">window.onload=beginrefresh</script>' + prhtml += '\n<form name="refreshForm">' + prhtml += '\n <div class="searchform"><br><b>Refresh In :</b>' + prhtml += '\n <input type="text" name="visited" value="1" size="4" align="middle" />' + prhtml += '\n </div>' + prhtml += '\n</form><br>' + return prhtml class YouTubeResource(FreevoResource): def _render(self, request): fv = HTMLResource() - fv.printHeader(_('YouTube'), 'styles/main.css',selected=_('YouTube')) form = request.args - # Check to see if youtube-dl script exists. + blxml = fv.formValue(form,"xml") + if blxml : + fdisplay = getXML() + fv.res += fdisplay[0] + return String( fv.res ) + + fv.printHeader(_('YouTube'), 'styles/main.css',selected=_('YouTube')) + + yterrors = [] + if (not config.__dict__.has_key('YOUTUBE_DIR')): + yterrors.append('Unable to Find YOUTUDE_DIR setting in local_conf.py') + yterrors.append('Add YOUTUBE_DIR = "Directory to Save Downloads." to your local_conf.py') + config.YOUTUBE_DIR = "MISSING" + if (not config.__dict__.has_key('YOUTUBE_DL')): + yterrors.append('Unable to Find YOUTUDE_DL setting in local_conf.py') + yterrors.append('Add YOUTUBE_DL = "Path to youtube-dl script" to your local_conf.py') + config.YOUTUBE_DL = "MISSING" + if (not config.__dict__.has_key('DOWNLOAD_DL')): + yterrors.append('Unable to Find DOWNLOAD_DL setting in local_conf.py') + yterrors.append('Add DOWNLOAD_DL = "Path to downloadurl.py script" to your local_conf.py') + config.DOWNLOAD_DL = "MISSING" + + if len(yterrors) > 0: + fv.printMessages(yterrors) + return String( fv.res ) + + + fldelete = fv.formValue(form,'Delete') + if fldelete: + filename = fv.formValue(form,'file') + if filename: + filename = config.YOUTUBE_DIR + filename + if os.path.exists(filename): + os.remove(filename) + fv.res += "DELETED FILE - " + filename + + yturl = "" if not os.path.exists(config.YOUTUBE_DL): - fv.res += '<br/><br/><br/><b>Unable to locate youtube-dl script "' + config.YOUTUBE_DL + '" </b><br/>' + fv.res += '<br><br><br><b>Unable to locate youtube-dl script "' + config.YOUTUBE_DL + '" </b><br>' fv.res += 'Download scripts from <a href="http://www.arrakis.es/~rggi3/youtube-dl/";>http://www.arrakis.es/~rggi3/youtube-dl/</a>' - fv.res += '<br/>Add YOUTUBE_DL = "path and file name to youtube_dl script"<br/>' + fv.res += '<br>Add YOUTUBE_DL = "path and file name to youtube_dl script"<br>' + else: + fv.res += '\n<br><form id="YouTube Download" action="youtube.rpy" method="get">' + fv.res += '\n<div class="searchform"><br><b>Youtube URL :</b><input type="text" name="yt_url" size="40" value="' + yturl + '" />' + fv.res += '\n<input type="submit" value=" Download! " />' + fv.res += '\n</div>' + fv.res += '\n</form>' + + yturl = fv.formValue(form,'yt_url') + if yturl : + yt_status = str(download_youtube(yturl,"")) + refreshon = True if not os.path.exists(config.YOUTUBE_DIR): - fv.res += '<br/><b>Unable to locate youtube download location "' + config.YOUTUBE_DIR + '" </b><br/>' + fv.res += '<br><b>Unable to locate youtube download location "' + config.YOUTUBE_DIR + '" </b><br>' fv.res += 'Add YOUTUBE_DIR = "download directory" to your local_conf.py' - yturl = fv.formValue(form,'yt_url') - if yturl : - print yturl - yt_status = download_youtube(yturl,"") - fv.res += "<br/>" + yt_status - - yturl = "" - fv.res += '<br/><form id="YouTube Download" action="youtube.rpy" method="get">' - fv.res += '<div class="searchform"><br/><b>Youtube URL :</b><input type="text" name="yt_url" size="40" value="' + yturl + '" />' - fv.res += '<input type="submit" value=" Download! " />' - fv.res += '</div>' - fv.res += '</form>' + if os.path.exists(config.YOUTUBE_DIR): + if not os.path.exists(config.YOUTUBE_DIR + ".tmp"): + os.mkdir(config.YOUTUBE_DIR + ".tmp") dlurl = fv.formValue(form,'dl_url') if dlurl : - dl_status = download_url(dlurl) - fv.res += "<br/>" + dl_status - else : - dlurl = "" - - fv.res += '<br/><form id="Url Download" action="youtube.rpy" method="get">' - fv.res += '<div class="searchform"><br/><b>Download URL :</b><input type="text" name="dl_url" size="40" value="' + dlurl + '" />' - fv.res += '<input type="submit" value=" Download! " />' - fv.res += '</div>' - fv.res += '</form>' + dl_status = download_url(dlurl,"") + fv.res += "<br>" + dl_status + refreshon = True + + dlurl = "" + fv.res += '\n<form id="Url Download" action="youtube.rpy" method="get">' + fv.res += '\n <div class="searchform"><br><b>Download URL :</b>' + fv.res += '\n <input type="text" name="dl_url" size="40" value="' + dlurl + '" />' + fv.res += '\n <input type="submit" value=" Download! " />' + fv.res += '\n </div>' + fv.res += '\n</form><br>\n' + + fdisplay = displayfiles(fv) + fv.res += addPageRefresh() return String( fv.res ) ------------------------------------------------------------------------- This SF.net email is sponsored by: Microsoft Defy all challenges. Microsoft(R) Visual Studio 2005. http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/ _______________________________________________ Freevo-cvslog mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/freevo-cvslog
