YouTube changed the format of the JavaScript in the web page that I was
scraping, but not in an obfuscationalist way.
It just requires parsing out a JavaScript dict of strings and converting
it into a url parameter, instead of grabbing the raw url parameter as
before.
Please try this patch to youtube.jsp videoGetFlvUrl.
-Don
public void videoGetFlvUrl(
String id,
Document result)
{
String pageUrl =
"http://www.YouTube.com/watch?v=";
pageUrl += id;
BufferedReader inputFile = null;
try {
URL u = new URL(pageUrl);
inputFile =
new BufferedReader(
new InputStreamReader(
u.openStream()));
} catch (Exception e) {
reportError("Could not load url.", result);
return;
} // try
while (true) {
String line = null;
try {
line = inputFile.readLine();
} catch (IOException e) {
line = null;
}
if (line == null) {
break;
}
String target =
"var swfArgs = {";
int start =
line.indexOf(target);
if (start == -1) {
continue;
}
start += target.length();
String params = "";
while (true) {
int colon =
line.indexOf(":", start);
if (colon == -1) {
break;
}
String varName =
line.substring(start, colon);
int openQuote =
line.indexOf("'", colon);
if (openQuote == -1) {
break;
}
int closeQuote =
line.indexOf("'", openQuote + 1);
if (closeQuote == -1) {
break;
}
String varValue =
line.substring(openQuote + 1, closeQuote);
if (params.length() > 0) {
params += "&";
}
try {
varName = URLEncoder.encode(varName, "UTF-8");
varValue = URLEncoder.encode(varValue, "UTF-8");
} catch (UnsupportedEncodingException e) {}
params +=
varName + "=" + varValue;
int comma =
line.indexOf(",", closeQuote + 1);
if (comma == -1) {
break;
}
start = comma + 1;
}
String url =
"http://www.YouTube.com/get_video.php?" +
params +
"&.flv";
Element resultEl =
new Element("videoGetFlvUrlResult");
resultEl.setAttribute("id", id);
resultEl.setAttribute("url", url);
result.setRootElement(
resultEl);
return;
}
reportError("Could not find SWFObject at url " + pageUrl, result);
}