YouTube changed the format of the JavaScript in the web page that I was scraping, but not in an obfuscationalist way. It just requires parsing out a JavaScript dict of strings and converting it into a url parameter, instead of grabbing the raw url parameter as before.

Please try this patch to youtube.jsp videoGetFlvUrl.

   -Don


   public void videoGetFlvUrl(
       String id,
       Document result)
   {
       String pageUrl =
           "http://www.YouTube.com/watch?v=";;
       pageUrl += id;

       BufferedReader inputFile = null;
       try {
           URL u = new URL(pageUrl);
           inputFile =
               new BufferedReader(
                   new InputStreamReader(
                       u.openStream()));
       } catch (Exception e) {
           reportError("Could not load url.", result);
           return;
       } // try

       while (true) {
           String line = null;

           try {
               line = inputFile.readLine();
           } catch (IOException e) {
               line = null;
           }

           if (line == null) {
               break;
           }

           String target =
               "var swfArgs = {";
           int start =
               line.indexOf(target);
           if (start == -1) {
               continue;
           }

           start += target.length();

           String params = "";

           while (true) {
               int colon =
                   line.indexOf(":", start);
               if (colon == -1) {
                   break;
               }

               String varName =
                   line.substring(start, colon);

               int openQuote =
                   line.indexOf("'", colon);
               if (openQuote == -1) {
                   break;
               }

               int closeQuote =
                   line.indexOf("'", openQuote + 1);
               if (closeQuote == -1) {
                   break;
               }

               String varValue =
                   line.substring(openQuote + 1, closeQuote);

               if (params.length() > 0) {
                   params += "&";
               }

               try {
                   varName = URLEncoder.encode(varName, "UTF-8");
                   varValue = URLEncoder.encode(varValue, "UTF-8");
               } catch (UnsupportedEncodingException e) {}

               params +=
                   varName + "=" + varValue;

               int comma =
                   line.indexOf(",", closeQuote + 1);

               if (comma == -1) {
                   break;
               }

               start = comma + 1;
           }

           String url =
               "http://www.YouTube.com/get_video.php?"; +
               params +
               "&.flv";

           Element resultEl =
               new Element("videoGetFlvUrlResult");

           resultEl.setAttribute("id", id);
           resultEl.setAttribute("url", url);

           result.setRootElement(
               resultEl);

           return;
       }

       reportError("Could not find SWFObject at url " + pageUrl, result);
   }


Reply via email to