Using the wiki results script on wiki.canonical.com
-------- Original Message --------
Subject: Parse Wiki Script
Date: Sat, 21 Feb 2009 12:16:00 -0800
From: Leann Ogasawara <leann.ogasaw...@canonical.com>
To: Chris Gregan <chris.gre...@canonical.com>
Hi Chris,
I've got the script working, you just need to do a small initial one
time setup which I'll describe below. I'm basically using the same code
that editmoin has.
1) create a ~/.moin_ids file which contains the following line:
https://wiki.canonical.com (paste your MOIN_ID here)
You can find what your MOIN_ID is by first making sure you've
logged into https://wiki.canonical.com . Assuming you use
Firefox, from your browser go to Edit->Preferences . Click on
the Privacy icon then click on the "Show Cookies..." button.
Scroll down to wiki.canonical.com and expand the view. Select
the MOIN_SESSION cookie. Your MOIN_ID is what's in the
"Content:" field (it's a long string of letters and numbers).
Copy and paste that to the .moin_ids file so it'll look
something similar to:
https://wiki.canonical.com f7asd89asdfa-ys_k7oof2-_d-m
That's pretty much all you need to do before running the script. You'll
need to update the urls part of the script to include all the new links
for the test cases. I only put two in there for now to test with. Let
me know if this new script works or not.
Thanks,
Leann
--
Chris Gregan
QA Manager
OEM Services
Canonical USA Inc.
cgre...@canonical.com
cgre...@freenode
M-240-422-9224
#!/usr/bin/env python
"""
Site-scraping script
"""
import urllib
import re
import sys
import getopt
import string
import os
import time
IDFILENAME = os.path.expanduser("~/.moin_ids")
def get_id(moinurl):
if os.path.isfile(IDFILENAME):
file = open(IDFILENAME)
for line in file.readlines():
line = line.strip()
if line and line[0] != "#":
tokens = line.split()
if len(tokens) > 1:
url, id = tokens[:2]
else:
url, id = tokens[0], None
if moinurl.startswith(url):
return id
return None
def help():
print "Usage: python parse-wiki.py [options]"
print "options:"
print " --help"
print " --url=url[,url,url. . .]"
print " --format=[wiki|csv]"
print " --print-last"
#default parameter options
format = "wiki"
print_last = 0
urls = ["https://wiki.canonical.com/OEMServices/QA/Testing/Elvis/Elvis-boot","https://wiki.canonical.com/OEMServices/QA/Testing/Elvis/Elvis-connect";,]
try:
opts, args = getopt.getopt(sys.argv[1:], "hu:f:", ["help", "url=", "format=", "print-last"])
for opt,arg in opts:
if opt in ("h", "--help"):
raise
if opt in ("u", "--url"):
urls = arg.split(",")
if opt in ("f", "--format"):
if arg == "csv":
format = "csv"
if opt in ("p", "--print-last"):
print_last = 1
except:
help()
sys.exit(1)
for url in urls:
url_orig = url
id = get_id(url)
url = url + "?action=raw"
urlopener = urllib.FancyURLopener()
urlopener.addheader("Cookie", "MOIN_SESSION=\"%s\"" % id)
filename, headers = urlopener.retrieve(url)
wiki_url = open(filename)
reg_exp = "===(.*?)==="
reg_exp2 = "(\|\|.*\|\|)"
wiki_table = []
csv_table = []
csv = ""
wiki = ""
for line in wiki_url.readlines():
test_case = re.search(reg_exp, line)
table_row = re.search(reg_exp2, line)
if test_case != None:
wiki += "\n=== %s ===\n" % (test_case.group(1))
wiki += "%s\n" % (url_orig)
wiki_table.append(test_case.group(1).strip() + "\n")
wiki_table.append(url_orig + "\n")
wiki_table.append("")
csv += test_case.group(1).strip() + "\n"
csv += "\t" + url_orig + "\n"
csv_table.append(test_case.group(1).strip() + "\n")
csv_table.append("\t" + url_orig + "\n")
csv_table.append("")
elif table_row != None:
wiki += table_row.group(1) + "\n"
wiki_table[-1] = table_row.group(1) + "\n"
line = line.strip()
line = line.lstrip("||")
cells = line.split("||")
count = 0
for x in cells:
cells[count] = x.strip()
count = count + 1
csv += "\t" + string.join(cells, ",") + "\n"
csv_table[-1] = "\t" + string.join(cells, ",") + "\n"
if format == "csv":
if print_last:
print string.join(csv_table, "")
else:
print csv
else:
if print_last:
print string.join(wiki_table, "")
else:
print wiki
time.sleep(15)
_______________________________________________
Mailing list: https://launchpad.net/~oem-qa
Post to : oem-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~oem-qa
More help : https://help.launchpad.net/ListHelp
