Author: plessy Date: 2010-04-10 01:38:35 +0000 (Sat, 10 Apr 2010) New Revision: 1733
Added: udd/scripts/fetch_bibref.sh udd/udd/bibref_gatherer.py Modified: udd/config-org.yaml Log: Bibliographic reference gatherer, untested and still in development. Somewhat discussed in the following thread and others: http://lists.debian.org/msgid-search/[email protected] Modified: udd/config-org.yaml =================================================================== --- udd/config-org.yaml 2010-03-27 09:00:41 UTC (rev 1732) +++ udd/config-org.yaml 2010-04-10 01:38:35 UTC (rev 1733) @@ -19,6 +19,7 @@ ddtp: module udd.ddtp_gatherer ftpnew: module udd.ftpnew_gatherer screenshots: module udd.screenshot_gatherer + bibref: module udd.bibref_gatherer dehs: module udd.dehs_gatherer ldap: module udd.ldap_gatherer wannabuild: module udd.wannabuild_gatherer @@ -557,6 +558,14 @@ table: screenshots screenshots_json: /org/udd.debian.org/mirrors/screenshots/screenshots.json +bibref: + type: bibref + update-command: /org/udd.debian.org/udd/scripts/fetch_bibref.sh + path: /org/udd.debian.org/mirrors/bibref + cache: /org/udd.debian.org/mirrors/cache + table: bibref + bibref_yaml: /org/udd.debian.org/mirrors/bibref/bibref.yaml + wannabuild: type: wannabuild wbdb: "dbname=wanna-build host=localhost port=5433 user=guest" Added: udd/scripts/fetch_bibref.sh =================================================================== --- udd/scripts/fetch_bibref.sh (rev 0) +++ udd/scripts/fetch_bibref.sh 2010-04-10 01:38:35 UTC (rev 1733) @@ -0,0 +1,5 @@ +#!/bin/sh +TARGETDIR=/org/udd.debian.org/mirrors/bibref +mkdir -p $TARGETDIR +rm -rf $TARGETDIR/* +wget -q http://upstream-metadata.debian.net/for_UDD/biblio.yaml -O ${TARGETDIR}/bibref.yaml Property changes on: udd/scripts/fetch_bibref.sh ___________________________________________________________________ Added: svn:executable + * Copied: udd/udd/bibref_gatherer.py (from rev 1732, udd/udd/screenshot_gatherer.py) =================================================================== --- udd/udd/bibref_gatherer.py (rev 0) +++ udd/udd/bibref_gatherer.py 2010-04-10 01:38:35 UTC (rev 1733) @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +""" +This script imports bibliographic references from upstream-metadata.debian.net. +""" + +from gatherer import gatherer +from sys import stderr, exit + +online=0 + +def get_gatherer(connection, config, source): + return bibref_gatherer(connection, config, source) + +class screenshot_gatherer(gatherer): + """ + Bibliographic references from upstream-metadata.debian.net. + """ + + def __init__(self, connection, config, source): + gatherer.__init__(self, connection, config, source) + self.assert_my_config('table') + my_config = self.my_config + + cur = self.cursor() + query = "DELETE FROM %s" % my_config['table'] + cur.execute(query) + query = """PREPARE bibref_insert (text, text, text) AS + INSERT INTO %s + (package, key, value) + VALUES ($1, $2, $3)""" % (my_config['table']) + cur.execute(query) + + pkg = None + + def run(self): + my_config = self.my_config + #start harassing the DB, preparing the final inserts and making place + #for the new data: + cur = self.cursor() + + bibref_file = my_config['bibref_yaml'] + fp = open(bibref_file, 'r') + result = fp.read() + fp.close() + + for res in safe_load_all(result): + package, key, value = res + query = """EXECUTE bibref_insert + (%(package)s, %(key)s, %(value)s)""" + try: + cur.execute(query, res) + except UnicodeEncodeError, err: + print >>stderr, "Unable to inject data for package %s. %s" % (res['name'], err) + print >>stderr, "-->", res + cur.execute("DEALLOCATE bibref_insert") + cur.execute("ANALYZE %s" % my_config['table']) + +if __name__ == '__main__': + main() + +# vim:set et tabstop=2: + Property changes on: udd/udd/bibref_gatherer.py ___________________________________________________________________ Added: svn:mergeinfo + _______________________________________________ Collab-qa-commits mailing list [email protected] http://lists.alioth.debian.org/mailman/listinfo/collab-qa-commits
