Author: neronus-guest
Date: 2008-07-26 13:06:58 +0000 (Sat, 26 Jul 2008)
New Revision: 936
Modified:
udd/src/udd/bugs_gatherer.pl
udd/src/udd/gatherer.py
udd/src/udd/packages_gatherer.py
udd/src/udd/sources_gatherer.py
udd/src/udd/testing_migrations_gatherer.py
Log:
* Removed bugs gatherer optimization
* Code cleanups
* comments
Modified: udd/src/udd/bugs_gatherer.pl
===================================================================
--- udd/src/udd/bugs_gatherer.pl 2008-07-26 12:46:07 UTC (rev 935)
+++ udd/src/udd/bugs_gatherer.pl 2008-07-26 13:06:58 UTC (rev 936)
@@ -1,5 +1,5 @@
#!/usr/bin/perl
-# Last-Modified: <Thu Jul 24 17:54:33 2008>
+# Last-Modified: <Sat Jul 26 12:01:01 2008>
use strict;
use warnings;
@@ -45,33 +45,17 @@
# We want to commit the transaction as a hole at the end
$dbh->{AutoCommit} = 0;
- # We want to know the last modification date of the bugs
- my $sth = $dbh->prepare("SELECT MAX(last_modified) FROM bugs");
- $sth->execute();
- my $max_last_modified = $sth->fetchrow_array();
+ #delete the bug, if it exists
+ $dbh->prepare("DELETE FROM bugs")->execute();
+ $dbh->prepare("DELETE FROM bug_found_in")->execute();
+ $dbh->prepare("DELETE FROM bug_fixed_in")->execute();
+ $dbh->prepare("DELETE FROM bug_merged_with")->execute();
- #$dbh->prepare("DELETE FROM bugs")->execute();
- #$dbh->prepare("DELETE from bug_found_in")->execute();
- #$dbh->prepare("DELETE from bug_fixed_in")->execute();
- #$dbh->prepare("DELETE FROM bug_merged_with")->execute();
-
- my %pkgsrcmap = %{getpkgsrc()};
-
- my $counter = 0;
-
- my ($year, $month, $day, $hour, $minute, $second) =
parse_time($max_last_modified);
- $max_last_modified = timelocal($second, $minute, $hour, $day, $month-1,
$year);
-
# Read all bugs
foreach my $bug_nr (get_bugs()) {
# Fetch bug using Debbugs
my %bug = %{get_bug_status($bug_nr)};
-
- # Check if the bug was last changed since we updated the DB
- next if $max_last_modified > $bug{log_modified};
-
- print "Working bug $bug_nr\n";
-
+
# Convert data where necessary
my $date = strftime("%Y-%m-%d %T", localtime($bug{date}));
my $log_modified = strftime("%Y-%m-%d %T",
localtime($bug{log_modified}));
@@ -114,16 +98,11 @@
$present_in_unstable = 'TRUE';
}
- #delete the bug, if it exists
- $dbh->prepare("DELETE FROM bugs WHERE id = $bug_nr")->execute();
- $dbh->prepare("DELETE FROM bug_found_in WHERE id =
$bug_nr")->execute();
- $dbh->prepare("DELETE FROM bug_fixed_in WHERE id =
$bug_nr")->execute();
- $dbh->prepare("DELETE FROM bug_merged_with WHERE bug =
$bug_nr")->execute();
# Insert data into bugs table
my $query = "INSERT INTO bugs VALUES ($bug_nr, '$bug{package}',
$source, '$date', \
- $bug{pending}, '$bug{severity}', '$bug{keywords}',
$bug{originator}, $bug{owner}, \
- $bug{subject}, '$log_modified',
$present_in_stable,
+ E$bug{pending}, '$bug{severity}',
'$bug{keywords}', E$bug{originator}, E$bug{owner}, \
+ E$bug{subject}, '$log_modified',
$present_in_stable,
$present_in_testing,
$present_in_unstable)";
# Execute insertion
my $sth = $dbh->prepare($query);
@@ -142,7 +121,6 @@
$query = "INSERT INTO bug_merged_with VALUES ($bug_nr,
$mergee)";
$dbh->prepare($query)->execute() or die $!;
}
- print "$counter\n" if ++$counter % 500 == 0;
}
$dbh->commit();
Modified: udd/src/udd/gatherer.py
===================================================================
--- udd/src/udd/gatherer.py 2008-07-26 12:46:07 UTC (rev 935)
+++ udd/src/udd/gatherer.py 2008-07-26 13:06:58 UTC (rev 936)
@@ -1,15 +1,20 @@
-"""
-This is the base class of all gatherers which want to use the python
-interface to be called by the dispatcher
-"""
+# This file is part of the Ultimate Debian Database project
+class gatherer:
+ """
+ This is the base class of all gatherers which want to use the python
+ interface to be called by the dispatcher
-class gatherer:
+ Attributes:
+ connection: The connection to the SQL database
+ config: The hashmap representing the configuration"""
def __init__(self, connection, config):
self.connection = connection
self.config = config
def run(self, source):
+ """Called by the dispatcher for a source"""
raise NotImplementedError
def cursor(self):
+ """Return the cursor for the current connection"""
return self.connection.cursor()
Modified: udd/src/udd/packages_gatherer.py
===================================================================
--- udd/src/udd/packages_gatherer.py 2008-07-26 12:46:07 UTC (rev 935)
+++ udd/src/udd/packages_gatherer.py 2008-07-26 13:06:58 UTC (rev 936)
@@ -1,5 +1,6 @@
-#/usr/bin/env python
-# Last-Modified: <Sun Jun 29 11:49:49 2008>
+# /usr/bin/env python
+# Last-Modified: <Sat Jul 26 12:58:43 2008>
+# This file is a part of the Ultimate Debian Database project
import debian_bundle.deb822
import gzip
@@ -15,7 +16,12 @@
return packages_gatherer(connection, config)
class packages_gatherer(gatherer):
+ "This class imports the data from Packages.gz files into the database"
# For efficiency, these are dictionaries
+ # mandatory: list of fields which each package has to provide
+ # non_mandatory: list of fields which are possibly provided by packages
+ # ignorable: fields which are not useful for the database,
+ # but for which no warning should be printed
mandatory = {'Package': 0, 'Version': 0, 'Architecture': 0, 'Maintainer': 0,
'Description': 0}
non_mandatory = {'Source': 0, 'Essential': 0, 'Depends': 0, 'Recommends': 0,
@@ -140,11 +146,10 @@
aux.debug = self.config['general']['debug']
- # Get distribution ID. If it does not exist, create it
+ # Get distribution ID
self._distr = src_cfg['distribution']
cur = self.cursor()
- #cur.execute("PREPARE pkg_insert AS INSERT INTO pkgs (name, distr_id,
arch_id, version, src_id) VALUES ($1, $2, $3, $4, $5);")
# For every part and every architecture, import the packages into the DB
for comp in src_cfg['components']:
Modified: udd/src/udd/sources_gatherer.py
===================================================================
--- udd/src/udd/sources_gatherer.py 2008-07-26 12:46:07 UTC (rev 935)
+++ udd/src/udd/sources_gatherer.py 2008-07-26 13:06:58 UTC (rev 936)
@@ -1,5 +1,6 @@
#/usr/bin/env python
-# Last-Modified: <Thu Jul 24 12:26:00 2008>
+# Last-Modified: <Sat Jul 26 12:59:30 2008>
+# This file is a part of the Ultimate Debian Database project
import debian_bundle.deb822
import gzip
@@ -15,6 +16,7 @@
return sources_gatherer(connection, config)
class sources_gatherer(gatherer):
+ "This class imports the data from Sources.gz files into the database"
mandatory = {'Format': 0, 'Maintainer': 0, 'Package': 0, 'Version': 0,
'Files': 0}
non_mandatory = {'Uploaders': 0, 'Binary': 0, 'Architecture': 0,
'Standards-Version': 0, 'Homepage': 0, 'Build-Depends': 0,
Modified: udd/src/udd/testing_migrations_gatherer.py
===================================================================
--- udd/src/udd/testing_migrations_gatherer.py 2008-07-26 12:46:07 UTC (rev
935)
+++ udd/src/udd/testing_migrations_gatherer.py 2008-07-26 13:06:58 UTC (rev
936)
@@ -1,5 +1,7 @@
-# Last-Modified: <Sat Jul 12 17:31:22 2008>
+# Last-Modified: <Sat Jul 26 13:03:16 2008>
+# This file is a part of the Ultimate Debian Database Project
+
from gatherer import gatherer
from aux import ConfigException, quote
from time import strptime
@@ -11,6 +13,9 @@
class testing_migrations_gatherer(gatherer):
+ """This class imports testing migrations data into the database.
+
+ For the files, see http://qa.debian.org/~lucas/testing-status.raw""";
def __init__(self, connection, config):
gatherer.__init__(self, connection, config)
@@ -35,8 +40,6 @@
exec "is_null = %s == ZERO_DATE" % field
if is_null:
exec "%s = 'NULL'" % field
- #else:
- #exec field + " = strptime('%Y-%m-%d', " + field + ")"
else:
exec "%s = quote(%s)" % (field, field)
_______________________________________________
Collab-qa-commits mailing list
[email protected]
http://lists.alioth.debian.org/mailman/listinfo/collab-qa-commits
