commit: aac3ed04e3bc5d555b2432ea40dd262335dfd776
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 3 21:02:29 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Sep 3 21:02:29 2015 +0000
URL:
https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aac3ed04
Add support move from cvs to git
pym/tbc/db_mapping.py | 7 ++--
pym/tbc/package.py | 91 ++++++++++++++++++-----------------------------
pym/tbc/qachecks.py | 97 ---------------------------------------------------
pym/tbc/sync.py | 17 ++++-----
4 files changed, 44 insertions(+), 168 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index dbae5a8..82318a1 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -103,7 +103,7 @@ class PackagesEmails(Base):
class PackagesMetadata(Base):
Id = Column('id', Integer, primary_key=True)
PackageId = Column('package_id', Integer,
ForeignKey('packages.package_id'))
- Changlog = Column('changlog', Text)
+ Gitlog = Column('gitlog', Text)
Descriptions = Column('descriptions', Text)
__tablename__ = 'packages_metadata'
@@ -278,6 +278,7 @@ class EbuildsKeywords(Base):
class EbuildsMetadata(Base):
Id = Column('id', Integer, primary_key=True)
EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
- Revision = Column('revision', String(30))
- Descriptions = Column('descriptions', String(200))
+ Commit = Column('commit', String(100))
+ New = Column('new', Boolean, default=False)
+ Descriptions = Column('descriptions', Text)
__tablename__ = 'ebuilds_metadata'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 980b089..6224846 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -5,12 +5,13 @@ from __future__ import print_function
import re
import hashlib
import os
+import git
import portage
from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman
+from tbc.qachecks import check_repoman
from tbc.build_log import check_repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list,
add_old_ebuild, \
@@ -99,9 +100,10 @@ class tbc_package(object):
add_logs(self._session, log_msg, "info",
self._config_id)
log_msg = "C %s:%s ... Fail." % (cpv, repo)
add_logs(self._session, log_msg, "info",
self._config_id)
- ebuild_version_cvs_revision_tree = '0'
+ git_commit = '0'
else:
- ebuild_version_cvs_revision_tree =
get_ebuild_cvs_revision(pkgdir + "/" + package + "-" + ebuild_version_tree +
".ebuild")
+ git_commit = '0'
+ #FIXME The git commit need to be fixed or when $ID get fixed
# Get the ebuild metadata
ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv,
repo)
@@ -123,7 +125,8 @@ class tbc_package(object):
attDict['checksum']= ebuild_version_checksum_tree
attDict['ebuild_version_metadata_tree'] =
ebuild_version_metadata_tree
#attDict['ebuild_version_text_tree'] =
ebuild_version_text_tree[0]
- attDict['ebuild_version_revision_tree'] =
ebuild_version_cvs_revision_tree
+ attDict['git_commit'] = git_commit
+ attDict['New'] = False
attDict['ebuild_version_descriptions_tree'] =
ebuild_version_metadata_tree[7]
return attDict
@@ -161,33 +164,12 @@ class tbc_package(object):
add_logs(self._session,
log_msg, "info", self._config_id)
i = i +1
- def get_changelog_text(self, pkgdir):
- changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir
+ "/ChangeLog")
- spec = 3
- spec_tmp = 1
- changelog_text_tree = ''
- for index, text_line in changelog_text_dict.items():
- if index == max_text_lines:
- if not re.search('^\n', text_line):
- changelog_text_tree =
changelog_text_tree + text_line
- break
- elif re.search('^#', text_line):
- pass
- elif re.search('^\n', text_line) and re.search('^#',
changelog_text_dict[index - 1]):
- pass
- elif re.search('^\n', text_line) and re.search('^\*',
changelog_text_dict[index + 1]):
- changelog_text_tree = changelog_text_tree +
text_line
- spec_tmp = spec_tmp + 1
- spec = spec + 1
- elif re.search('^\n', text_line) and not
re.search('^\*', changelog_text_dict[index + 1]):
- if spec_tmp == spec:
- break
- else:
- spec_tmp = spec_tmp + 1
- changelog_text_tree =
changelog_text_tree + text_line
- else:
- changelog_text_tree = changelog_text_tree +
text_line
- return changelog_text_tree
+ def get_git_log_pkg_text(self, repodir, cp):
+ n = '5'
+ git_log_pkg = ''
+ g = git.Git(repodir)
+ git_log_pkg = g.log('-n ' + n, '--grep=' + cp)
+ return git_log_pkg
def get_package_metadataDict(self, pkgdir, package_id):
# Make package_metadataDict
@@ -195,20 +177,26 @@ class tbc_package(object):
package_metadataDict = {}
md_email_list = []
herd = None
- pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
- attDict['changelog_text'] = self.get_changelog_text(pkgdir)
- tmp_herds = pkg_md.herds()
- if tmp_herds != ():
- attDict['metadata_xml_herds'] = tmp_herds[0]
- md_email_list.append(attDict['metadata_xml_herds'] +
'@gentoo.org')
- for maint in pkg_md.maintainers():
- md_email_list.append(maint.email)
- if md_email_list != []:
- attDict['metadata_xml_email'] = md_email_list
- else:
- log_msg = "Metadata file %s missing Email" % (pkgdir +
"/metadata.xml")
+ try:
+ pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
+ except:
+ log_msg = "Metadata file %s is missing or has errors" %
(pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
attDict['metadata_xml_email'] = False
+ else:
+ attDict['git_log_pkg_text'] =
self.get_git_log_pkg_text(repodir, cp)
+ tmp_herds = pkg_md.herds()
+ if tmp_herds != ():
+ attDict['metadata_xml_herds'] = tmp_herds[0]
+
md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org')
+ for maint in pkg_md.maintainers():
+ md_email_list.append(maint.email)
+ if md_email_list != []:
+ attDict['metadata_xml_email'] = md_email_list
+ else:
+ log_msg = "Metadata file %s missing Email" %
(pkgdir + "/metadata.xml")
+ add_logs(self._session, log_msg, "qa",
self._config_id)
+ attDict['metadata_xml_email'] = False
attDict['metadata_xml_descriptions'] = ''
package_metadataDict[package_id] = attDict
return package_metadataDict
@@ -248,7 +236,6 @@ class tbc_package(object):
self.add_new_build_job_db(ebuild_id_list, packageDict,
config_cpv_listDict)
def get_manifest_checksum_tree(self, pkgdir, cp, repo, mytree):
-
# Get the cp manifest file checksum.
try:
manifest_checksum_tree =
portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
@@ -261,13 +248,6 @@ class tbc_package(object):
log_msg = "C %s:%s ... Fail." % (cp, repo)
add_logs(self._session, log_msg, "error",
self._config_id)
return "0"
- fail_msg = digestcheck(self._mysettings, pkgdir)
- if fail_msg:
- log_msg = "QA: Manifest file has errors. :%s:%s" % (cp,
repo,)
- add_logs(self._session, log_msg, "error",
self._config_id)
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "error",
self._config_id)
- return None
return manifest_checksum_tree
def add_new_package_db(self, cp, repo):
@@ -282,10 +262,7 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-
manifest_checksum_tree =
self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
- if manifest_checksum_tree is None:
- return
package_id = add_new_package_sql(self._session, cp, repo)
@@ -316,6 +293,7 @@ class tbc_package(object):
if
package_metadataDict[package_id]['metadata_xml_descriptions'] !=
packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] =
packageDict[cpv]['ebuild_version_descriptions_tree']
+ packageDict[cpv]['new'] = True
self.add_package(packageDict, package_metadataDict, package_id,
new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -332,10 +310,7 @@ class tbc_package(object):
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
mytree = []
mytree.append(repodir)
-
manifest_checksum_tree =
self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
- if manifest_checksum_tree is None:
- return
# if we NOT have the same checksum in the db update the package
if manifest_checksum_tree != PackageInfo.Checksum:
@@ -361,7 +336,6 @@ class tbc_package(object):
add_old_ebuild(self._session,
old_ebuild_id_list)
log_msg = "C %s:%s ... Done." % (cp,
repo)
add_logs(self._session, log_msg,
"info", self._config_id)
-
else:
log_msg = "QA: Can't get the ebuilds
list. %s:%s" % (cp, repo,)
add_logs(self._session, log_msg,
"info", self._config_id)
@@ -414,6 +388,7 @@ class tbc_package(object):
# N = New ebuild
log_msg = "N %s:%s" % (cpv, repo,)
add_logs(self._session, log_msg,
"info", self._config_id)
+ packageDict[cpv]['new'] = True
elif ebuild_version_checksum_tree !=
ebuild_version_manifest_checksum_db:
# U = Updated ebuild
log_msg = "U %s:%s" % (cpv, repo,)
diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 766b572..67b1448 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -19,103 +19,6 @@ from tbc.repoman.main import repoman_main
from tbc.sqlquerys import get_configmetadata_info, get_config_info,
get_setup_info
import portage
-# Copy of portage.digestcheck() but without the writemsg() stuff
-def digestcheck(mysettings, pkgdir):
- """
- Verifies checksums. Assumes all files have been downloaded.
- @rtype: int
- @return: 1 on success and 0 on failure
- """
- strict=False
- myfiles = []
- justmanifest = None
- mf = None
-
- if mysettings.get("EBUILD_SKIP_MANIFEST") == "1":
- return False
- hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER",
""))
- if hash_filter.transparent:
- hash_filter = None
- if mf is None:
- mf = mysettings.repositories.get_repo_for_location(
- os.path.dirname(os.path.dirname(pkgdir)))
- mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
- try:
- if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not
in mysettings:
- if mf.fhashdict.get("EBUILD"):
- mf.checkTypeHashes("EBUILD",
hash_filter=hash_filter)
- if mf.fhashdict.get("AUX"):
- mf.checkTypeHashes("AUX",
hash_filter=hash_filter)
- if mf.fhashdict.get("MISC"):
- mf.checkTypeHashes("MISC",
ignoreMissingFiles=True,
- hash_filter=hash_filter)
- for f in myfiles:
- ftype = mf.findFile(f)
- if ftype is None:
- if mf.allow_missing:
- continue
- return ("\n!!! Missing digest for '%s'\n") % f
- mf.checkFileHashes(ftype, f, hash_filter=hash_filter)
- except FileNotFound as e:
- return ("\n!!! A file listed in the Manifest could not be
found: %s\n") % str(e)
- except DigestException as e:
- return ("!!! Digest verification failed: %s\nReason: %s\nGot:
%s\nExpected: %s\n") \
- % (e.value[0], e.value[1], e.value[2],
e.value[3])
- if mf.thin or mf.allow_missing:
- # In this case we ignore any missing digests that
- # would otherwise be detected below.
- return False
- # Make sure that all of the ebuilds are actually listed in the Manifest.
- for f in os.listdir(pkgdir):
- pf = None
- if f[-7:] == '.ebuild':
- pf = f[:-7]
- if pf is not None and not mf.hasFile("EBUILD", f):
- return ("!!! A file is not listed in the Manifest:
'%s'\n") % os.path.join(pkgdir, f)
-
- # epatch will just grab all the patches out of a directory, so we have
to
- # make sure there aren't any foreign files that it might grab.
- filesdir = os.path.join(pkgdir, "files")
-
- for parent, dirs, files in os.walk(filesdir):
- try:
- parent = _unicode_decode(parent,
- encoding=_encodings['fs'], errors='strict')
- except UnicodeDecodeError:
- parent = _unicode_decode(parent,
- encoding=_encodings['fs'], errors='replace')
- return ("!!! Path contains invalid character(s) for
encoding '%s': '%s'\n") % (_encodings['fs'], parent)
- for d in dirs:
- d_bytes = d
- try:
- d = _unicode_decode(d,
- encoding=_encodings['fs'],
errors='strict')
- except UnicodeDecodeError:
- d = _unicode_decode(d,
- encoding=_encodings['fs'],
errors='replace')
- return ("!!! Path contains invalid character(s)
for encoding '%s': '%s'\n") % (_encodings['fs'], os.path.join(parent, d))
- if d.startswith(".") or d == "CVS":
- dirs.remove(d_bytes)
- for f in files:
- try:
- f = _unicode_decode(f,
- encoding=_encodings['fs'],
errors='strict')
- except UnicodeDecodeError:
- f = _unicode_decode(f,
- encoding=_encodings['fs'],
errors='replace')
- if f.startswith("."):
- continue
- f = os.path.join(parent, f)[len(filesdir) + 1:]
- return ("!!! File name contains invalid
character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], f)
- if f.startswith("."):
- continue
- f = os.path.join(parent, f)[len(filesdir) + 1:]
- file_type = mf.findFile(f)
- if file_type != "AUX" and not f.startswith("digest-"):
- return ("!!! A file is not listed in the
Manifest: '%s'\n") % os.path.join(filesdir, f)
- return False
-
-
def check_file_in_manifest(pkgdir, mysettings, portdb, cpv,
build_use_flags_list, repo):
myfetchlistdict = portage.FetchlistDict(pkgdir, mysettings, portdb)
my_manifest = portage.Manifest(pkgdir, mysettings['DISTDIR'],
fetchlist_dict=myfetchlistdict, manifest1_compat=False, from_scratch=False)
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 12c4c9f..610f357 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -74,20 +74,16 @@ def git_sync_main(session):
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
+ log_msg = "Checking repo %s" % (reponame)
+ add_logs(session, log_msg, "info", config_id)
info_list, repouptodate = git_fetch(repo)
if not repouptodate:
cp_list = []
attr = {}
# We check for Manifest changes and add the package to
a list
- for diff_line in repo.git.diff('origin').splitlines():
- if re.search("^diff --git.*/Manifest",
diff_line):
- diff_line2 = re.split(' b/',
re.sub('diff --git', '', diff_line))
- diff_line3 = re.sub(' a/', '',
diff_line2[0])
- if diff_line3 == diff_line2[1] or
"Manifest" in diff_line3:
- cp = re.sub('/Manifest', '',
diff_line3)
- cp_list.append(cp)
- else:
- cp = re.sub('/Manifest', '',
diff_line2[1])
+ for diff_line in repo.git.diff('origin',
'--name-only').splitlines():
+ if re.search("Manifest$", diff_line):
+ cp = re.sub('/Manifest', '',
diff_line)
cp_list.append(cp)
attr['cp_list'] = cp_list
repo_cp_dict[reponame] = attr
@@ -95,7 +91,8 @@ def git_sync_main(session):
else:
log_msg = "Repo %s is up to date" % (reponame)
add_logs(session, log_msg, "info", config_id)
-
+ log_msg = "Checking repo %s Done" % (reponame)
+ add_logs(session, log_msg, "info", config_id)
# Need to add a clone of profiles/base for reading the tree
try:
os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
