Author: saq Date: Tue Sep 13 18:48:27 2005 New Revision: 6373 Added: cvs2svn-migration/trunk/ cvs2svn-migration/trunk/README cvs2svn-migration/trunk/builder (contents, props changed) cvs2svn-migration/trunk/invoke_cvs2svn (contents, props changed) cvs2svn-migration/trunk/macros cvs2svn-migration/trunk/migrator (contents, props changed) Log: - initial checkin
Added: cvs2svn-migration/trunk/README ============================================================================== --- (empty file) +++ cvs2svn-migration/trunk/README Tue Sep 13 18:48:27 2005 @@ -0,0 +1,70 @@ +=== + +Steps to migrate the PLD repo cvs->svn + +1. Disable write access to the CVS repo +2. Use migrator to reorganize the repo according to the dir-per-package scheme +3. Perform some manual clean-ups. +4. Use cvs2svn to generate svn entries and revisions. +5. Perform some manual clean-ups. +6. Celebrate. + +After step 3 you can try working with CVS on the new structure (read-only), +you will have to `mkdir CVSROOT` for that. + +=== + +migrator quick docs + +What it does: creates a copy of the existing PLD repo (or its rsync copy), +reorganizing the files on the fly. The result is a flat structure: each +package has a directory of its own, S{PEC,OURCE}S are no more. Don't worry +about disk space, it's a cheap (hardlink-based) copy. Do place the source +and target on the same filesystem. + +For configuration, see the sources. For testing purposes you can limit +yourself to a subset of the specs; again, see the sources. + +A source is considered to belong to a given package if it's a Source or a +Patch of at least one revision of a given foo.spec,v (malformed revisions +are discarded). rpm parsing of each single release is probably the most +costly operation. Converting a*.spec took 30 minutes. + +The above rule means that if a source belongs to various specs, it will +get duplicated in the resulting repo. Sources which don't belong anywhere +go to a common directory (that is, no file gets omitted, except for files +that weren't valid CVS-controlled files in the first place). + +=== + +migrator open issues: + +- There are files in SPECS which aren't valid specs (builder,v for + example). Find a cosy place for them. +- There are files in SOURCES which apparently don't belong to any package. + Find a good place for them (bitbucket is an option). + +=== + +Other open issues: + +- How to write %changelog under svn? What to do with the existing entries? +- What tasks are necessary in steps 3 and 5 above? +- What needs to be done to adapt src builder? +- What about the filter_tags.sh rules? +- What directory structure for tags and branches? cvs2svn gives: +trunk/MathReader/MathReader.spec +tags/auto-ac-MathPlanner-3_1_3-1/MathPlanner/MathPlanner.spec +branches/RA-branch/MathPlanner/MathPlanner.spec + +=== + +Changes to the development environment: + +- topdir/S{PEC,OURCE}S are no more. topdir/{BUILD,{,S}RPMS} will stay there +- topdir/foo looks like the right checkout spot for foo.spec and its sources +- topdir/foo/foo-0.00.tar for distfiles +- need new macros to find the sources in the new place +- builder will never be the same again + +=== Added: cvs2svn-migration/trunk/builder ============================================================================== --- (empty file) +++ cvs2svn-migration/trunk/builder Tue Sep 13 18:48:27 2005 @@ -0,0 +1,43 @@ +#!/bin/sh + +# Note: it's only useful for the conversion proof-of-concept. + +# Hint: -r branches/AC-branch or -r tags/auto-ac-bofh-0_2-4 + +SVNROOT="file:///home/users/saq/tmp/svn" +TAGDIR="trunk" +PACKAGE="" + +BUILD="yes" + +while test $# -gt 0; do + case "$1" in + -g ) + BUILD="no";shift;; + -r ) + TAGDIR="$2";shift;shift;; + * ) + PACKAGE="$1";shift;; + esac +done +if [ "$PACKAGE" = "" ]; then + echo "Usage: builder [-g] [-r foo] package" + exit 1 +fi +cd $(rpmbuild --eval %_topdir 2>/dev/null) +svn co $SVNROOT/$TAGDIR/$PACKAGE +cd $PACKAGE +dump="$(rpmbuild --nodigest --nosignature --nodeps --define 'prep %dump' $PACKAGE.spec 2>&1)" +source0url=$(echo "$dump"|awk '/SOURCEURL0/ {print $3}') +source0=$(echo "$dump"|awk '/SOURCE0/ {print $3}') +#echo S $source0url $source0 +have_md5=$(md5sum $source0|awk '{print $1}') +need_md5=$(cat $PACKAGE.spec|awk '/ Source0-md5/{print $3}') +#echo M $have_md5 $need_md5 +if [ "$have_md5" != "$need_md5" ]; then + rm -f $source0 + wget http://distfiles.pld-linux.org/by-md5/$(echo $need_md5|sed -e 's|^\(.\)\(.\)|\1/\2/&|')/$(basename $source0) +fi +if [ "$BUILD" = "yes" ]; then + rpmbuild -bb $PACKAGE.spec +fi Added: cvs2svn-migration/trunk/invoke_cvs2svn ============================================================================== --- (empty file) +++ cvs2svn-migration/trunk/invoke_cvs2svn Tue Sep 13 18:48:27 2005 @@ -0,0 +1,14 @@ +#!/bin/sh + +OPTS="--encoding=ISO-8859-2" + +# Checking for tag/branch mismatches... +# ERROR: The following symbols are tags in some files and branches in others. +# yees, the beauty of PLD +OPTS="$OPTS --force-branch=DEVEL --force-branch=RA-branch --force-branch=AC-branch" +OPTS="$OPTS --force-branch=RA-branch_general" +OPTS="$OPTS --force-branch=LINUX_2_6 --force-branch=LINUX_2_4 --force-branch=LINUX_2_2_DEVEL" +OPTS="$OPTS --force-branch=GNOME_2_0 --force-branch=rpm3" +# you'll find many others + +exec cvs2svn $OPTS -s /home/users/saq/tmp/svn /home/users/saq/tmp/repo Added: cvs2svn-migration/trunk/macros ============================================================================== --- (empty file) +++ cvs2svn-migration/trunk/macros Tue Sep 13 18:48:27 2005 @@ -0,0 +1,7 @@ +# The macros that need to be modified. Not a very long list. + +# A cool feature: rpmbuild --rebuild foo...src.rpm mkdirs those if they +# don't exist + +%_sourcedir %{_topdir}/%{name} +%_specdir %{_topdir}/%{name} Added: cvs2svn-migration/trunk/migrator ============================================================================== --- (empty file) +++ cvs2svn-migration/trunk/migrator Tue Sep 13 18:48:27 2005 @@ -0,0 +1,164 @@ +#!/usr/bin/python + +# requires: cvs2svn, rcs, rpm-build + +# source +cvsdir="/home/users/saq/PLD/repo" +# target +tmpcvsdir="/home/users/saq/tmp/repo" +# rpm can't read a spec from stdin +tmpspecfile="/home/users/saq/tmp/foo.spec" +# reasons of failures +logdir="/home/users/saq/tmp/migrator-log" + +# only those SPECS/ files will be copied +spec_re="^c.*" +#spec_re=".*" +# only those SOURCES/ files will be considered for SOURCES.old +# (files for specs are sought among all sources) +source_re="^c.*" +#source_re=".*" + +# internals follow + +import cvs2svn_rcsparse +import os +import re +from subprocess import Popen,PIPE + +spec_re=re.compile(spec_re) +source_re=re.compile(source_re) + +def cvspackagedir(package): + dir=tmpcvsdir+"/"+package + return dir + +specsdir=cvsdir+"/SPECS" +sourcesdir=cvsdir+"/SOURCES" +specsolddir=tmpcvsdir+"/SPECS.old" +sourcesolddir=tmpcvsdir+"/SOURCES.old" + +# utilities + +class Sink(cvs2svn_rcsparse.Sink): + def __init__(self): + self.revs=[] + def define_revision(self,rev,*args): + self.revs.append(rev) + +def get_revisions(f): + """Get all revision numbers from a ,v file""" + sink=Sink() + cvs2svn_rcsparse.parse(f,sink) + return sink.revs + +def mkdir(dir): + # might exist already + try: + os.makedirs(dir) + except OSError: + pass + +def ln(oldpath,newdir): + lastdir,file=oldpath.split("/")[-2:] + if lastdir=="Attic": newdir+="/Attic" + newpath="%s/%s"%(newdir,file) + # the noble exception of nps.spec + try: + link=os.readlink(oldpath) + oldpath+="/"+link + except OSError: + # not a symlink + pass + os.link(oldpath,newpath) + +def listdir(dir): + """For a given CVS module directory, get a mapping: + version controlled file->path to its ,v""" + # ERROR: A CVS repository cannot contain both /tmp/cvs/SPECS/Mis.spec,v and /tmp/cvs/SPECS/Attic/Mis.spec,v + # PLD has always been exceptional :) + # Having no better idea, discard the Attic version. As well as all non-,v files. + livefiles=os.listdir(dir) + atticfiles=os.listdir(dir+"/Attic") + files={} + for f in atticfiles: + if not f.endswith(",v"): continue + files[f[:-2]]="%s/Attic/%s"%(dir,f) + for f in livefiles: + if not f.endswith(",v"): continue + files[f[:-2]]="%s/%s"%(dir,f) + return files + +def spec2sources(f): + """ + Get a sequence of the sources that belong to the given spec. + This means checking each revision to follow the sources' list changes. + """ + assert f.endswith(",v") + all_sources=set() + for rev in get_revisions(file(f)): + # This part would have been easier in another language + p1=Popen(["co","-p","-r"+rev,f[:-2]],stdout=PIPE,stderr=file("/dev/null","w")) + # post-processing to maximize the chance of a successful parse + p2=Popen(["sed",r"/^Icon:/d;/^Serial:/d;/^%define.*_kernel_ver/d;s/^Copyright:/License:/"], + stdin=p1.stdout,stdout=file(tmpspecfile,"w") + ) + if p1.wait()!=0 or p2.wait()!=0: + print "WARN: [EMAIL PROTECTED]: checkout failed"%(f,rev) + continue + + cmd1=["rpmbuild","--nodigest","--nosignature","--nodeps", + "--define","_kernel_ver_str 2.0.28", + "--define","prep %dump",tmpspecfile] + p1=Popen(cmd1,stderr=PIPE) + p2=Popen(["awk",'/SOURCE[0-9]+|PATCH[0-9]+/ {sub(".*/","",$3);print $3}'],stdin=p1.stderr,stdout=PIPE) + # strip trailing \n + sources=[line[:-1] for line in p2.stdout] + if p1.wait()!=0 or p2.wait()!=0: + what="[EMAIL PROTECTED]"%(f.split('/')[-1],rev) + print "WARN: %s: unable to parse, see log for details"%(what,) + Popen(cmd1,stderr=file("%s/%s.log"%(logdir,what),"w")).wait() + continue + all_sources.update(sources) + return all_sources + +# let's go + +specs_all=listdir(specsdir) +specs_used=set() +sources_all=listdir(sourcesdir) +sources_used=set() + +print "0. Cleanup" +os.system("rm -rf %s"%(tmpcvsdir,)) +os.system("rm -rf %s"%(logdir,)) +mkdir(logdir) + +print "1. Reorganizing CVS repo" +for spec,path in specs_all.iteritems(): + if spec_re.match(spec) is None: continue + if not spec.endswith(".spec"): + print "WARN: what the hell is SPECS/%s?"%(spec,) + continue + package=spec[:-5] + dir=cvspackagedir(package) + mkdir(dir+"/Attic") + ln(path,dir) + sources=spec2sources(path) + for source in sources: + if not source in sources_all: continue + ln(sources_all[source],dir) + sources_used.update(sources) + specs_used.add(spec) + +print "2. Keeping remaining SPECS and SOURCES" +mkdir(specsolddir+"/Attic") +mkdir(sourcesolddir+"/Attic") +for spec,path in specs_all.iteritems(): + if spec in specs_used: continue + if spec_re.match(spec) is None: continue + ln(path,specsolddir) +for source,path in sources_all.iteritems(): + if source in sources_used: continue + if source_re.match(source) is None: continue + ln(path,sourcesolddir) _______________________________________________ pld-cvs-commit mailing list pld-cvs-commit@lists.pld-linux.org http://lists.pld-linux.org/mailman/listinfo/pld-cvs-commit