Revision: 78361
http://sourceforge.net/p/brlcad/code/78361
Author: brlcad
Date: 2021-03-03 06:01:01 +0000 (Wed, 03 Mar 2021)
Log Message:
-----------
check in a slew more scripting and notes/stats on the breakdown.
Modified Paths:
--------------
brlcad/trunk/misc/repoconv/REPO.NOTES
Modified: brlcad/trunk/misc/repoconv/REPO.NOTES
===================================================================
--- brlcad/trunk/misc/repoconv/REPO.NOTES 2021-03-03 04:14:18 UTC (rev
78360)
+++ brlcad/trunk/misc/repoconv/REPO.NOTES 2021-03-03 06:01:01 UTC (rev
78361)
@@ -10,7 +10,12 @@
git --follow . > git.follow.log
+GET THE GIT LOG
git log --all > git.log
+
+TRANSFORM GIT LOG INTO ONE PER LINE
+cat git.log | tr '\n' ' ' | perl -0777 -pe 's/(commit [[:alnum:]]{40}
Author:)/\n\1/g' | perl -0777 -pe 's/(commit [[:alnum:]]{40} Merge:)/\n\1/g' >
git.1.log
+
agua:brlcad_conv11 morrison$ grep -A2 svn:revision git.log | grep -v account |
perl -0777 -p -e 's/\n [cs]v[sn]:branch:/ branch /g' | perl -0777 -p -e
's/--\n//g' | grep trunk | awk '{print $1}' | cut -f3 -d: | sort -n >
git.commits
wc
67967 67967 397262
@@ -30,17 +35,34 @@
GETTING EMPTY LOG MESSAGES
grep -C4 "empty log message" git.log | tr '\n' ';' | perl -0777 -p -e
's/Author:/\nAuthor:/g' | grep branch:trunk|grep -v ':revision:' |less
-GET ALL SVN LOG MESSAGES FOR MISSING COMMITS
+GET SVN LOG MESSAGES (FOR MISSING COMMITS)
cat commits.missing | while read i ; do echo $i ; svn log -r$i ~/brlcad.trunk
; done >> commits.missing.logs
cat commits.missing2 | while read i ; do if ! test -f diffs.svn.sans/$i.log ;
then svn log -c$i ~/brlcad.trunk | tee diffs.svn.sans/$i.log ; fi ; done
-for i in diffs.svn.sans/*.log ; do echo $i ; if ! test -f ${i}.sans.log ; then
tail +4 $i | perl -0777 -pe 's/[-[:space:]]*//g' > ${i}.sans.log ; fi ; done
+for i in diffs.svn.sans/*.log ; do echo $i ; if ! test -f ${i}.log ; then tail
+4 $i | perl -0777 -pe 's/[-[:space:]]*//g' > ${i}.log ; fi ; done
+
+GET SVN FILES CHANGED (FOR MISSING COMMITS)
cat commits.missing2 | while read i ; do echo $i ; if ! test -f
diffs.svn.sans/$i.files ; then svn diff --ignore-properties -c$i ~/brlcad.trunk
| grep '^[-+][-+][-+] ' | grep -v "nonexistent" | awk '{print $1, $2}' | sed
's/\/Users\/morrison\/brlcad.trunk\///g' | sort > diffs.svn.sans/$i.files ; fi
; done
+GET SVN DIFFS (FOR MISSING COMMITS)
+cat commits.missing2 | while read i ; do echo $i ; if ! test -f
diffs.svn.sans/$i.diff ; then svn diff --ignore-properties -c$i
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep "^[-+][^-+]" |
sort | LC_ALL=C sed 's/[[:space:]]*//g' > diffs.svn.sans/$i.diff ; fi ; done
+
+
GET ALL GIT LOG MESSAGES
-grep "^commit " git.log | awk '{print $2}' | while read i ; do echo -n "$i " ;
if ! test -f diffs.git.sans/${i}.log ; then git log --format="%B" ${i}^! | tee
diffs.git.sans/${i}.log ; fi ; done
-for i in diffs.git.sans/*.log ; do echo $i ; if ! test -f ${i}.sans.log ; then
cat $i | grep -v '^cvs:' | grep -v '^svn:' | perl -0777 -pe
's/[-[:space:]]*//g' > ${i}.sans.log ; fi ; done
-grep "^commit " git.log | awk '{print $2}' | while read i ; do echo -n "$i " ;
if ! test -f diffs.git.sans/${i}.files ; then git show ${i}^! | grep
'^[-+][-+][-+] ' | grep -v '/dev/null' | sed 's/--- a\//--- /g' | sed 's/+++
b\//+++ /g' | sort > diffs.git.sans/${i}.files ; fi ; done
+grep "^commit " git.log | awk '{print $2}' | while read i ; do echo "$i " ; if
! test -f diffs.git.sans/${i}.log ; then git log --format="%B" ${i}^! | tee
diffs.git.sans/${i}.log ; fi ; done
+for i in diffs.git.sans/*.log ; do echo $i ; if ! test -f ${i}.log ; then cat
$i | grep -v '^cvs:' | grep -v '^svn:' | perl -0777 -pe 's/[-[:space:]]*//g' >
${i}.log ; fi ; done
+GET ALL GIT FILES CHANGED
+grep "^commit " git.log | awk '{print $2}' | while read i ; do if ! test -f
diffs.git.sans/${i}.files ; then echo $i ; git show ${i}^! | grep
'^[-+][-+][-+] ' | grep -v '/dev/null' | sed 's/--- a\//--- /g' | sed 's/+++
b\//+++ /g' | sort > diffs.git.sans/${i}.files ; fi ; done
+
+GET ALL GIT DIFFS
+grep "^commit " git.log | awk '{print $2}' | while read i ; do echo $i ; git
diff ${i}^! | grep "^[-+][^-+]" | sort | LC_ALL=C sed 's/[[:space:]]*//g' |
sort > diffs.git.sans/${i}.diff ; done
+
+for i in `grep "^commit " git.log | awk '{print $2}'` ; do p="`cat parallel`"
; if test $p -lt 8 ; then git diff ${i}^! | grep "^[-+][^-+]" | sort | LC_ALL=C
sed 's/[[:space:]]*//g' | sort > diffs.git.sans/${i}.diff.new & ; else echo 0 >
parallel ; git diff ${i}^! | grep "^[-+][^-+]" | sort | LC_ALL=C sed
's/[[:space:]]*//g' | sort > diffs.git.sans/${i}.diff.new ; fi ; done
+
+
+CALCULATE GIT MD5
+for i in `grep "^commit " git.log | cut -f2 -d' '` ; do echo $i log `md5
diffs.git.sans/$i.log` ; echo $i files `md5 diffs.git.sans/$i.files` ; echo $i
diff `md5 diffs.git.sans/$i.diff` ; done | tee git.md5
+
CATEGORIZE THE TYPES OF COMMITS THAT COULDN'T BE FOUND
cat commits.missing | while read i ; do echo -n "$i ... " ; log="`svn log -r$i
~/brlcad.trunk`" ; if test "x`echo $log | grep \"New repository\"`" = "x" -a
"x`echo $log | grep \"Initial revision\"`" = "x" ; then if test "x`echo $log |
grep \"empty log message\"`" = "x" ; then echo "kept" ; echo "$log" >>
commits.missing.logs.kept ; else echo "empty" ; echo "$log" >>
commits.missing.logs.empty ; fi ; else echo "skipped" ; fi ; done >
commits.missing.transcript
cat commits.missing2 | while read i ; do echo -n "$i ... " ; log="`cat
diffs.svn.sans/$i.log`" ; if test "x`echo $log | grep \"New repository\"`" =
"x" -a "x`echo $log | grep \"Initial revision\"`" = "x" ; then if test "x`echo
$log | grep \"empty log message\"`" = "x" ; then echo "MISSING" ; echo "$log"
>> commits.missing.logs.kept2 ; else echo "EMPTY" ; echo "$log" >>
commits.missing.logs.empty2 ; fi ; else echo "SKIP" ; fi ; done | tee
commits.missing.transcript2
@@ -47,12 +69,8 @@
CAPTURE SVN MD5 SUMS OF DIFF, LOG, AND FILES
cat commits.missing | while read i ; do echo -n "$i " ; log="`svn diff
--ignore-properties -c$i ~/brlcad.trunk | grep \"^[-+][^-+]\"`" ; echo -n
"`echo \"$log\"|wc | awk '{print $1}'` " ; echo "$log" > diffs/${i}.diff ; echo
"$log" | md5 ; done | tee svn.md5.missing
-cat commits.missing2 | while read i ; do echo $i log `md5
diffs.svn.sans/$i.log.sans.log` ; echo $i files `md5 diffs.svn.sans/$i.files` ;
echo $i diff `md5 diffs.svn.sans/$i.diff` ; done | tee svn.md5.missing2
+cat commits.missing2 | while read i ; do echo $i log `md5
diffs.svn.sans/$i.log` ; echo $i files `md5 diffs.svn.sans/$i.files` ; echo $i
diff `md5 diffs.svn.sans/$i.diff` ; done | tee svn.md5.missing2
-CAPTURE GIT MD5 SUMS OF DIFF
-grep "^commit " git.log | awk '{print $2}' | while read i ; do echo -n "$i " ;
log="`git diff ${i}^! | grep \"^[-+][^-+]\"`" ; echo -n "`echo \"$log\"|wc |
awk '{print $1}'` " ; echo "$log" > diffs.git/${i}.diff ; echo "$log" | md5 ;
done | tee git.md5.all
-grep "^commit " git.log | awk '{print $2}' | while read i ; do echo $i log
`md5 diffs.git.sans/$i.log.sans.log` ; echo $i files `md5
diffs.git.sans/$i.files` ; echo $i diff `md5 diffs.git.sans/$i.diff` ; done |
tee git.md5.all2
-
COMPARE THEM
cat svn.md5.missing | while read line ; do i=`echo $line | awk '{print $1}'` ;
svnmd5=`echo $line | awk '{print $3}'` ; echo -n "$i ... " ; if test "x$svnmd5"
= "xd41d8cd98f00b204e9800998ecf8427e" ; then echo "EMPTY" ; else grep $svnmd5
git.md5 || echo "NOT FOUND" ; fi ; done | tee svn.to.git
# cat svn.md5.missing2 | while read line ; do i=`echo $line | awk '{print
$1}'` ; type=`echo $line | awk '{print $2}'` ; md5=`echo $line | awk '{print
$6}'` ; if test "x$md5" = "xd41d8cd98f00b204e9800998ecf8427e" ; then echo "$i
$type EMPTY" ; else echo "$i $type `grep $md5 git.md5.all2 | grep $type || echo
"NOT FOUND"`" ; fi ; done | tee svn.to.git2
@@ -92,15 +110,37 @@
cat mappings2.txt | grep NOT | while read line ; do rev=`echo $line | awk
'{print $1}'` ; if grep -q "${rev}\$" git.all_revision_tags ; then echo "found
$rev" ; else echo "did not find $rev" ; fi ; done
-# EXTRACTED ALL SVN COMMITS
-seq 1 78156 | while read i ; do if ! test -f diffs.svn.all/$i.log ; then svn
log -c$i ~/brlcad.github/brlcad.mirror | tee diffs.svn.all/$i.log ; fi ; done
-for i in diffs.svn.all/*.log ; do echo $i ; if ! test -f ${i}.sans.log ; then
tail +4 $i | perl -0777 -pe 's/[-[:space:]]*//g' > ${i}.sans.log ; fi ; done
-seq 1 78156 | while read i ; do echo $i ; if ! test -f diffs.svn.all/$i.files
; then svn diff --ignore-properties -c$i ~/brlcad.github/brlcad.mirror | grep
'^[-+][-+][-+] ' | grep -v "nonexistent" | awk '{print $1, $2}' | sed
's/\/Users\/morrison\/brlcad.github\/brlcad.mirror\///g' | sort >
diffs.svn.all/$i.files ; fi ; done
-seq 1 78156 | while read i ; do echo -n "$i " ; log="`svn diff
--ignore-properties -c$i ~/brlcad.github/brlcad.mirror | grep \"^[-+][^-+]\"`"
; echo -n "`echo \"$log\"|wc | awk '{print $1}'` " ; echo "$log" >
diffs.svn.all/${i}.diff ; echo "$log" | md5 ; done | tee svn.md5.all
-seq 1 78156 | while read i ; do echo $i log `md5
diffs.svn.all/$i.log.sans.log` ; echo $i files `md5 diffs.svn.all/$i.files` ;
echo $i diff `md5 diffs.svn.all/$i.diff` ; done | tee svn.md5.all2
-./compare_all.sh
+# EXTRACT ALL SVN LOGS
+seq 1 78233 | while read i ; do echo $i ; svn log -c$i
file:///Users/morrison/brlcad.github/sfmirror | tail +4 | perl -0777 -pe
's/[-[:space:]]*//g' > diffs.svn.sans/$i.log ; done
+
+# EXTRACT ALL SVN FILES
+for i in `seq 1 78233` ; do echo $i ; svn diff --ignore-properties -c$i
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep '^[-+][-+][-+] '
| grep -v nonexistent | grep brlcad/trunk | awk '{print $1, $2}' | sed
's/brlcad\/trunk\///g' | sort > diffs.svn.sans/$i.files ; done
+
+# EXRACT MISSING SVN FILES
+for file in diffs.svn.sans/*.files ; do if ! test -s $file ; then
rev="`basename $file | sed 's/.files//'`" ; echo $rev ; svn diff
--ignore-properties -c$rev
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep '^[-+][-+][-+] '
| grep -v nonexistent | grep brlcad/trunk | awk '{print $1, $2}' | sed
's/brlcad\/trunk\///g' | sort > diffs.svn.sans/$rev.files ; fi ; done
+
+# extract files in parallel
+grep empty svn.to.git3 | grep MIS | cut -f1 -d' ' | while read rev ; do echo
$rev ; p="`cat parallel`" ; if test $p -lt 8 ; then echo "`expr $p + 1`" >
parallel ; svn diff --ignore-properties -c$rev
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep '^[-+][-+][-+] '
| grep -v nonexistent | grep brlcad/trunk | awk '{print $1, $2}' | sed
's/brlcad\/trunk\///g' | sort > diffs.svn.sans/$rev.files & ; else echo 0 >
parallel ; svn diff --ignore-properties -c$rev
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep '^[-+][-+][-+] '
| grep -v nonexistent | grep brlcad/trunk | awk '{print $1, $2}' | sed
's/brlcad\/trunk\///g' | sort > diffs.svn.sans/$rev.files ; fi ; done
+
+# EXTRACT ALL SVN DIFFS
+for i in `seq 1 78233` ; do echo $i ; svn diff --ignore-properties -c$i
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep "^[-+][^-+]" |
sort | LC_ALL=C sed 's/[[:space:]]*//g' | sort > diffs.svn.sans/$i.diff ; done
+
+seq 1 78156 | while read i ; do echo $i log `md5 diffs.svn.all/$i.log` ; echo
$i files `md5 diffs.svn.all/$i.files` ; echo $i diff `md5
diffs.svn.all/$i.diff` ; done | tee svn.md5.all2
+
+# extract diffs in parallel
+grep empty svn.to.git3 | grep MIS | cut -f1 -d' ' | while read rev ; do echo
$rev ; p="`cat parallel`" ; if test $p -lt 5 ; then echo "`expr $p + 1`" >
parallel ; svn diff --ignore-properties -c$rev
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep "^[-+][^-+]" |
sort | LC_ALL=C sed 's/[[:space:]]*//g' | sort > diffs.svn.sans/$rev.diff & ;
else echo 0 > parallel ; svn diff --ignore-properties -c$rev
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep "^[-+][^-+]" |
sort | LC_ALL=C sed 's/[[:space:]]*//g' | sort > diffs.svn.sans/$rev.diff ; fi
; done
+
+# COMPUTE SVN MD5 SUMS
+for i in `seq 1 78233` ; do echo $i log `md5 diffs.svn.sans/$i.log` ; echo $i
files `md5 diffs.svn.sans/$i.files` ; echo $i diff `md5 diffs.svn.sans/$i.diff`
; done | tee svn.md5
+
+# DO THE COMPARISON
+cat svn.md5 | ./compare_all.sh | tee svn.to.git2
+cat svn.md5.16000 | ./compare_parallel.sh | tee svn.to.git3.16000
+
+
+# CONVERT NON-SANS DIFFS TO SANS
cd diffs.svn.all ;; for i in *.diff ; do echo $i ; cat $i | LC_ALL=C sed
's/[[:space:]]*//g' > ../diffs.svn.all.sans/$i ; done ;; mv
../diffs.svn.all.sans/* . ;; rmdir ../diffs.svn.all.sans
-
+tr
# FIND TAGGINGS FOR ALL TRUNK COMMITS
for i in `cat svn.commits2` ; do echo -n $i ; grep "revision:${i}$" -C3
git.log | grep :branch: ; done | tee trunk_tags_in_git.log
grep ":revision:" -C3 git.log > git.revision.log
@@ -113,3 +153,28 @@
rev=1 ; while test $rev -lt 78155 ; do echo -n "$rev " ; svn diff -c$rev
file:///Users/morrison/brlcad.github/sfmirror | grep "^[-+][-+][-+] " | grep -e
"\(branches\|trunk\|tags\)" | cut -f2,3 -d/ | sed 's/trunk.*/trunk/g' | sort |
uniq | tr '\n' ' ' ; echo "" ; rev="`expr $rev + 1`" ; done | tee
all_branches.log
rev=1 ; while test $rev -lt 78155 ; do echo -n "$rev " ; svn diff -c$rev
file:///Users/morrison/brlcad.github/svn.sfmirror/code | grep "^Index: " | sed
's/Index: //g' | cut -f1,2,3 -d/ | sed 's/trunk.*/trunk/g' | sort | uniq | tr
'\n' ' ' ; echo "" ; rev="`expr $rev + 1`" ; done | tee all_branches2.log
+
+cat svn.to.git3 | grep -E '[0-9]{1,5} '|wc
+cat svn.to.git3 | grep -E '[0-9]{1,5} '| grep -v PERFECT | grep -v "NOT FOUND"
| grep -v -i empty | less
+cat svn.to.git3 | grep -v -i empty | grep -v PERFECT | grep -v " NOT FOUND" |
grep -v " LOG MATCH" | grep -v " FILE MATCH" | grep -v " LOG+FILE MATCH" | grep
-v " LOG+FILE+DIFF MATCH" | grep -v " LOG+DIFF MATCH" | grep -v "FILE+DIFF"
+
+78819 commits
+-6874 PERFECT MATCH
+-----
+71945
+-9055 NOT FOUND (branch changes)
+-----
+62890
+- 932 EMPTY (prop changes)
+-----
+61958
+
+46569 LOG+FILE+DIFF MATCH (279 mismatch?)
+14066 LOG+FILE MATCH (1582 mismatch?)
+ 1156 LOG MATCH (393 mismatch?)
+ 138 LOG+DIFF MATCH (8 mismatch?)
+ 24 FILE+DIFF MATCH
+ 5 FILE MATCH (5 mismatch?)
+ 0 DIFF MATCH
+
+
This was sent by the SourceForge.net collaborative development platform, the
world's largest Open Source development site.
_______________________________________________
BRL-CAD Source Commits mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/brlcad-commits
