The branch, master has been updated via 991ab811 Turn nightly-rsync into a python script. via 32498242 Some more rsync-ssl improvements: via 1c465dc3 Change the name of the rsh-ssl-rsync script. from 2a87d78f Change the rsync-ssl helper script
https://git.samba.org/?p=rsync.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit 991ab811cbc4f65f17f46a810848f1eb4e058b05 Author: Wayne Davison <wa...@opencoder.net> Date: Wed Apr 22 12:00:08 2020 -0700 Turn nightly-rsync into a python script. commit 3249824264f90241a8acd0bbf7aa5ab3c88442ff Author: Wayne Davison <wa...@opencoder.net> Date: Wed Apr 22 10:53:38 2020 -0700 Some more rsync-ssl improvements: - Make the rsync-ssl default behavior more user friendly. - Install rsync-ssl & rsync-ssl-rsh in the regular install rules. - Add a manpage for rsync-ssl (which is also installed). - Get rid of the rsync-ssl-client package in our spec file. commit 1c465dc33a6ac7dd2c67cd7ee2a40b6ff18e4ae3 Author: Wayne Davison <wa...@opencoder.net> Date: Wed Apr 22 10:52:01 2020 -0700 Change the name of the rsh-ssl-rsync script. ----------------------------------------------------------------------- Summary of changes: .gitignore | 4 +- Makefile.in | 28 +++-- packaging/lsb/rsync.spec | 19 +-- packaging/nightly-rsync | 267 ++++++++++++++++------------------------- packaging/pkglib.py | 57 +++++++++ packaging/release-rsync | 49 ++------ rsh-ssl-rsync => rsync-ssl-rsh | 66 ++++++---- rsync-ssl.in | 2 +- rsync-ssl.yo | 69 +++++++++++ 9 files changed, 306 insertions(+), 255 deletions(-) rename rsh-ssl-rsync => rsync-ssl-rsh (63%) create mode 100644 rsync-ssl.yo Changeset truncated at 500 lines: diff --git a/.gitignore b/.gitignore index 519bd3bc..f601caed 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,8 @@ config.status aclocal.m4 /proto.h /proto.h-tstamp -/rsync.1 -/rsyncd.conf.5 +/*.1 +/*.5 /autom4te*.cache /confdefs.h /conftest* diff --git a/Makefile.in b/Makefile.in index 3057dc33..6ce3c74c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -29,7 +29,7 @@ VERSION=@RSYNC_VERSION@ .SUFFIXES: .SUFFIXES: .c .o -GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsyncd.conf.5 +GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync-ssl.1 rsyncd.conf.5 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \ lib/pool_alloc.h LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \ @@ -68,16 +68,14 @@ all: Makefile rsync$(EXEEXT) rsync-ssl stunnel-rsyncd.conf @MAKE_MAN@ install: all -${MKDIR_P} ${DESTDIR}${bindir} ${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir} + ${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir} + ${INSTALLCMD} -m 755 rsync-ssl-rsh ${DESTDIR}${bindir} -${MKDIR_P} ${DESTDIR}${mandir}/man1 -${MKDIR_P} ${DESTDIR}${mandir}/man5 if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi + if test -f rsync-ssl.1; then ${INSTALLMAN} -m 644 rsync-ssl.1 ${DESTDIR}${mandir}/man1; fi if test -f rsyncd.conf.5; then ${INSTALLMAN} -m 644 rsyncd.conf.5 ${DESTDIR}${mandir}/man5; fi -install-ssl-client: rsync-ssl - -${MKDIR_P} ${DESTDIR}${bindir} - ${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir} - ${INSTALLCMD} -m 755 rsh-ssl-rsync ${DESTDIR}${bindir} - install-ssl-daemon: stunnel-rsyncd.conf -${MKDIR_P} ${DESTDIR}/etc/stunnel ${INSTALLCMD} -m 644 stunnel-rsyncd.conf ${DESTDIR}/etc/stunnel/rsyncd.conf @@ -209,16 +207,28 @@ proto.h: proto.h-tstamp proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h awk -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c -man: rsync.1 rsyncd.conf.5 man-copy +man: rsync.1 rsync-ssl.1 rsyncd.conf.5 man-copy man-copy: - @if test -f rsync.1; then :; elif test -f $(srcdir)/rsync.1; then echo 'Copying srcdir rsync.1'; cp -p $(srcdir)/rsync.1 .; else echo "NOTE: rsync.1 cannot be created."; fi - @if test -f rsyncd.conf.5; then :; elif test -f $(srcdir)/rsyncd.conf.5; then echo 'Copying srcdir rsyncd.conf.5'; cp -p $(srcdir)/rsyncd.conf.5 .; else echo "NOTE: rsyncd.conf.5 cannot be created."; fi + @for fn in rsync.1 rsync-ssl.1 rsyncd.conf.5; do \ + if test -f $$fn; then \ + : ; \ + elif test -f $(srcdir)/$$fn; then \ + echo "Copying srcdir $$fn" ; \ + cp -p $(srcdir)/$$fn . ; \ + else \ + echo "NOTE: $$fn cannot be created." ; \ + fi ; \ + done rsync.1: rsync.yo yodl2man -o rsync.1 $(srcdir)/rsync.yo -$(srcdir)/tweak_manpage rsync.1 +rsync-ssl.1: rsync-ssl.yo + yodl2man -o rsync-ssl.1 $(srcdir)/rsync-ssl.yo + -$(srcdir)/tweak_manpage rsync-ssl.1 + rsyncd.conf.5: rsyncd.conf.yo yodl2man -o rsyncd.conf.5 $(srcdir)/rsyncd.conf.yo -$(srcdir)/tweak_manpage rsyncd.conf.5 diff --git a/packaging/lsb/rsync.spec b/packaging/lsb/rsync.spec index f527976e..65a442e8 100644 --- a/packaging/lsb/rsync.spec +++ b/packaging/lsb/rsync.spec @@ -13,11 +13,6 @@ URL: http://rsync.samba.org/ Prefix: %{_prefix} BuildRoot: /var/tmp/%{name}-root -%package ssl-client -Summary: Provides rsync-ssl -Group: Applications/Internet -Requires: rsync, stunnel >= 4 - %package ssl-daemon Summary: An stunnel config file to support ssl rsync daemon connections. Group: Applications/Internet @@ -34,11 +29,6 @@ differences between the source files and the existing files in the destination. Rsync is widely used for backups and mirroring and as an improved copy command for everyday use. -%description ssl-client -Provides the rsync-ssl script that makes use of stunnel 4 to open an ssl -connection to an rsync daemon (on port 874). This setup does NOT require -any local stunnel daemon to be running to connect to the remote ssl rsyncd. - %description ssl-daemon Provides a config file for stunnel that will (if you start your stunnel service) cause stunnel to listen for ssl rsync-daemon connections and run @@ -66,7 +56,7 @@ make %install rm -rf $RPM_BUILD_ROOT -make install install-ssl-client install-ssl-daemon DESTDIR=$RPM_BUILD_ROOT +make install install-ssl-daemon DESTDIR=$RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT/etc/xinetd.d $RPM_BUILD_ROOT/etc/rsync-ssl/certs install -m 644 packaging/lsb/rsync.xinetd $RPM_BUILD_ROOT/etc/xinetd.d/rsync @@ -79,13 +69,12 @@ rm -rf $RPM_BUILD_ROOT %doc COPYING NEWS OLDNEWS README support/ tech_report.tex %config(noreplace) /etc/xinetd.d/rsync %{_prefix}/bin/rsync +%{_prefix}/bin/rsync-ssl +%{_prefix}/bin/rsync-ssl-rsh %{_mandir}/man1/rsync.1* +%{_mandir}/man1/rsync-ssl.1* %{_mandir}/man5/rsyncd.conf.5* -%files ssl-client -%{_prefix}/bin/rsync-ssl -%{_prefix}/bin/rsh-ssl-rsync - %files ssl-daemon %config(noreplace) /etc/stunnel/rsyncd.conf %dir /etc/rsync-ssl/certs diff --git a/packaging/nightly-rsync b/packaging/nightly-rsync index 060f1a40..3e8895b9 100755 --- a/packaging/nightly-rsync +++ b/packaging/nightly-rsync @@ -1,174 +1,117 @@ -#!/usr/bin/perl -use strict; +#!/usr/bin/python3 -B # This script expects the directory ~/samba-rsync-ftp to exist and to be a # copy of the /home/ftp/pub/rsync dir on samba.org. It also requires a # git checkout of rsync (feel free to use your normal rsync build dir as # long as it doesn't have any uncommitted changes). # -# If this is run with -ctu, it will make an updated "nightly" tar file in +# If this is run with -tu, it will make an updated "nightly" tar file in # the nightly dir. It will also remove any old tar files, regenerate the # HTML man pages in the nightly dir, and then rsync the changes to the # samba.org server. -use Getopt::Long; -use Date::Format; +import os, sys, re, argparse, glob +from datetime import datetime, timezone +from getpass import getpass + +sys.path = ['packaging'] + sys.path + +from pkglib import * # Where the local copy of /home/ftp/pub/rsync/dev/nightly should be updated. -our $dest = $ENV{HOME} . '/samba-rsync-ftp/dev/nightly'; -our $samba_host = $ENV{SAMBA_HOST} || 'samba.org'; -our $nightly_symlink = "$dest/rsync-HEAD.tar.gz"; - -our($make_tar, $upload, $help_opt); -&Getopt::Long::Configure('bundling'); -&usage if !&GetOptions( - 'make-tar|t' => \$make_tar, - 'upload|u' => \$upload, - 'help|h' => \$help_opt, -) || $help_opt; - -our $name = time2str('rsync-HEAD-%Y%m%d-%H%M%Z', time, 'GMT'); -our $ztoday = time2str('%d %b %Y', time); -our $today = $ztoday; -our $gen_target = $upload ? 'gensend' : 'gen'; - -die "$dest does not exist\n" unless -d $dest; -die "There is no .git dir in the current directory.\n" unless -d '.git'; -die "There is no rsync checkout in the current directory.\n" unless -f 'rsyncd.conf.yo'; - -if ($make_tar) { - open(IN, '-|', 'git status') or die $!; - my $status = join('', <IN>); - close IN; - die "The checkout is not clean:\n", $status unless $status =~ /\nnothing to commit.+working (directory|tree) clean/; - die "The checkout is not on the master branch.\n" unless $status =~ /^(?:# )?On branch master\n/; - system "make $gen_target" and die "make $gen_target failed!\n"; - - my @extra_files; - open(IN, '<', 'Makefile.in') or die "Couldn't open Makefile.in: $!\n"; - while (<IN>) { - if (s/^GENFILES=//) { - while (s/\\$//) { - $_ .= <IN>; - } - @extra_files = split(' ', $_); - last; - } - } - close IN; - - my $confversion; - open(IN, '<', 'configure.ac') or die "Unable to open configure.ac: $!\n"; - while (<IN>) { - if (/^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]/) { - $confversion = $1; - last; - } - } - close IN; - die "Unable to find AC_INIT with version in configure.ac\n" unless defined $confversion; - - open(IN, '<', 'OLDNEWS') or die "Unable to open OLDNEWS: $!\n"; - $_ = <IN>; - my($lastversion) = /(\d+\.\d+\.\d+)/; - my $last_protocol_version; - while (<IN>) { - if (my($ver,$pdate,$pver) = /^\s+\S\S\s\S\S\S\s\d\d\d\d\s+(\d+\.\d+\.\d+)\s+(\d\d \w\w\w \d\d\d\d\s+)?(\d+)$/) { - $last_protocol_version = $pver if $ver eq $lastversion; - } - } - close IN; - die "Unable to determine protocol_version for $lastversion.\n" unless defined $last_protocol_version; - - my($protocol_version,$subprotocol_version); - open(IN, '<', 'rsync.h') or die "Unable to open rsync.h: $!\n"; - while (<IN>) { - if (/^#define\s+PROTOCOL_VERSION\s+(\d+)/) { - $protocol_version = $1; - } elsif (/^#define\s+SUBPROTOCOL_VERSION\s+(\d+)/) { - $subprotocol_version = $1; - } - } - close IN; - die "Unable to determine the current PROTOCOL_VERSION.\n" unless defined $protocol_version; - die "Unable to determine the current SUBPROTOCOL_VERSION.\n" unless defined $subprotocol_version; - - if ($confversion =~ /dev|pre/) { - if ($last_protocol_version ne $protocol_version) { - if ($subprotocol_version == 0) { - die "SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.\n"; - } - } else { - if ($subprotocol_version != 0) { - die "SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.\n"; - } - } - } else { - if ($subprotocol_version != 0) { - die "SUBPROTOCOL_VERSION must be 0 for a final release.\n"; - } - } - - print "Creating $name.tar.gz\n"; - system "rsync -a @extra_files $name/"; - system "git archive --format=tar --prefix=$name/ HEAD | tar xf -"; - system "support/git-set-file-times --quiet --prefix=$name/"; - system "fakeroot tar czf $dest/$name.tar.gz $name; rm -rf $name"; - - unlink($nightly_symlink); - symlink("$name.tar.gz", $nightly_symlink); -} - -foreach my $fn (qw( rsync.yo rsyncd.conf.yo )) { - my $yo_tmp = "$dest/$fn"; - (my $html_fn = "$dest/$fn") =~ s/\.yo/.html/; - - open(IN, '<', $fn) or die $!; - undef $/; $_ = <IN>; $/ = "\n"; - close IN; - - s/^(manpage\([^)]+\)\(\d+\)\()[^)]+(\).*)/$1$today$2/m; - #s/^(This man ?page is current for version) \S+ (of rsync)/$1 $version $2/m; - - open(OUT, '>', $yo_tmp) or die $!; - print OUT $_; - close OUT; - - system 'yodl2html', '-o', $html_fn, $yo_tmp; - - unlink($yo_tmp); -} - -chdir($dest) or die $!; - -my $cnt = 0; -open(PIPE, '-|', 'ls -1t rsync-HEAD-*') or die $!; -while (<PIPE>) { - chomp; - next if $cnt++ < 10; - unlink($_); -} -close PIPE; - -system 'ls -ltr'; - -if ($upload) { - my $opt = ''; - if (defined $ENV{RSYNC_PARTIAL_DIR}) { - $opt = " -f 'R $ENV{RSYNC_PARTIAL_DIR}'"; - } - system "rsync$opt -aviHP --delete-after . $samba_host\:/home/ftp/pub/rsync/dev/nightly"; -} - -exit; - -sub usage -{ - die <<EOT; -Usage: nightly-rsync [OPTIONS] - - -t, --make-tar create a new tar file in $dest - -u, --upload upload the revised nightly dir to samba.org - -h, --help display this help -EOT -} +dest = os.environ['HOME'] + '/samba-rsync-ftp/dev/nightly' +samba_host = os.environ['SAMBA_HOST'] +nightly_symlink = f"{dest}/rsync-HEAD.tar.gz" + +def main(): + now = datetime.now(timezone.utc) + name = now.strftime('rsync-HEAD-%Y%m%d-%H%MGMT') + ztoday = now.strftime('%d %b %Y') + today = ztoday.lstrip('0') + gen_target = 'gensend' if args.upload else 'gen' + + if not os.path.isdir(dest): + die("$dest does not exist") + if not os.path.isdir('.git'): + die("There is no .git dir in the current directory.") + if not os.path.exists('rsyncd.conf.yo'): + die("There is no rsync checkout in the current directory.") + + if args.make_tar: + check_git_state('master') + cmd_chk(['make', gen_target]) + + extra_files = get_extra_files() + + confversion = get_configure_version() + + # All version values are strings! + last_version, last_protocol_version = get_OLDNEWS_version_info() + protocol_version, subprotocol_version = get_protocol_versions() + + if 'dev' in confversion or 'pre' in confversion: + if last_protocol_version != protocol_version: + if subprotocol_version == '0': + die("SUBPROTOCOL_VERSION must not be 0 for a non-final release with a changed PROTOCOL_VERSION.") + elif subprotocol_version != '0': + die("SUBPROTOCOL_VERSION must be 0 when the PROTOCOL_VERSION hasn't changed from the last release.") + elif subprotocol_version != '0': + die("SUBPROTOCOL_VERSION must be 0 for a final release.") + + name_slash = name + '/' + tar_name = f"{name}.tar.gz" + + print('Creating', tar_name) + + cmd_chk(['rsync', '-a', *extra_files, name_slash]) + cmd_chk(f"git archive --format=tar --prefix={name}/ HEAD | tar xf -") + cmd_chk(['support/git-set-file-times', '--quiet', '--prefix', name_slash]) + cmd_chk(['fakeroot', 'tar', 'czf', os.path.join(dest, tar_name), name]) + cmd_chk(['rm', '-rf', name]) + + if os.path.lexists(nightly_symlink): + os.unlink(nightly_symlink) + os.symlink(tar_name, nightly_symlink) + + for fn in sorted(glob.glob('*.yo')): + yo_tmp = os.path.join(dest, fn) + html_fn = yo_tmp.replace('.yo', '.html') + + with open(fn, 'r', encoding='utf-8') as fh: + txt = fh.read() + + txt = re.sub(r'^(manpage\([^)]+\)\(\d+\)\()[^)]+(\).*)', r'\1%s\2' % today, txt, flags=re.M) + + with open(yo_tmp, 'w', encoding='utf-8') as fh: + fh.write(txt) + + cmd_chk(['yodl2html', '-o', html_fn, yo_tmp]) + os.unlink(yo_tmp) + + os.chdir(dest) + + tar_files = list(reversed(sorted(glob.glob('rsync-HEAD-*')))) + if len(tar_files) > 10: + for fn in tar_files[10:]: + print('Removing', fn) + os.unlink(fn) + + cmd_run('ls -ltr'.split()) + + if args.upload: + cmd = 'rsync -aivHP --delete-after'.split() + partial_dir = os.environ.get('RSYNC_PARTIAL_DIR', None) + if partial_dir: + cmd.append('-fR ' + partial_dir) + cmd_chk([*cmd, '.', f"{samba_host}:/home/ftp/pub/rsync/dev/nightly"]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='A helper script for "nightly" tar files.', add_help=False) + parser.add_argument('--make-tar', '-t', action='store_true', help=f"Create a new tar file in {dest}.") + parser.add_argument('--upload', '-u', action='store_true', help="Upload the revised nightly dir to {samba_host}.") + parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") + args = parser.parse_args() + main() + +# vim: sw=4 et diff --git a/packaging/pkglib.py b/packaging/pkglib.py index fd6359e4..cf50f1a9 100644 --- a/packaging/pkglib.py +++ b/packaging/pkglib.py @@ -157,4 +157,61 @@ def get_extra_files(): return extras + +def get_configure_version(): + with open('configure.ac', 'r', encoding='utf-8') as fh: + for line in fh: + m = re.match(r'^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]', line) + if m: + return m[1] + die("Unable to find AC_INIT with version in configure.ac") + + +def get_OLDNEWS_version_info(): + rel_re = re.compile(r'^\s+\S{2}\s\S{3}\s\d{4}\s+(?P<ver>\d+\.\d+\.\d+)\s+(?P<pdate>\d{2} \w{3} \d{4}\s+)?(?P<pver>\d+)$') + last_version = last_protocol_version = None + pdate = { } + + with open('OLDNEWS', 'r', encoding='utf-8') as fh: + for line in fh: + if not last_version: + m = re.search(r'(\d+\.\d+\.\d+)', line) + if m: + last_version = m[1] + m = rel_re.match(line) + if m: + if m['pdate']: + pdate[m['ver']] = m['pdate'] + if m['ver'] == last_version: + last_protocol_version = m['pver'] + break + + if not last_protocol_version: + die(f"Unable to determine protocol_version for {last_version}.") + + return last_version, last_protocol_version + + +def get_protocol_versions(): + protocol_version = subprotocol_version = None + + with open('rsync.h', 'r', encoding='utf-8') as fh: + for line in fh: + m = re.match(r'^#define\s+PROTOCOL_VERSION\s+(\d+)', line) + if m: + protocol_version = m[1] + continue + m = re.match(r'^#define\s+SUBPROTOCOL_VERSION\s+(\d+)', line) + if m: + subprotocol_version = m[1] + break + + if not protocol_version: + die("Unable to determine the current PROTOCOL_VERSION.") + + if not subprotocol_version: + die("Unable to determine the current SUBPROTOCOL_VERSION.") + + return protocol_version, subprotocol_version + # vim: sw=4 et diff --git a/packaging/release-rsync b/packaging/release-rsync index 20d8ea07..b0e55387 100755 --- a/packaging/release-rsync +++ b/packaging/release-rsync @@ -5,7 +5,7 @@ # the git repository in the current directory will be updated, and the local # ~/samba-rsync-ftp dir will be ready to be rsynced to samba.org. -import os, sys, re, argparse, glob, time, shutil, atexit, signal +import os, sys, re, argparse, glob, shutil, atexit, signal from datetime import datetime from getpass import getpass @@ -21,7 +21,6 @@ MAKE_GEN_CMDS = [ './config.status'.split(), 'make gen'.split(), -- The rsync repository. _______________________________________________ rsync-cvs mailing list rsync-cvs@lists.samba.org https://lists.samba.org/mailman/listinfo/rsync-cvs