Re: [PATCH][RFC] space saving incrementals
On Mon, Mar 11, 2002 at 11:32:11AM -0800, jw schultz wrote: On Mon, Mar 11, 2002 at 12:41:07PM -0600, Dave Dykstra wrote: On Fri, Mar 08, 2002 at 03:05:14PM -0800, jw schultz wrote: On Fri, Mar 08, 2002 at 03:45:04PM -0600, Dave Dykstra wrote: I like link-dest, and the - for exclude-from/include-from was already something I was planning to add one of these days along with the --files-from option I still plan to write, but --compare-perms confuses me. Can you give examples of when you need it? - Dave Dykstra sequence of commands: admin# rsync /project /vault/monday admin# chmod 666 /project/somefile admin# chown bin /project/someotherfile admin# rsync --link-dest=/vault/monday /project /vault/tuesday Now /vault/monday/somefile has perms of 666 and someotherfile is owned by bin. I cannot restore the perms from /vault/monday. That info is lost. /vault/monday no longer is an accurate representative of what existed when it was created. With --compare-perms somefile and someotherfile will not be linked between monday and tuesday even though the contents haven't changed. compare-perms only makes sense when you are using link-dest. Why not just always do --compare-perms functionality with --link-dest? I'd rather not have more hard-to-explain options if they can be avoided. - Dave Dykstra I personally wouldn't mind. --compare-perms existed before --link-dest. I started by doing a cp -al before the rsync but that seemed wastefull especially with how little it took to add the --link-dest onto --compare-dest. However, i suspect that some others would gripe at rsync --link-dest not linking if all that changed was file meta-data. I can't imagine anybody griping at that. I could be wrong but i think --link-dest without a seperate --compare-perms or its inverse might cause more confusion although users could leave off -pgo if they don't care about the meta-data. Right. This may become more of an issue with ACLs and other extended file attributes. But that is at least another year in the future. By the way, really appreciate rsync. I tried this with rdist because it didn't have the permissions problem but the performnce stank. Monitoring it i found the slowdown wasn't bandwith, disk or CPU. Rsync even did the initial transfer (nothing on dest) at over 10x the speed with the limitation being the 350MHz cpu on the backup server. - Dave Dykstra -- To unsubscribe or change options: http://lists.samba.org/mailman/listinfo/rsync Before posting, read: http://www.tuxedo.org/~esr/faqs/smart-questions.html
Re: [PATCH][RFC] space saving incrementals
On Mon, Mar 11, 2002 at 12:41:07PM -0600, Dave Dykstra wrote:
On Fri, Mar 08, 2002 at 03:05:14PM -0800, jw schultz wrote:
On Fri, Mar 08, 2002 at 03:45:04PM -0600, Dave Dykstra wrote:
I like link-dest, and the - for exclude-from/include-from was already
something I was planning to add one of these days along with the
--files-from option I still plan to write, but --compare-perms confuses
me. Can you give examples of when you need it?
- Dave Dykstra
sequence of commands:
admin# rsync /project /vault/monday
admin# chmod 666 /project/somefile
admin# chown bin /project/someotherfile
admin# rsync --link-dest=/vault/monday /project /vault/tuesday
Now /vault/monday/somefile has perms of 666 and
someotherfile is owned by bin. I cannot restore the perms
from /vault/monday. That info is lost. /vault/monday no
longer is an accurate representative of what existed when it
was created.
With --compare-perms somefile and someotherfile will not be
linked between monday and tuesday even though the contents
haven't changed.
compare-perms only makes sense when you are using link-dest.
Why not just always do --compare-perms functionality with --link-dest? I'd
rather not have more hard-to-explain options if they can be avoided.
- Dave Dykstra
New patch against current CVS.
Adds --link-dest with compare-perms functionality
and adds the - for (ex|in)clude-from.
made with cvs diff -u
--
J.W. SchultzPegasystems Technologies
email address: [EMAIL PROTECTED]
Remember Cernan and Schmitt
Index: exclude.c
===
RCS file: /cvsroot/rsync/exclude.c,v
retrieving revision 1.42
diff -u -r1.42 exclude.c
--- exclude.c 18 Feb 2002 19:10:28 - 1.42
+++ exclude.c 13 Mar 2002 00:25:45 -
@@ -219,8 +219,14 @@
int fatal, int include)
{
struct exclude_struct **list=list1;
- FILE *f = fopen(fname,r);
+ FILE *f;
char line[MAXPATHLEN];
+
+ if (strcmp(fname, -)) {
+ f = fopen(fname,r);
+ } else {
+ f = fdopen(0, r);
+ }
if (!f) {
if (fatal) {
rsyserr(FERROR, errno,
Index: generator.c
===
RCS file: /cvsroot/rsync/generator.c,v
retrieving revision 1.33
diff -u -r1.33 generator.c
--- generator.c 7 Feb 2002 16:36:12 - 1.33
+++ generator.c 13 Mar 2002 00:25:45 -
@@ -42,6 +42,7 @@
extern int always_checksum;
extern int modify_window;
extern char *compare_dest;
+extern int link_dest;
/* choose whether to skip a particular file */
@@ -51,6 +52,15 @@
if (st-st_size != file-length) {
return 0;
}
+ if (link_dest) {
+ if((st-st_mode ~_S_IFMT) != (file-mode ~_S_IFMT)) {
+ return 0;
+ }
+ if (st-st_uid != file-uid || st-st_gid != file-gid) {
+ return 0;
+ }
+ }
+
/* if always checksum is set then we use the checksum instead
of the file time to determine whether to sync */
@@ -352,6 +362,17 @@
statret = -1;
if (statret == -1)
errno = saveerrno;
+#if HAVE_LINK
+ else if (link_dest)
+ if (do_link(fnamecmpbuf, fname) != 0) {
+ if (verbose 0)
+ rprintf(FINFO,link %s = %s : %s\n,
+ fnamecmpbuf,
+ fname,
+ strerror(errno));
+ fnamecmp = fnamecmpbuf;
+ }
+#endif
else
fnamecmp = fnamecmpbuf;
}
Index: options.c
===
RCS file: /cvsroot/rsync/options.c,v
retrieving revision 1.80
diff -u -r1.80 options.c
--- options.c 27 Feb 2002 22:49:57 - 1.80
+++ options.c 13 Mar 2002 00:25:47 -
@@ -105,6 +105,7 @@
int quiet = 0;
int always_checksum = 0;
int list_only = 0;
+int link_dest = 0;
char *batch_prefix = NULL;
@@ -224,6 +225,7 @@
rprintf(F, --modify-window=NUM Timestamp window (seconds) for file match
(default=%d)\n,modify_window);
rprintf(F, -T --temp-dir=DIR create temporary files in directory DIR\n);
rprintf(F, --compare-dest=DIR also compare destination files relative to
DIR\n);
+ rprintf(F, --link-dest=DIR create hardlinks to DIR for unchanged
+files\n);
rprintf(F, -P equivalent to --partial --progress\n);
rprintf(F, -z, --compress compress file data\n);
Re: [PATCH][RFC] space saving incrementals
On Fri, Mar 08, 2002 at 03:05:14PM -0800, jw schultz wrote: On Fri, Mar 08, 2002 at 03:45:04PM -0600, Dave Dykstra wrote: I like link-dest, and the - for exclude-from/include-from was already something I was planning to add one of these days along with the --files-from option I still plan to write, but --compare-perms confuses me. Can you give examples of when you need it? - Dave Dykstra sequence of commands: admin# rsync /project /vault/monday admin# chmod 666 /project/somefile admin# chown bin /project/someotherfile admin# rsync --link-dest=/vault/monday /project /vault/tuesday Now /vault/monday/somefile has perms of 666 and someotherfile is owned by bin. I cannot restore the perms from /vault/monday. That info is lost. /vault/monday no longer is an accurate representative of what existed when it was created. With --compare-perms somefile and someotherfile will not be linked between monday and tuesday even though the contents haven't changed. compare-perms only makes sense when you are using link-dest. Why not just always do --compare-perms functionality with --link-dest? I'd rather not have more hard-to-explain options if they can be avoided. - Dave Dykstra -- To unsubscribe or change options: http://lists.samba.org/mailman/listinfo/rsync Before posting, read: http://www.tuxedo.org/~esr/faqs/smart-questions.html
Re: [PATCH][RFC] space saving incrementals
On Mon, Mar 11, 2002 at 12:41:07PM -0600, Dave Dykstra wrote: On Fri, Mar 08, 2002 at 03:05:14PM -0800, jw schultz wrote: On Fri, Mar 08, 2002 at 03:45:04PM -0600, Dave Dykstra wrote: I like link-dest, and the - for exclude-from/include-from was already something I was planning to add one of these days along with the --files-from option I still plan to write, but --compare-perms confuses me. Can you give examples of when you need it? - Dave Dykstra sequence of commands: admin# rsync /project /vault/monday admin# chmod 666 /project/somefile admin# chown bin /project/someotherfile admin# rsync --link-dest=/vault/monday /project /vault/tuesday Now /vault/monday/somefile has perms of 666 and someotherfile is owned by bin. I cannot restore the perms from /vault/monday. That info is lost. /vault/monday no longer is an accurate representative of what existed when it was created. With --compare-perms somefile and someotherfile will not be linked between monday and tuesday even though the contents haven't changed. compare-perms only makes sense when you are using link-dest. Why not just always do --compare-perms functionality with --link-dest? I'd rather not have more hard-to-explain options if they can be avoided. - Dave Dykstra I personally wouldn't mind. --compare-perms existed before --link-dest. I started by doing a cp -al before the rsync but that seemed wastefull especially with how little it took to add the --link-dest onto --compare-dest. However, i suspect that some others would gripe at rsync --link-dest not linking if all that changed was file meta-data. I could be wrong but i think --link-dest without a seperate --compare-perms or its inverse might cause more confusion although users could leave off -pgo if they don't care about the meta-data. This may become more of an issue with ACLs and other extended file attributes. But that is at least another year in the future. By the way, really appreciate rsync. I tried this with rdist because it didn't have the permissions problem but the performnce stank. Monitoring it i found the slowdown wasn't bandwith, disk or CPU. Rsync even did the initial transfer (nothing on dest) at over 10x the speed with the limitation being the 350MHz cpu on the backup server. -- J.W. SchultzPegasystems Technologies email address: [EMAIL PROTECTED] Remember Cernan and Schmitt -- To unsubscribe or change options: http://lists.samba.org/mailman/listinfo/rsync Before posting, read: http://www.tuxedo.org/~esr/faqs/smart-questions.html
Re: [PATCH][RFC] space saving incrementals
I like link-dest, and the - for exclude-from/include-from was already
something I was planning to add one of these days along with the
--files-from option I still plan to write, but --compare-perms confuses
me. Can you give examples of when you need it?
- Dave Dykstra
On Thu, Mar 07, 2002 at 01:25:50PM -0800, jw schultz wrote:
Please CC me directly as i'm not on the list.
I have attached a patch against latest CVS (cvs diff -u)
that adds the following functionality. I can break it up if
you would prefer it in pieces. Comments welcome.
o add compare-perms option
This creates a new inode for a file even if only
the perms have changed. This way if a file
outside of destdir is hardlinked to a dentry
inside destdir the permissions (uid, gid,
mode) will be untouched preserving history
etc.
o link-dest option
After setting compare_dest this causes
unchanged files in destdir to be hardlinked
to link-dest.
o modified make_exclude_list to support stdin
if --exclude-from has argument of -
stdin will be read.
This lets us pipe a include/exclude list
into rsync so that we can generate it on the
fly.
The upshot of these is to allow rsync to make incremental
backups without modifying earlier versions but keep every
version as a complete tree. It then becomes possible to
make tapes or restore from any image.
Although --compare-perms saves on block count this patch
only applies it to regular files so symlinks and device
nodes will still chew up inodes. :(
for the sake of an example here is the command line i'm using:
/site/bin/rsync -v --stats -a -H -e ssh --compare-perms
--delete --delete-excluded --numeric-ids --exclude-from -
--link-dest /e/backup1/home/update1/tree leto:/efs/home/
/e/backup1/home/update2/tree /e/backup1/home/update2/log
i have tried to keep my changes small and consistent with the
existing coding style. Feel free to dink with it. I only
care about the performance.
--
J.W. SchultzPegasystems Technologies
email address: [EMAIL PROTECTED]
Remember Cernan and Schmitt
Index: exclude.c
===
RCS file: /cvsroot/rsync/exclude.c,v
retrieving revision 1.42
diff -u -r1.42 exclude.c
--- exclude.c 18 Feb 2002 19:10:28 - 1.42
+++ exclude.c 7 Mar 2002 20:56:02 -
@@ -219,8 +219,14 @@
int fatal, int include)
{
struct exclude_struct **list=list1;
- FILE *f = fopen(fname,r);
+ FILE *f;
char line[MAXPATHLEN];
+
+ if (strcmp(fname, -)) {
+ f = fopen(fname,r);
+ } else {
+ f = fdopen(0, r);
+ }
if (!f) {
if (fatal) {
rsyserr(FERROR, errno,
Index: generator.c
===
RCS file: /cvsroot/rsync/generator.c,v
retrieving revision 1.33
diff -u -r1.33 generator.c
--- generator.c 7 Feb 2002 16:36:12 - 1.33
+++ generator.c 7 Mar 2002 20:56:02 -
@@ -42,6 +42,8 @@
extern int always_checksum;
extern int modify_window;
extern char *compare_dest;
+extern int compare_perms;
+extern int link_dest;
/* choose whether to skip a particular file */
@@ -51,6 +53,15 @@
if (st-st_size != file-length) {
return 0;
}
+ if (compare_perms) {
+ if((st-st_mode ~_S_IFMT) != (file-mode ~_S_IFMT)) {
+ return 0;
+ }
+ if (st-st_uid != file-uid || st-st_gid != file-gid) {
+ return 0;
+ }
+ }
+
/* if always checksum is set then we use the checksum instead
of the file time to determine whether to sync */
@@ -352,6 +363,17 @@
statret = -1;
if (statret == -1)
errno = saveerrno;
+#if HAVE_LINK
+ else if (link_dest)
+ if (do_link(fnamecmpbuf, fname) != 0) {
+ if (verbose 0)
+ rprintf(FINFO,link %s = %s : %s\n,
+ fnamecmpbuf,
+ fname,
+ strerror(errno));
+ fnamecmp = fnamecmpbuf;
+ }
+#endif
else
fnamecmp = fnamecmpbuf;
}
Index: options.c
===
RCS file: /cvsroot/rsync/options.c,v
retrieving revision 1.80
diff -u -r1.80 options.c
--- options.c
[PATCH][RFC] space saving incrementals
Please CC me directly as i'm not on the list.
I have attached a patch against latest CVS (cvs diff -u)
that adds the following functionality. I can break it up if
you would prefer it in pieces. Comments welcome.
o add compare-perms option
This creates a new inode for a file even if only
the perms have changed. This way if a file
outside of destdir is hardlinked to a dentry
inside destdir the permissions (uid, gid,
mode) will be untouched preserving history
etc.
o link-dest option
After setting compare_dest this causes
unchanged files in destdir to be hardlinked
to link-dest.
o modified make_exclude_list to support stdin
if --exclude-from has argument of -
stdin will be read.
This lets us pipe a include/exclude list
into rsync so that we can generate it on the
fly.
The upshot of these is to allow rsync to make incremental
backups without modifying earlier versions but keep every
version as a complete tree. It then becomes possible to
make tapes or restore from any image.
Although --compare-perms saves on block count this patch
only applies it to regular files so symlinks and device
nodes will still chew up inodes. :(
for the sake of an example here is the command line i'm using:
/site/bin/rsync -v --stats -a -H -e ssh --compare-perms
--delete --delete-excluded --numeric-ids --exclude-from -
--link-dest /e/backup1/home/update1/tree leto:/efs/home/
/e/backup1/home/update2/tree /e/backup1/home/update2/log
i have tried to keep my changes small and consistent with the
existing coding style. Feel free to dink with it. I only
care about the performance.
--
J.W. SchultzPegasystems Technologies
email address: [EMAIL PROTECTED]
Remember Cernan and Schmitt
Index: exclude.c
===
RCS file: /cvsroot/rsync/exclude.c,v
retrieving revision 1.42
diff -u -r1.42 exclude.c
--- exclude.c 18 Feb 2002 19:10:28 - 1.42
+++ exclude.c 7 Mar 2002 20:56:02 -
@@ -219,8 +219,14 @@
int fatal, int include)
{
struct exclude_struct **list=list1;
- FILE *f = fopen(fname,r);
+ FILE *f;
char line[MAXPATHLEN];
+
+ if (strcmp(fname, -)) {
+ f = fopen(fname,r);
+ } else {
+ f = fdopen(0, r);
+ }
if (!f) {
if (fatal) {
rsyserr(FERROR, errno,
Index: generator.c
===
RCS file: /cvsroot/rsync/generator.c,v
retrieving revision 1.33
diff -u -r1.33 generator.c
--- generator.c 7 Feb 2002 16:36:12 - 1.33
+++ generator.c 7 Mar 2002 20:56:02 -
@@ -42,6 +42,8 @@
extern int always_checksum;
extern int modify_window;
extern char *compare_dest;
+extern int compare_perms;
+extern int link_dest;
/* choose whether to skip a particular file */
@@ -51,6 +53,15 @@
if (st-st_size != file-length) {
return 0;
}
+ if (compare_perms) {
+ if((st-st_mode ~_S_IFMT) != (file-mode ~_S_IFMT)) {
+ return 0;
+ }
+ if (st-st_uid != file-uid || st-st_gid != file-gid) {
+ return 0;
+ }
+ }
+
/* if always checksum is set then we use the checksum instead
of the file time to determine whether to sync */
@@ -352,6 +363,17 @@
statret = -1;
if (statret == -1)
errno = saveerrno;
+#if HAVE_LINK
+ else if (link_dest)
+ if (do_link(fnamecmpbuf, fname) != 0) {
+ if (verbose 0)
+ rprintf(FINFO,link %s = %s : %s\n,
+ fnamecmpbuf,
+ fname,
+ strerror(errno));
+ fnamecmp = fnamecmpbuf;
+ }
+#endif
else
fnamecmp = fnamecmpbuf;
}
Index: options.c
===
RCS file: /cvsroot/rsync/options.c,v
retrieving revision 1.80
diff -u -r1.80 options.c
--- options.c 27 Feb 2002 22:49:57 - 1.80
+++ options.c 7 Mar 2002 20:56:03 -
@@ -106,6 +106,9 @@
int always_checksum = 0;
int list_only = 0;
+int compare_perms = 0;
+int link_dest = 0;
+
char *batch_prefix = NULL;
static int modify_window_set;
@@ -195,6 +198,7 @@
rprintf(F, --safe-linksignore links
