Many days ago I asked for (and received, thank you) help about
'dsmc select' and 'dsmc expire' with the '-filelist' option. Here
there has been installed a 'Vault' like a HMC that moves files and
email messages that have not been used in some time out of the file
server and exchange server (can you tell we're using Microsoft)
to a different location. The 'Vault' server leaves behind 'stub'
files and the hooks in the OS allow for users to open the stub and
receive the file as if the file were still local to the file server.
This behavior is causing the TSM server some problems. The OS
hooks do set the 'O' or 'Offline' flag when a file is moved and a
stub remains.
Attached (if the list server lets me do attachments) is the program
I mentioned that I would pass on once I'd finished writing it. The
name is 'dsmpmj' for 'DSM - Poor Man's Journal' since the program
also does the journalling feature of the B/A client.
The program reads yesterday's file, scans the local filesystem,
and then compares the two lists. Anything that existed yesterday,
but is not there today goes into the EXPIRELIST. Anything that
has a timestamp newer than the timestamp of the YESTERDAY file goes
into BACKUPLIST. If any part of the lines in EXCLUDELIST match the
local files, the files are excluded. That means if BACKUPLIST has
a line '.sys' this will match the file 'c:\user\intricate.system',
the latter file not making it to BACKUPLIST.
The way to run this is:
dsmpmj -y YESTERDAY -b BACKUPLIST -e EXPIRELIST -x EXCLUDELIST
There is also a -r REFERENCEFILE that can be used the first time
the program is called to give dsmpmj a place to start for it's
backup timestamp (i.e. four days ago).
Once you have run dsmpmj feed the two generated files to dsmc:
c:\> dsmpmj -y yesterday -b backup -e expire -x excludes
c:\> dsmc expire -filelist=expire
c:\> dsmc sel -filelist=backup
That's about it. I hope it works for other people.
Mike
static char *rcsid = "$Id$";
/*
* $Log$
*/
/* generate a list of all files on the server
* accept a list of all files in the TSM library
* compare the two lists of files
* output the differences... files on the server not in the library
*/
#include <stdio.h>
#include <io.h>
#include <time.h>
#include <string.h>
#include <direct.h>
#include <windows.h>
#include <winbase.h>
typedef struct _file {
FILETIME mtime;
unsigned int size;
unsigned int flags;
char *name;
} File;
/* global variables */
int verbose = 0;
/* logfile */
static FILE *logfp;
static char *logfn;
/* simple error routing */
void err(char *msg)
{
(void) fprintf(stderr, "dsmpmj: %s\n", msg);
if(logfp) {
(void) fprintf(logfp, "dsmpmj: %s\n", msg);
(void) fclose(logfp);
}
(void) exit(1);
}
/* warning messages */
void warn(char *msg)
{
time_t t;
char buf[256];
(void) time(&t);
(void) strcpy(buf, ctime(&t));
if(strchr(buf, '\n')) {
*(strchr(buf, '\n')) = '\0';
}
(void) fprintf(stderr, "%s: dsmpmj: %s\n", buf, msg);
if(logfp) {
(void) fprintf(logfp, "%s: dsmpmj: %s\n", buf, msg);
}
}
/* comparison function for qsort */
int filesCompare(const void *a, const void *b)
{
File *af = (File *) a;
File *bf = (File *) b;
return strcmp(af->name, bf->name);
}
/* comparison function for bsearch */
int keyCompare(const void *a, const void *b)
{
File *bf = (File *) b;
return strcmp((char *) a, bf->name);
}
/* change text to lower case */
void toLower(char *s)
{
while(s && *s) {
*s = tolower(*s);
s++;
}
}
/* load the file previos day's run list of files */
File *loadFiles(char *fn, int *n, int missingok)
{
int i, nfiles;
FILE *fp;
File *files;
char buf[1024], *p;
if((fp = fopen(fn, "r")) == (FILE *) NULL) {
if(missingok) {
*n = 0;
return (File*) NULL;
}
(void) sprintf(buf, "unable to read file '%s'", fn);
err(buf);
}
/* how many files found? */
if(verbose) {
warn("counting output from filesystem scans");
}
(void) fseek(fp, 0, SEEK_SET); /* rewind to beginning of file
*/
nfiles = 0;
while(fgets(buf, sizeof(buf) - 1, fp)) { /* count number of
lines/files in fn */
nfiles++;
}
if(verbose) {
(void) sprintf(buf, "counted a total of %d filenames", nfiles);
warn(buf);
}
/* allocate memory for the file data */
if((files = (File *) malloc(sizeof(File) * nfiles)) == (File *) NULL) {
err("unable to allocate memory for local files");
}
if(verbose) {
(void) sprintf(buf, "allocate %d bytes", sizeof(File) * nfiles);
warn(buf);
}
/* load the data */
i = 0;
(void) fseek(fp, 0, SEEK_SET); /* rewind to beginning of file
*/
while(fgets(buf, sizeof(buf) - 1, fp)) {
files[i].flags = (unsigned int) atoi(buf);
p = strchr(buf, '\t') + 1;
files[i].mtime.dwLowDateTime = (unsigned int) atoi(p);
p = strchr(p, '\t') + 1;
files[i].mtime.dwHighDateTime = (unsigned int) atoi(p);
p = strchr(p, '\t') + 1;
files[i].size = (unsigned int) atoi(p);
p = strchr(p, '\t') + 1;
if(strchr(p, '\n')) {
*(strchr(p, '\n')) = '\0';
}
files[i++].name = strdup(p);
}
(void) fclose(fp);
/* sort the lines */
(void) qsort(files, nfiles, sizeof(File), filesCompare);
/* remove the temporary file */
(void) unlink(fn);
if(verbose) {
warn("done");
}
*n = nfiles;
return files;
}
/* get a list of all local files */
void findFiles(FILE *fp, char drive, char *fn, int *nfiles)
{
HANDLE h;
WIN32_FIND_DATA finddata;
char dir[1024], buf[2048];
LPVOID msgbuf;
int errnum;
(void) strcpy(dir, fn);
(void) strcat(dir, "\\*.*");
h = FindFirstFile(dir, &finddata);
if(h == INVALID_HANDLE_VALUE) {
errnum = GetLastError();
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
FORMAT_MESSAGE_FROM_SYSTEM,
NULL, errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPSTR) &msgbuf, 0, NULL);
if(strchr(msgbuf, '\n')) {
*(strchr(msgbuf, '\n')) = '\0';
}
(void) sprintf(buf, "FindFirstFile() failed with error %d for
path '%s': %s", errnum, dir, msgbuf);
warn(buf);
return;
}
do {
if(!strcmp(finddata.cFileName, ".") ||
!strcmp(finddata.cFileName, "..")) {
continue;
}
if(finddata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
char path[1024];
(void) strcpy(path, dir);
if(strchr(path, '*')) {
*(strchr(path, '*')) = '\0';
}
(void) strcat(path, finddata.cFileName);
findFiles(fp, drive, path, nfiles);
} else {
char path[1024];
(void) strcpy(path, dir);
if(strchr(path, '*')) {
*(strchr(path, '*')) = '\0';
}
(void) strcat(path, finddata.cFileName);
(void) fprintf(fp, "%u\t%u\t%u\t%u\t%c:%s\n",
finddata.dwFileAttributes,
/* flags */
finddata.ftLastWriteTime.dwLowDateTime,
/* mtime (first part) */
finddata.ftLastWriteTime.dwHighDateTime,
/* mtime (second part) */
finddata.nFileSizeLow,
/* size */
drive,
/* drive letter */
path);
/* path on that drive */
}
(*nfiles)++;
if(verbose && (*nfiles % 1000) == 0) {
(void) fprintf(stderr, "scanned %d files\n", *nfiles);
}
} while(FindNextFile(h, &finddata));
FindClose(h);
}
/* return a list of all local files */
File *getLocalFiles(char *drives, int *n)
{
FILE *fp;
int i, nfiles;
char buf[1024], fn[1024], d;
File *files;
(void) sprintf(fn, "%s%s", getenv("TEMP"), tmpnam((char *) NULL));
(void) unlink(fn);
if(verbose) {
(void) sprintf(buf, "scanning drives '%s' temporary file at
%s", drives, fn);
warn(buf);
}
if((fp = fopen(fn, "w+")) == (FILE *) NULL) {
(void) sprintf(buf, "unable to create temporary file for local
files '%s'", fn);
err(buf);
}
nfiles = 0;
for(i = 3; i < 27; i++) { /* switch drives for finding
files */
if(drives) { /* if only doing
specific drives */
d = (char) ('a' + i - 1);
if(strchr(drives, d)) {
if(verbose) {
(void) sprintf(buf, "changing to drive
%c:", 'a' + i - 1);
warn(buf);
}
if(_chdrive(i)) {
continue;
}
*buf = '\0';
findFiles(fp, (char) ('a' + i - 1), buf,
&nfiles);
}
} else {
if(verbose) {
(void) sprintf(buf, "changing to drive %c:",
'a' + i - 1);
warn(buf);
}
if(_chdrive(i)) {
continue;
}
*buf = '\0';
findFiles(fp, (char) ('a' + i - 1), buf, &nfiles);
}
}
(void) fclose(fp);
if(verbose) {
(void) sprintf(buf, "scanned %d files", nfiles);
warn(buf);
}
/* reload the files */
files = loadFiles(fn, n, 0);
/* remove the temporary file */
(void) unlink(fn);
return files;
}
/* load exclude patterns */
char **loadExcludePatterns(char *fn, int *n)
{
FILE *fp;
int i, nlines;
char buf[1024], **lines;
lines = (char **) NULL;
nlines = *n = 0;
/* open the excludes file */
if((fp = fopen(fn, "r")) == (FILE *) NULL) {
(void) sprintf(buf, "unable to load exclude patterns from file
'%s': skipping excludes", fn);
warn(buf);
return lines;
}
/* how many files found? */
if(verbose) {
warn("counting exclude patterns");
}
nlines = 0;
while(fgets(buf, sizeof(buf) - 1, fp)) { /* count number of
lines/files in fn */
nlines++;
}
if(verbose) {
(void) sprintf(buf, "counted a total of %d filenames", nlines);
warn(buf);
}
/* read the lines into memory */
if(verbose) {
warn("loading exclude patterns");
}
(void) fseek(fp, 0, SEEK_SET); /* rewind to beginning of file
*/
if((lines = (char **) malloc(sizeof(char *) * nlines)) == (char **)
NULL) {
err("unable to allocate memory for exclude patterns");
}
i = 0;
while(fgets(buf, sizeof(buf) - 1, fp)) {
if(strchr(buf, '\n')) {
*(strchr(buf, '\n')) = '\0';
}
toLower(buf);
lines[i++] = strdup(buf);
}
(void) fclose(fp);
if(verbose) {
(void) sprintf(buf, "%d exclude patterns loaded", nlines);
warn(buf);
}
*n = nlines;
return (char **) lines;
}
/* write today's list to disk for tomorrow */
void writeFiles(char *fn, File *files, int nfiles)
{
FILE *fp;
int i;
char buf[1024];
if(verbose) {
(void) sprintf(buf, "writing to file '%s'", fn);
warn(buf);
}
/* open the file */
if((fp = fopen(fn, "w")) == (FILE *) NULL) {
(void) sprintf(buf, "unable to create file '%s'", fn);
err(buf);
}
/* write the files to the file */
for(i = 0; i < nfiles; i++) {
if(!files[i].name) {
continue;
}
(void) fprintf(fp, "%u\t%u\t%u\t%u\t%s\n",
files[i].flags,
/* flags */
files[i].mtime.dwLowDateTime,
/* mtime (first part) */
files[i].mtime.dwHighDateTime,
/* mtime (second part) */
files[i].size,
/* size */
files[i].name);
/* filename */
}
/* close the file */
(void) fclose(fp);
if(verbose) {
warn("done");
}
}
/* compare the previous files with the current files
* generate the expire list
*/
void genExpireList(char *fn, File *tfiles, int ntfiles, File *yfiles, int
nyfiles)
{
int i, nexpired;
FILE *fp;
char buf[1024];
/* make sure the previous file is gone */
(void) unlink(fn);
/* if there are no previous files, no need to expire any files */
if(nyfiles == 0) {
return;
}
if(verbose) {
(void) sprintf(buf, "creating expire list to file '%s'", fn);
warn(buf);
}
/* open file for list of expired files */
if((fp = fopen(fn, "w")) == (FILE *) NULL) {
(void) sprintf(buf, "unable to create expired list file '%s'",
fn);
err(buf);
}
/* create the expired list */
nexpired = 0;
for(i = 0; i < nyfiles; i++) {
if(!bsearch(yfiles[i].name, tfiles, ntfiles, sizeof(File),
keyCompare)) {
(void) fprintf(fp, "\"%s\"\n", yfiles[i].name);
nexpired++;
}
}
/* close the file */
(void) fclose(fp);
if(verbose) {
(void) sprintf(buf, "done: expiring %d files", nexpired);
warn(buf);
}
/* if there are no files to expire, remove the expire list */
if(!nexpired) {
(void) unlink(fn);
if(verbose) {
warn("no files to expire: removing expire list");
}
}
}
/* using today's list exclude all files not modified since the timestamp
* also exclude all files with the Offline (O) flag
*/
void genBackupList(char *fn, File *files, int nfiles,
FILETIME *timestamp, char **pats, int npats)
{
FILE *fp;
int i, j, nbackup;
char buf[1024];
/* open the file */
if((fp = fopen(fn, "w")) == (FILE *) NULL) {
(void) sprintf(buf, "unable to create backup list file '%s'",
fn);
err(buf);
}
if(verbose) {
(void) sprintf(buf, "creating backup list to file '%s'", fn);
warn(buf);
}
/* exclude files */
nbackup = 0;
for(i = 0; i < nfiles; i++) {
/* if for some reason this file has already been handled */
if(!files[i].name) {
continue;
}
/* first remove all files from today's list where the timestamp
has not changed */
if(CompareFileTime(&files[i].mtime, timestamp) < 0) { /*
timestamp */
(void) free(files[i].name);
files[i].name = (char *) NULL;
continue;
}
/* remove any files with the OFFLINE flag set */
if((files[i].flags & FILE_ATTRIBUTE_OFFLINE) ==
FILE_ATTRIBUTE_OFFLINE) {
(void) free(files[i].name);
files[i].name = (char *) NULL;
continue;
}
/* remove any files that match something in the exclude list */
(void) strcpy(buf, files[i].name);
toLower(buf);
for(j = 0; j < npats; j++) {
if(strstr(buf, pats[j])) {
(void) free(files[i].name);
files[i].name = (char *) NULL;
break;
}
}
/* if the file made it through the exclude checks */
if(files[i].name) {
(void) fprintf(fp, "\"%s\"\n", files[i].name);
nbackup++;
}
}
/* close the file */
(void) fclose(fp);
if(verbose) {
(void) sprintf(buf, "done: backing up %d files", nbackup);
warn(buf);
}
}
void usage(char *msg)
{
(void) fprintf(stderr, "%s\n", msg);
(void) fprintf(stderr, "usage: dsmpmj [-v] -y SAVED-FILELIST -b
BACKUP-LIST -e EXPIRE-LIST -x EXCLUDE-PATTERNS -d DRIVE-LETTER [-r FILE][-l
LOGFILE]\n\n");
(void) fprintf(stderr, "\t-v - verbose\n");
(void) fprintf(stderr, "\t-y SAVED-FILELIST - filename of the file
having yesterday's files\n");
(void) fprintf(stderr, "\t-b BACKUP-LIST - filename to use for the
files to backup today\n");
(void) fprintf(stderr, "\t\t(ex. dsmc selective
-filelist=BACKUP-LIST\n");
(void) fprintf(stderr, "\t-e EXPIRE-LIST - filename to use for
storing the files to expire\n");
(void) fprintf(stderr, "\t\t(ex. dsmc expire -filelist=EXPIRE-LIST\n");
(void) fprintf(stderr, "\t-x EXCLUDE-PATTERNS - filename of the file
having the patterns of files to exclude from backup\n");
(void) fprintf(stderr, "\t-r REFERENCE-FILE - use the timestamp of
this file as a reference for backups\n");
(void) fprintf(stderr, "\t-d DRIVE-LETTER - drive letter to scan
(ex. -d cdegst)\n");
(void) fprintf(stderr, "\t-l LOGFILE - write errors here in
addition to stderr\n");
(void) exit(0);
}
int main(int argc, char **argv)
{
File *tfiles, *yfiles;
char **xfiles;
int ntfiles, nyfiles, nxfiles;
char *drives;
int i;
char *yfn, *bfn, *efn, *xfn, *rfn;
HANDLE h;
WIN32_FIND_DATA finddata;
/* parse the command-line */
yfn = bfn = efn = xfn = rfn = drives = (char *) NULL;
i = 1;
while(i < argc && argv[i] != (char *) NULL) {
if(!strcmp(argv[i], "-v")) {
verbose = 1;
}
if(!strcmp(argv[i], "-y")) {
yfn = argv[++i];
}
if(!strcmp(argv[i], "-b")) {
bfn = argv[++i];
}
if(!strcmp(argv[i], "-e")) {
efn = argv[++i];
}
if(!strcmp(argv[i], "-x")) {
xfn = argv[++i];
}
if(!strcmp(argv[i], "-r")) {
rfn = argv[++i];
}
if(!strcmp(argv[i], "-d")) {
drives = argv[++i];
}
if(!strcmp(argv[i], "-l")) {
logfn = argv[++i];
}
i++;
}
/* if no rfn is given, use yfn */
if(!rfn) {
rfn = yfn;
}
/* check command line arguments */
if(!yfn) {
usage("no -y filename given");
} else if(!bfn) {
usage("no backup filename given with -b FILENAME");
} else if(!efn) {
usage("no expire filename given with -e FILENAME");
} else if(!xfn) {
usage("no excludes given with -x FILENAME");
} else if(!drives) {
usage("no drives given with -d");
}
/* if logging, open the log file */
if(logfn) {
if((logfp = fopen(logfn, "w")) == (FILE *) NULL) {
err("unable to create logfile");
}
}
if(_chdrive(3)) {
err("trying to find dsmc: unable to change to C: drive\n");
}
if(chdir("/")) {
err("unable to cd to \\\n");
}
/* get a timestamp of the previous day's files */
h = FindFirstFile(rfn, &finddata);
if(h == (HANDLE) -1) {
finddata.ftLastWriteTime.dwLowDateTime = 0;
finddata.ftLastWriteTime.dwHighDateTime = 0;
}
tfiles = getLocalFiles(drives, &ntfiles); /* what files exist
today? */
yfiles = loadFiles(yfn, &nyfiles, 1); /* load the files that
existed yesterday */
writeFiles(yfn, tfiles, ntfiles); /* write
today's files for tomorrow */
genExpireList(efn, tfiles, ntfiles, yfiles, nyfiles); /* what files
are gone */
xfiles = loadExcludePatterns(xfn, &nxfiles); /* what files to
ignore? */
genBackupList(bfn, tfiles, ntfiles, &finddata.ftLastWriteTime, xfiles,
nxfiles);
if(logfp) {
(void) fclose(logfp);
}
return 0;
}
