make a qid-faithful copy of the differences between two directory structures. the manual page has a better description. Notes: Fri Apr 6 10:18:07 EDT 2007 rsc This is interesting but I'm not convinced it is generally useful enough to warrant putting in the main distribution and needing to maintain. In particular I don't really understand why it's useful to preserve qids. Qids are for the mount table, nothing more. Reference: /n/sources/patch/saved/cphist Date: Thu Mar 8 00:16:09 CET 2007 Signed-off-by: quanstro@quanstro.net Reviewed-by: geoff --- /sys/man/8/replica Thu Mar 8 00:14:46 2007 +++ /sys/man/8/replica Thu Mar 8 00:14:42 2007 @@ -1,6 +1,6 @@ .TH REPLICA 8 .SH NAME -applychanges, applylog, compactdb, updatedb \- simple client-server replica management +applychanges, applylog, compactdb, cphist, updatedb\- simple client-server replica management .SH SYNOPSIS .B replica/compactdb .I db @@ -69,8 +69,20 @@ [ .I path ... ] +.br +.B replica/cphist +[ +.B -evn +] +[ +.B -l +.I lineno +] +.I clientroot +.I serverroot +.I oldserverroot .SH DESCRIPTION -These four tools collectively provide simple log-based +These five tools collectively provide simple log-based client-server replica management. The shell scripts described in .IR replica (1) @@ -260,6 +272,47 @@ or .B -s flags. +.PP +.I Cphist +was designed to copy a dump faithfully from one file server to the next. +Unlike other replica tools it takes special precautions to ensure that +whenever files on the +.I server +and +.I oldserver +share a +.BR qid.path , +the client's corresponding file will be modified and +whenever they do not, the client's file will be deleted and +a copy made from +.IR server . +Thus the +.BR Qid -relationship +between the file on +.I server +and +.I oldserver +will be the same as for the current file on the client +and that file in the client's most recent dump. +Also, +.B muid +is set to preserve the output of +.BR history (1). +The +.B -e +flag toggles exiting on first error. The default is to exit on +any error. The +.B -l +.I lineno +sets the first line of the log (the +.I gen +field) to be processed while +.I -v +sets verbose output and +.I -n +prints what +.I cphist +would do, but does nothing. .SH EXAMPLE One might keep a client kfs file system up-to-date @@ -309,8 +362,19 @@ .SH SEE ALSO .IR replica (1) .SH BUGS -These tools assume that +These tools (excepting +.IR cphist ) +assume that .I mtime combined with .I length is a good indicator of changes to a file's contents. +.PP +Perhaps +.I cphist +should inspect the source files directly rather than +using a replica log, as it picks through the +.I oldserver +to discover +.B muid +and other bits of information not in the log. --- /sys/src/cmd/replica/cphist.c Thu Jan 1 00:00:00 1970 +++ /sys/src/cmd/replica/cphist.c Thu Mar 8 00:14:55 2007 @@ -0,0 +1,433 @@ +#include "all.h" + +enum{ + Dst, + Src, + Osrc, +}; + +int lineno; +int errors; +int errorexit = 1; +int donothing; +int verbose; +char *root[3]; +char nbuf[3][10240]; +char *mkname(char*, int, char*, char*); +int copyfile(char*, char*, char*, Dir*, int, int*); + +void +chat(char *f, ...) +{ + Fmt fmt; + char buf[256]; + va_list arg; + + if(!verbose) + return; + + fmtfdinit(&fmt, 1, buf, sizeof buf); + va_start(arg, f); + fmtvprint(&fmt, f, arg); + va_end(arg); + fmtfdflush(&fmt); +} + +void +fatal(char *f, ...) +{ + char *s; + va_list arg; + + va_start(arg, f); + s = vsmprint(f, arg); + va_end(arg); + fprint(2, "%d: error: %s\n", lineno, s); + free(s); + exits("errors"); +} + +void +error(char *f, ...) +{ + char *s; + va_list arg; + + va_start(arg, f); + s = vsmprint(f, arg); + va_end(arg); + fprint(2, "%d: error: %s\n", lineno, s); + free(s); + errors = 1; + if(errorexit) + exits("errors"); +} + +void +warn(char *f, ...) +{ + char *s; + va_list arg; + + va_start(arg, f); + s = vsmprint(f, arg); + va_end(arg); + fprint(2, "%d: warn: %s\n", lineno, s); + free(s); +} + +int +reason(char *s) +{ + char e[ERRMAX]; + + rerrstr(e, sizeof e); + if(strstr(s, e) == 0) + return 0; + return -1; +} + +char* +mkname(char *buf, int nbuf, char *a, char *b) +{ + if(strlen(a)+strlen(b)+2 > nbuf) + fatal("name too long"); + + strcpy(buf, a); + if(a[strlen(a)-1] != '/') + strcat(buf, "/"); + strcat(buf, b); + return buf; +} + +int +isdir(char *s) +{ + ulong m; + Dir *d; + + if((d = dirstat(s)) == nil) + return 0; + m = d->mode; + free(d); + return (m&DMDIR) != 0; +} + +void +usage(void) +{ + fprint(2, "usage: replica/cphist [-nv] dstroot newsrcroot oldsrcroot < log\n"); + exits("usage"); +} + +void +notexists(char *path) +{ + if(access(path, AEXIST) == -1) + fatal("%r"); +} + +void +updatestat(char *name, int fd, Dir *t) +{ + if(dirfwstat(fd, t) < 0) + error("dirfwstat %q: %r", name); +} + +long +preadn(int fd, void *av, long n, vlong p) +{ + char *a; + long m, t; + + a = av; + t = 0; + while(t < n){ + m = pread(fd, a+t, n-t, p); + if(m <= 0){ + if(t == 0) + return m; + break; + } + t += m; + p += m; + } + return t; +} + +char buf[8192]; +char obuf[8192]; + +int +copy(int fdf, int fdt, char *from, char *to, vlong p) +{ + long n; + + while(n = pread(fdf, buf, sizeof buf, p)){ + if(n < 0) + error("reading %s: %r", from); + if(pwrite(fdt, buf, n, p) != n) + error("writing %s: %r\n", to); + p += n; +// len -= n; + } + return 0; +} + +int +change0(int fd[3], char *name[3], Dir *d[3]) +{ + vlong o, sz; + long n; + + if(d[Osrc]->qid.path != d[Src]->qid.path){ + close(fd[Dst]); + if(remove(name[Dst]) == -1) + fatal("can't remove old file %q: %r", name[Dst]); + if((fd[Dst] = create(name[Dst], ORDWR, 0666)) == -1) + fatal("create %q: %r", name[Dst]); + return copy(fd[Src], fd[Dst], name[Src], name[Dst], 0); + } + if((d[Osrc]->qid.type&QTAPPEND) && (d[Src]->qid.type&QTAPPEND)) + return copy(fd[Src], fd[Dst], name[Src], name[Dst], d[Osrc]->length); + /* hard case. we need to copy only changed blocks */ + sz = d[Src]->length; + if(sz > d[Osrc]->length) + sz = d[Osrc]->length; + for(o = 0; o < sz; o += n){ + n = 8192; + if(n > sz-o) + n = sz-o; + if(preadn(fd[Src], buf, n, o) != n) + fatal("pread %q: short read", name[Src]); + if(preadn(fd[Osrc], obuf, n, o) != n) + fatal("pread %q: short read", name[Src]); + if(memcmp(buf, obuf, n) != 0){ + if(pwrite(fd[Dst], buf, n, o) != n) + fatal("pwrite %q: %r", name[Dst]); + } + } + if(sz < d[Src]->length) + return copy(fd[Src], fd[Dst], name[Src], name[Dst], sz); + return 0; +} + +static char muid[28]; + +void +assignmuid(int fd, char *name, Dir *rd) +{ + Dir *d; + + if((d = dirfstat(fd)) == 0) + error("dirfstat %q: %r", name); + strecpy(muid, muid+sizeof muid-1, d->muid); + rd->muid = muid; + free(d); +} + +int +change(int fd[3], char *name[3], Dir *rd) +{ + Dir *d[3]; + int n, i; + + for(i = 0; i < 3; i++) + if((d[i] = dirfstat(fd[i])) == 0) + error("dirfstat %q: %r", name[i]); + n = change0(fd, name, d); + strecpy(muid, muid+sizeof muid-1, d[Src]->muid); + rd->muid = muid; + for(i = 0; i < 3; i++) + free(d[i]); + return n; +} + +char * +basename(char *name) +{ + char *r; + + r = strrchr(name, '/'); + if(r) + return r+1; + return name; +} + +int mtab[] = { +[Dst] ORDWR, +[Src] OREAD, +[Osrc] OREAD, +}; + +void +doclose(int fd[3]) +{ + for(int i = 0; i < 3; i++){ + close(fd[i]); + fd[i] = -1; + } +} + +void +fddump(char *file[3], int fd[3]) +{ + for(int i = 0; i < 3; i++) + print(">> %s: %d %d %.3s\n", file[i], fd[i], mtab[i], "dstsrcosr"+3*i); +} + +int +doopen(char *file[3], int fd[3]) +{ + int i; + + for(i = 0; i < 3; i++){ + if((fd[i] = open(file[i], mtab[i])) == -1){ + if(reason("file is locked") == -1) + error("open: %r", file[i]); + else{ + // gruesome hack to get around exclusive mode files. + if(i == 2) + fd[2] = dup(fd[1], -1); + if(fd[i] == -1) + warn("open %d: %r", mtab[i], file[i]); + else + continue; + } + fddump(file, fd); + doclose(fd); + error("open: %r", file[i]); + return -1; + } + } + return 0; +} + +void +main(int argc, char **argv) +{ + char *f[10], *name, *s, *t, verb, *file[3]; + int fd[3], i, nf, line0; + Dir rd; + Biobuf bin; + + quotefmtinstall(); + line0 = 0; + ARGBEGIN{ + case 'n': + donothing = 1; + verbose = 1; + break; + case 'e': + errorexit ^= 1; + break; + case 'l': + line0 = atoi(EARGF(usage())); + break; + case 'v': + verbose++; + break; + default: + usage(); + }ARGEND + + if(argc != 3) + usage(); + for(i = 0; i < 3; i++) + if(!isdir(root[i] = argv[i])) + fatal("bad root directory %q", argv[i]); + + Binit(&bin, 0, OREAD); + for(; s=Brdstr(&bin, '\n', 1); free(s)){ + t = estrdup(s); + nf = tokenize(s, f, nelem(f)); + if(nf != 10 || strlen(f[2]) != 1){ + error("bad log entry <%s>\n", t); + free(t); + continue; + } + free(t); +// now = strtoul(f[0], 0, 0); + lineno = atoi(f[1]); + if(lineno < line0) + continue; + verb = f[2][0]; + name = f[3]; + file[Dst] = mkname(nbuf[Dst], sizeof nbuf[Dst], root[Dst], name); + if(strcmp(f[4], "-") == 0) + f[4] = f[3]; + file[Src] = mkname(nbuf[Src], sizeof nbuf[Src], root[Src], f[4]); + file[Osrc] = mkname(nbuf[Osrc], sizeof nbuf[Osrc], root[Osrc], f[4]); + nulldir(&rd); +// rd.name = basename(f[4]); + rd.mode = strtoul(f[5], 0, 8); + rd.uid = f[6]; + rd.gid = f[7]; + rd.mtime = strtoul(f[8], 0, 10); + rd.length = strtoll(f[9], 0, 10); + + switch(verb){ + case 'd': + chat("d %q\n", name); + if(donothing) + break; + if(remove(file[Dst]) == -1){ + if(access(file[Dst], AEXIST) == -1) + warn("remove: %r", file[Dst]); + else + error("remove: %r", file[Dst]); + continue; + } + break; + case 'a': + chat("a %q %luo %q %q %lud\n", name, rd.mode, rd.uid, rd.gid, rd.mtime); + if(donothing) + break; + if((fd[Src] = open(file[Src], OREAD)) == -1) + fatal("open %q: %r", file[Src]); + if(rd.mode&DMDIR) + i = OREAD|OEXEC; + else + i = ORDWR; + if((fd[Dst] = create(file[Dst], i, rd.mode)) == -1) + fatal("create: %r"); + if((rd.mode&DMDIR) == 0) + copy(fd[Src], fd[Dst], file[Src], file[Dst], 0); + assignmuid(fd[Src], file[Src], &rd); + updatestat(file[Dst], fd[Dst], &rd); + close(fd[Src]); + close(fd[Dst]); + break; + case 'c': /* change contents */ + chat("c %q\n", name); + if(donothing) + break; + werrstr(""); + if(doopen(file, fd) == -1) + continue; + change(fd, file, &rd); + updatestat(file[Dst], fd[Dst], &rd); + doclose(fd); + break; + case 'm': /* change metadata */ + notexists(file[Src]); + chat("m %q %luo %q %q %lud\n", name, rd.mode, rd.uid, rd.gid, rd.mtime); + if(donothing) + break; + if((fd[Src] = open(file[Src], OREAD)) < 0) + fatal("open %q: %r", file[Src]); + if((fd[Dst] = open(file[Dst], OREAD)) < 0) + fatal("open %q: %r", file[Dst]); + if(rd.mode&DMDIR) + rd.length = ~0ULL; + assignmuid(fd[Src], file[Src], &rd); + updatestat(file[Dst], fd[Dst], &rd); + close(fd[Src]); + close(fd[Dst]); + break; + } + } + if(errors) + exits("errors"); + exits(nil); +}