from brian stuart

add additional level of indirection so a snapshot requires updating one block
and writing one new one.  It also opens up the possibility for growing
the underlying store after the snapshotting has already been started.
It's been running for a couple of days here, so I think it should be
fairly solid.  I haven't tested changing the sector size to 4096, but
as I mentioned before, so little depends on it, that I expect it would
work fine.  I'm continuing to bring the file system itself up to a
point where it should be more generally usable.

Reference: /n/atom/patch/applied/snap2
Date: Mon Feb 10 17:39:43 CET 2014
Signed-off-by: quanstro@quanstro.net

--- /sys/src/nix/port/devsnap.c	Mon Feb 10 17:39:31 2014
+++ /sys/src/nix/port/devsnap.c	Mon Feb 10 17:39:32 2014
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2013, Coraid, Inc.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Coraid nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL CORAID BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
 #include	"u.h"
 #include	"../port/lib.h"
 #include	"mem.h"
@@ -14,7 +41,6 @@
 
 	SecSize = 512,
 	PperSec = SecSize/sizeof(uvlong),
-	SDBSec = 1,
 	BlkSize = 1024*1024,
 	PperBlk = BlkSize/sizeof(uvlong),
 	MaxStores = 128,
@@ -46,6 +72,7 @@
 	uvlong beta;
 	uvlong nmap;
 	uvlong next;
+	uvlong mapmap[1024];
 	uvlong mblk;
 	uvlong *pi;
 };
@@ -67,6 +94,11 @@
 	uvlong nmap;
 	uvlong next;
 	char name[128];
+	uvlong mapmap[1024];	/* Good for 128TB */
+};
+
+enum {
+	SDBSec = (sizeof(SDB) + SecSize - 1) / SecSize,
 };
 
 static Store stores[MaxStores];
@@ -99,23 +131,37 @@
 	sblk->beta = s->beta;
 	sblk->nmap = s->nmap;
 	sblk->next = s->next;
+	memmove(sblk->mapmap, s->mapmap, nelem(s->mapmap) * sizeof(uvlong));
 	strncpy(sblk->name, s->name, 127);
 	s->st->dchan->dev->write(s->st->dchan,sblk, SDBSec * SecSize, s->alpha * BlkSize);
 	poperror();
 	free(sblk);
 }
 
-static void
-updmap(Chan *c, uvlong *map, uvlong blkno, uvlong mblk, uvlong beta)
+static uvlong
+updmap(Snap *s, uvlong blkno, uvlong mblk)
 {
 	uvlong *msec;
 	uvlong idx, sec;
 
 	idx = blkno % PperBlk;
-	map[idx] = beta;
-	sec = idx / PperSec;
-	msec = map + sec * PperSec;
-	c->dev->write(c,msec, SecSize, mblk * BlkSize + sec * SecSize);
+	if(mblk < s->alpha) {
+		mblk = s->beta;
+		s->mblk = mblk;
+		++s->beta;
+		s->mapmap[idx/PperBlk] = mblk;
+		s->pi[idx] = s->beta;
+		s->st->dchan->dev->write(s->st->dchan,
+			s->pi, BlkSize, mblk * BlkSize);
+	}
+	else {
+		s->pi[idx] = s->beta;
+		sec = idx / PperSec;
+		msec = s->pi + sec * PperSec;
+		s->st->dchan->dev->write(s->st->dchan,
+			msec, SecSize, mblk * BlkSize + sec * SecSize);
+	}
+	return mblk;
 }
 
 static uvlong
@@ -125,7 +171,7 @@
 	uvlong blkno, mblk, p;
 
 	blkno = l / BlkSize;
-	mblk = blkno / PperBlk + s->alpha + 1;
+	mblk = s->mapmap[blkno/PperBlk];
 	qlock(s);
 	if(waserror()) {
 		qunlock(s);
@@ -133,6 +179,20 @@
 	}
 	if(s->pi == nil)
 		s->pi = malloc(BlkSize);
+	if(mblk == 0) {
+		if(doalloc) {
+			mblk = s->beta;
+			s->mapmap[blkno/PperBlk] = mblk;
+			s->mblk = mblk;
+			++s->beta;
+			memset(s->pi, 0, BlkSize);
+		}
+		else {
+			poperror();
+			qunlock(s);
+			return 0;
+		}
+	}
 	if(mblk != s->mblk) {
 		s->st->dchan->dev->read(s->st->dchan,s->pi, BlkSize, mblk * BlkSize);
 		s->mblk = mblk;
@@ -144,7 +204,7 @@
 			qunlock(s);
 			return 0;
 		}
-		updmap(s->st->dchan, s->pi, blkno, mblk, s->beta);
+		updmap(s, blkno, mblk);
 		p = s->beta;
 		++s->beta;
 		++s->inuse;
@@ -154,7 +214,7 @@
 		/*
 		 * Do COW
 		 */
-		updmap(s->st->dchan, s->pi, blkno, mblk, s->beta);
+		updmap(s, blkno, mblk);
 		tblk = malloc(BlkSize);
 		if(waserror()) {
 			free(tblk);
@@ -351,6 +411,8 @@
 			st->snaps[st->nsnaps].beta = s->beta;
 			st->snaps[st->nsnaps].nmap = s->nmap;
 			st->snaps[st->nsnaps].next = s->next;
+			memmove(st->snaps[st->nsnaps].mapmap, s->mapmap,
+				nelem(s->mapmap) * sizeof(uvlong));
 			st->snaps[st->nsnaps].qid.path = st->nsnaps + QTdata;
 			st->snaps[st->nsnaps].qid.vers = 0;
 			st->snaps[st->nsnaps].qid.type = QTFILE;
@@ -371,7 +433,7 @@
 	Chan *c;
 	Store *st;
 	uchar *buf;
-	int i, j;
+	int i;
 
 	if(stores[0].nsnaps != 0)
 		error("multiple binds not yet supported");
@@ -401,13 +463,9 @@
 	st->snaps[0].size = d.length;
 	st->snaps[0].alpha = 0;
 	st->snaps[0].nmap = ((st->snaps[0].size + BlkSize - 1)/ BlkSize + PperBlk - 1) / PperBlk;
-	st->snaps[0].beta = st->snaps[0].nmap + 1;
+	st->snaps[0].beta = 1;
 	st->snaps[0].inuse = st->snaps[0].beta;
-	/* Write the l2p map */
-	st->snaps[0].pi = malloc(BlkSize);
-	memset(st->snaps[0].pi, 0, BlkSize);
-	for(j = 0; j < st->snaps[0].nmap; ++j)
-		c->dev->write(c,st->snaps[0].pi, BlkSize, (j+1) * BlkSize);
+	memset(st->snaps[0].mapmap, 0, nelem(st->snaps[0].mapmap) * sizeof(uvlong));
 	writesdb(&st->snaps[0]);
 	poperror();
 	free(buf);
@@ -446,6 +504,8 @@
 	st->snaps[i].inuse = st->snaps[j].inuse;
 	st->snaps[i].alpha = st->snaps[j].alpha;
 	st->snaps[i].beta = st->snaps[j].beta;
+	memmove(st->snaps[i].mapmap, st->snaps[j].mapmap,
+		nelem(st->snaps[i].mapmap) * sizeof(uvlong));
 	st->snaps[i].mblk = 0;
 	st->snaps[i].flags = SFvalid;
 	writesdb(&st->snaps[i]);
@@ -457,12 +517,10 @@
 static void
 snapsnap(Cmdbuf *cb)
 {
-	Chan *c;
 	Store *st;
-	int i, j, slot;
+	int i, slot;
 
 	st = &stores[0];
-	c = st->dchan;
 	for(i = 0; i < st->nsnaps && strcmp(st->snaps[i].name, cb->f[1]) != 0; ++i) ;
 	if(i >= st->nsnaps)
 		error(Enonexist);
@@ -489,23 +547,17 @@
 	st->snaps[slot].beta = st->snaps[i].beta;
 	st->snaps[slot].nmap = st->snaps[i].nmap;
 	st->snaps[slot].next = st->snaps[i].beta;
+	memmove(st->snaps[slot].mapmap, st->snaps[i].mapmap,
+		nelem(st->snaps[i].mapmap) * sizeof(uvlong));
 	st->snaps[slot].mblk = 0;
 	free(st->snaps[slot].pi);
 	st->snaps[slot].pi = nil;
 	/* adjust the original to point to the new space */
 	st->snaps[i].alpha = st->snaps[i].beta;
-	st->snaps[i].beta = st->snaps[i].alpha + st->snaps[i].nmap + 1;
+	st->snaps[i].beta = st->snaps[i].alpha + 1;
 	/* write the sdb blocks */
 	writesdb(&st->snaps[i]);
 	writesdb(&st->snaps[slot]);
-	/* copy the l2p map */
-	if(st->snaps[i].pi == nil)
-		st->snaps[i].pi = malloc(BlkSize);
-	for(j = 0; j < st->snaps[i].nmap; ++j) {
-		c->dev->read(c,st->snaps[i].pi, BlkSize, (st->snaps[slot].alpha + j + 1) * BlkSize);
-		st->snaps[i].mblk = st->snaps[i].alpha + j + 1;
-		c->dev->write(c,st->snaps[i].pi, BlkSize, st->snaps[i].mblk * BlkSize);
-	}
 	poperror();
 	qunlock(&stores[0].snaps[i]);
 	qunlock(&stores[0].snaps[slot]);
--- /sys/src/9/port/devsnap.c	Mon Feb 10 17:39:33 2014
+++ /sys/src/9/port/devsnap.c	Mon Feb 10 17:39:35 2014
@@ -1,3 +1,30 @@
+/*
+ * Copyright (c) 2013, Coraid, Inc.
+ * All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Coraid nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL CORAID BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
 #include	"u.h"
 #include	"../port/lib.h"
 #include	"mem.h"
@@ -14,7 +41,6 @@
 
 	SecSize = 512,
 	PperSec = SecSize/sizeof(uvlong),
-	SDBSec = 1,
 	BlkSize = 1024*1024,
 	PperBlk = BlkSize/sizeof(uvlong),
 	MaxStores = 128,
@@ -46,6 +72,7 @@
 	uvlong beta;
 	uvlong nmap;
 	uvlong next;
+	uvlong mapmap[1024];
 	uvlong mblk;
 	uvlong *pi;
 };
@@ -67,6 +94,11 @@
 	uvlong nmap;
 	uvlong next;
 	char name[128];
+	uvlong mapmap[1024];	/* Good for 128TB */
+};
+
+enum {
+	SDBSec = (sizeof(SDB) + SecSize - 1) / SecSize,
 };
 
 static Store stores[MaxStores];
@@ -99,23 +131,37 @@
 	sblk->beta = s->beta;
 	sblk->nmap = s->nmap;
 	sblk->next = s->next;
+	memmove(sblk->mapmap, s->mapmap, nelem(s->mapmap) * sizeof(uvlong));
 	strncpy(sblk->name, s->name, 127);
 	devtab[s->st->dchan->type]->write(s->st->dchan, sblk, SDBSec * SecSize, s->alpha * BlkSize);
 	poperror();
 	free(sblk);
 }
 
-static void
-updmap(Chan *c, uvlong *map, uvlong blkno, uvlong mblk, uvlong beta)
+static uvlong
+updmap(Snap *s, uvlong blkno, uvlong mblk)
 {
 	uvlong *msec;
 	uvlong idx, sec;
 
 	idx = blkno % PperBlk;
-	map[idx] = beta;
-	sec = idx / PperSec;
-	msec = map + sec * PperSec;
-	devtab[c->type]->write(c, msec, SecSize, mblk * BlkSize + sec * SecSize);
+	if(mblk < s->alpha) {
+		mblk = s->beta;
+		s->mblk = mblk;
+		++s->beta;
+		s->mapmap[idx/PperBlk] = mblk;
+		s->pi[idx] = s->beta;
+		devtab[s->st->dchan->type]->write(s->st->dchan,
+			s->pi, BlkSize, mblk * BlkSize);
+	}
+	else {
+		s->pi[idx] = s->beta;
+		sec = idx / PperSec;
+		msec = s->pi + sec * PperSec;
+		devtab[s->st->dchan->type]->write(s->st->dchan,
+			msec, SecSize, mblk * BlkSize + sec * SecSize);
+	}
+	return mblk;
 }
 
 static uvlong
@@ -125,7 +171,7 @@
 	uvlong blkno, mblk, p;
 
 	blkno = l / BlkSize;
-	mblk = blkno / PperBlk + s->alpha + 1;
+	mblk = s->mapmap[blkno/PperBlk];
 	qlock(s);
 	if(waserror()) {
 		qunlock(s);
@@ -133,6 +179,20 @@
 	}
 	if(s->pi == nil)
 		s->pi = malloc(BlkSize);
+	if(mblk == 0) {
+		if(doalloc) {
+			mblk = s->beta;
+			s->mapmap[blkno/PperBlk] = mblk;
+			s->mblk = mblk;
+			++s->beta;
+			memset(s->pi, 0, BlkSize);
+		}
+		else {
+			poperror();
+			qunlock(s);
+			return 0;
+		}
+	}
 	if(mblk != s->mblk) {
 		devtab[s->st->dchan->type]->read(s->st->dchan, s->pi, BlkSize, mblk * BlkSize);
 		s->mblk = mblk;
@@ -144,7 +204,7 @@
 			qunlock(s);
 			return 0;
 		}
-		updmap(s->st->dchan, s->pi, blkno, mblk, s->beta);
+		updmap(s, blkno, mblk);
 		p = s->beta;
 		++s->beta;
 		++s->inuse;
@@ -154,7 +214,7 @@
 		/*
 		 * Do COW
 		 */
-		updmap(s->st->dchan, s->pi, blkno, mblk, s->beta);
+		updmap(s, blkno, mblk);
 		tblk = malloc(BlkSize);
 		if(waserror()) {
 			free(tblk);
@@ -351,6 +411,8 @@
 			st->snaps[st->nsnaps].beta = s->beta;
 			st->snaps[st->nsnaps].nmap = s->nmap;
 			st->snaps[st->nsnaps].next = s->next;
+			memmove(st->snaps[st->nsnaps].mapmap, s->mapmap,
+				nelem(s->mapmap) * sizeof(uvlong));
 			st->snaps[st->nsnaps].qid.path = st->nsnaps + QTdata;
 			st->snaps[st->nsnaps].qid.vers = 0;
 			st->snaps[st->nsnaps].qid.type = QTFILE;
@@ -371,7 +433,7 @@
 	Chan *c;
 	Store *st;
 	uchar *buf;
-	int i, j;
+	int i;
 
 	if(stores[0].nsnaps != 0)
 		error("multiple binds not yet supported");
@@ -401,13 +463,9 @@
 	st->snaps[0].size = d.length;
 	st->snaps[0].alpha = 0;
 	st->snaps[0].nmap = ((st->snaps[0].size + BlkSize - 1)/ BlkSize + PperBlk - 1) / PperBlk;
-	st->snaps[0].beta = st->snaps[0].nmap + 1;
+	st->snaps[0].beta = 1;
 	st->snaps[0].inuse = st->snaps[0].beta;
-	/* Write the l2p map */
-	st->snaps[0].pi = malloc(BlkSize);
-	memset(st->snaps[0].pi, 0, BlkSize);
-	for(j = 0; j < st->snaps[0].nmap; ++j)
-		devtab[c->type]->write(c, st->snaps[0].pi, BlkSize, (j+1) * BlkSize);
+	memset(st->snaps[0].mapmap, 0, nelem(st->snaps[0].mapmap) * sizeof(uvlong));
 	writesdb(&st->snaps[0]);
 	poperror();
 	free(buf);
@@ -446,6 +504,8 @@
 	st->snaps[i].inuse = st->snaps[j].inuse;
 	st->snaps[i].alpha = st->snaps[j].alpha;
 	st->snaps[i].beta = st->snaps[j].beta;
+	memmove(st->snaps[i].mapmap, st->snaps[j].mapmap,
+		nelem(st->snaps[i].mapmap) * sizeof(uvlong));
 	st->snaps[i].mblk = 0;
 	st->snaps[i].flags = SFvalid;
 	writesdb(&st->snaps[i]);
@@ -457,12 +517,10 @@
 static void
 snapsnap(Cmdbuf *cb)
 {
-	Chan *c;
 	Store *st;
-	int i, j, slot;
+	int i, slot;
 
 	st = &stores[0];
-	c = st->dchan;
 	for(i = 0; i < st->nsnaps && strcmp(st->snaps[i].name, cb->f[1]) != 0; ++i) ;
 	if(i >= st->nsnaps)
 		error(Enonexist);
@@ -489,23 +547,17 @@
 	st->snaps[slot].beta = st->snaps[i].beta;
 	st->snaps[slot].nmap = st->snaps[i].nmap;
 	st->snaps[slot].next = st->snaps[i].beta;
+	memmove(st->snaps[slot].mapmap, st->snaps[i].mapmap,
+		nelem(st->snaps[i].mapmap) * sizeof(uvlong));
 	st->snaps[slot].mblk = 0;
 	free(st->snaps[slot].pi);
 	st->snaps[slot].pi = nil;
 	/* adjust the original to point to the new space */
 	st->snaps[i].alpha = st->snaps[i].beta;
-	st->snaps[i].beta = st->snaps[i].alpha + st->snaps[i].nmap + 1;
+	st->snaps[i].beta = st->snaps[i].alpha + 1;
 	/* write the sdb blocks */
 	writesdb(&st->snaps[i]);
 	writesdb(&st->snaps[slot]);
-	/* copy the l2p map */
-	if(st->snaps[i].pi == nil)
-		st->snaps[i].pi = malloc(BlkSize);
-	for(j = 0; j < st->snaps[i].nmap; ++j) {
-		devtab[c->type]->read(c, st->snaps[i].pi, BlkSize, (st->snaps[slot].alpha + j + 1) * BlkSize);
-		st->snaps[i].mblk = st->snaps[i].alpha + j + 1;
-		devtab[c->type]->write(c, st->snaps[i].pi, BlkSize, st->snaps[i].mblk * BlkSize);
-	}
 	poperror();
 	qunlock(&stores[0].snaps[i]);
 	qunlock(&stores[0].snaps[slot]);