# HG changeset patch
# User Noah Evans <npe@plan9.bell-labs.com>
# Date 1307437707 0
# Node ID ddaa629716d50dce42e334f51fbcbbf92c70c3f4
# Parent  35d39df170775f9e0d9ac82aef1c62025a7948c0
daily push: Mon Jun 6 11:08:21 CET 2011

diff -r 35d39df17077 -r ddaa629716d5 src/9kron/include/libc.h
--- a/src/9kron/include/libc.h	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/include/libc.h	Tue Jun 07 09:08:27 2011 +0000
@@ -743,7 +743,7 @@
 	PmcEnable = 4,
 };
 
-extern int confpmc(int, int, char *);
+extern int confpmc(int, int, int, char *);
 extern uvlong rdpmc(int);
 
 #pragma	varargck	argpos	werrstr	1
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/k8/acore.c
--- a/src/9kron/k8/acore.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/k8/acore.c	Tue Jun 07 09:08:27 2011 +0000
@@ -9,6 +9,7 @@
 #include "amd64.h"
 #include "ureg.h"
 #include "io.h"
+#include "../port/pmc.h"
 
 /*
  * NIX code run at the AC.
@@ -135,6 +136,9 @@
 	ACVctl *v;
 
 	n = nil;
+
+	pmcupdate(m);
+
 	if(u->type < nelem(acvctl)){
 		v = acvctl[u->type];
 		if(v != nil){
@@ -170,6 +174,7 @@
 	memmove(m->proc->dbgreg, u, sizeof *u);
 	m->icc->note = n;
 	fpuprocsave(m->proc);
+	pmcupdate(m);
 	mfence();
 	m->icc->fn = nil;
 	ready(m->proc);
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/k8/dat.h
--- a/src/9kron/k8/dat.h	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/k8/dat.h	Tue Jun 07 09:08:27 2011 +0000
@@ -23,6 +23,9 @@
 typedef struct Ureg Ureg;
 typedef struct Vctl Vctl;
 typedef struct ACVctl ACVctl;
+typedef struct PmcCtr PmcCtr;
+typedef struct PmcCtl PmcCtl;
+
 
 #pragma incomplete Ureg
 
@@ -183,6 +186,35 @@
 };
 
 /*
+ * hw perf counters
+ */
+struct PmcCtl {
+	u32int coreno;
+	int enab;
+	int user;
+	int os;
+	int nodesc;
+	char descstr[KNAMELEN];
+	int reset;
+};
+
+struct PmcCtr{
+	int stale;
+	Rendez r;
+	u64int ctr;
+	int ctrset;
+	PmcCtl;
+	int ctlset;
+};
+
+enum {
+	PmcMaxCtrs = 4,
+	PmcIgn = 0,
+	PmcGet = 1,
+	PmcSet = 2,
+};
+
+/*
  * Per processor information.
  *
  * The offsets of the first few elements may be known
@@ -238,6 +270,9 @@
 	MFPU;
 	MCPU;
 
+	Lock pmclock;
+	PmcCtr pmc[PmcMaxCtrs];
+
 	NIX;
 };
 
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/k8/pmcio.c
--- a/src/9kron/k8/pmcio.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/k8/pmcio.c	Tue Jun 07 09:08:27 2011 +0000
@@ -12,8 +12,6 @@
 #include	"../port/pmc.h"
 
 
-static QLock pmclck;
-
 /* non portable, for intel will be CPUID.0AH.EDX */
 
 enum {
@@ -23,6 +21,9 @@
 int
 pmcnregs(void)
 {
+	/* could run CPUID to see if there are registers,
+	 * PmcMaxCtrs
+ 	*/
 	return PeNreg;
 }
 
@@ -48,7 +49,7 @@
 	return cr4&Pce;
 }
 
-PmcCtrId pmcids[] = {
+PmcCtlCtrId pmcids[] = {
 	{"locked instr", "0x024 0x0"},
 	{"SMI intr", "0x02b 0"},
 	{"data access", "0x040 0x0"},
@@ -67,9 +68,9 @@
 };
 
 int
-pmctrans(Pmc *p)
+pmctrans(PmcCtl *p)
 {
-	PmcCtrId *pi;
+	PmcCtlCtrId *pi;
 
 	for (pi = &pmcids[0]; pi->portdesc[0] != '\0'; pi++){
 		if ( strncmp(p->descstr, pi->portdesc, strlen(pi->portdesc)) == 0){
@@ -80,13 +81,12 @@
 	return 1;
 }
 
-int
-pmcgetctl(Pmc *p, u32int regno)
+static int
+getctl(PmcCtl *p, u32int regno)
 {
 	u64int r, e, u;
 
 	r = rdmsr(regno + PerfEvtbase);
-	p->regno = regno;
 	p->enab = (r&PeCtEna) != 0;
 	p->user = (r&PeUsr) != 0;
 	p->os = (r&PeOS) != 0;
@@ -94,91 +94,117 @@
 	u = GetUMsk(r);
 	//TODO inverse translation
 	snprint(p->descstr, KNAMELEN, "%#ullx %#ullx", e, u);
+	p->nodesc = 0;
+	return 0;
+}
+
+int
+pmcanyenab(void)
+{
+	int i;
+	PmcCtl p;
+
+	for (i = 0; i < pmcnregs(); i++) {
+		if (getctl(&p, i) < 0)
+			return -1;
+		if (p.enab)
+			return 1;
+	}
+
 	return 0;
 }
 
 extern int pmcdebug;
 
-int
-pmcsetctl(Pmc *p)
+static int
+setctl(PmcCtl *p, int regno)
 {
 	u64int v, e, u;
 	char *toks[2];
+	char str[KNAMELEN];
 
-	if (p->regno >= pmcnregs())
+	if (regno >= pmcnregs())
 		error("invalid reg");
 
-	qlock(&pmclck);
-	v = rdmsr(p->regno + PerfEvtbase);
+	v = rdmsr(regno + PerfEvtbase);
 	v &= PeEvMskH|PeEvMskL|PeCtEna|PeOS|PeUsr|PeUnMsk;
-	if (p->enab != PmcNullval)
+	if (p->enab != PmcCtlNullval){
 		if (p->enab)
 			v |= PeCtEna;
 		else
 			v &= ~PeCtEna;
-	if (p->user != PmcNullval)
+		p->enab = PmcCtlNullval;
+	}
+	if (p->user != PmcCtlNullval){
 		if (p->user)
 			v |= PeUsr;
 		else
 			v &= ~PeUsr;
-	if (p->os != PmcNullval)
+		p->user = PmcCtlNullval;
+	}
+	if (p->os != PmcCtlNullval){
 		if (p->os)
 			v |= PeOS;
 		else
 			v &= ~PeOS;
-	if (pmctrans(p) < 0){
-		qunlock(&pmclck);
+		p->os = PmcCtlNullval;
+	}
+	if (pmctrans(p) < 0)
 		return -1;
-	}
 
-	if (!p->nodesc) {
-		if (tokenize(p->descstr, toks, 2) != 2)
+	if (p->nodesc == 0) {
+		memmove(str, p->descstr, KNAMELEN);
+		if (tokenize(str, toks, 2) != 2)
 			return -1;
 		e = atoi(toks[0]);
 		u = atoi(toks[1]);
 		v &= ~(PeEvMskL|PeEvMskH|PeUnMsk);
 		v |= SetEvMsk(v, e);
 		v |= SetUMsk(v, u);
+		p->nodesc = 1;
 	}
-	if (p->reset == 1){
+	if (p->reset != PmcCtlNullval && p->reset) {
 		v = 0;
-		wrmsr(p->regno+ PerfCtrbase, 0);
+		wrmsr(regno+ PerfCtrbase, 0);
+		p->reset = PmcCtlNullval;
 	}
-	wrmsr(p->regno+ PerfEvtbase, v);
+	wrmsr(regno+ PerfEvtbase, v);
 	pmcuserenab(pmcanyenab());
 	if (pmcdebug) {
-		v = rdmsr(p->regno+ PerfEvtbase);
-		print("conf pmc[%#ux]: %#llux\n", p->regno, v);
+		v = rdmsr(regno+ PerfEvtbase);
+		print("conf pmc[%#ux]: %#llux\n", regno, v);
 	}
-	qunlock(&pmclck);
 	return 0;
 }
 
 int
-pmcctlstr(char *str, int nstr, Pmc *p)
+pmcctlstr(char *str, int nstr, PmcCtl *p)
 {
 	int ns;
 
 	ns = 0;
-	if (p->enab)
+	if (p->enab && p->enab != PmcCtlNullval)
 		ns += snprint(str + ns, nstr - ns, "enable\n");
 	else
 		ns += snprint(str + ns, nstr - ns, "disable\n");
 		
-	if (p->user)
+	if (p->user && p->user != PmcCtlNullval)
 		ns += snprint(str + ns, nstr - ns, "user\n");
-	if (p->os)
+	if (p->os && p->user != PmcCtlNullval)
 		ns += snprint(str + ns, nstr - ns, "os\n");
 	
 	//TODO, inverse pmctrans?
-	ns += snprint(str + ns, nstr - ns, "%s\n", p->descstr);
+	if(!p->nodesc)
+		ns += snprint(str + ns, nstr - ns, "%s\n", p->descstr);
+	else 
+		ns += snprint(str + ns, nstr - ns, "no desc\n");
 	return ns;
 }
 
 int
 pmcdescstr(char *str, int nstr)
 {
-	PmcCtrId *pi;
+	PmcCtlCtrId *pi;
 	int ns;
 
 	ns = 0;
@@ -188,25 +214,159 @@
 	return ns;
 }
 
-u64int
-pmcgetctr(u32int regno)
+static u64int
+getctr(u32int regno)
 {
 	return rdmsr(regno + PerfCtrbase);
 }
 
-int
-pmcsetctr(u64int v, u32int regno)
+static int
+setctr(u64int v, u32int regno)
 {
 	wrmsr(regno + PerfCtrbase, v);
 	return 0;
 }
 
+int
+notstale(void *x)
+{
+	PmcCtr *p;
+	p = (PmcCtr *)x;
+	return !p->stale;
+}
 
+u64int
+pmcgetctr(u32int coreno, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
 
+	if(coreno == m->machno)
+		return getctr(regno);
 
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+	ilock(&m->pmclock);
+	p->ctrset |= PmcGet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	return p->ctr;
+}
 
+int
+pmcsetctr(u32int coreno, u64int v, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
 
+	if(coreno == m->machno)
+		return setctr(v, regno);
 
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+	ilock(&m->pmclock);
+	p->ctr = v;
+	p->ctrset |= PmcSet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	return 0;
+}
 
+static void
+ctl2ctl(PmcCtl *dctl, PmcCtl *sctl)
+{
+	if(sctl->enab != PmcCtlNullval)
+		dctl->enab = sctl->enab;
+	if(sctl->user != PmcCtlNullval)
+		dctl->user = sctl->user;
+	if(sctl->os != PmcCtlNullval)
+		dctl->os = sctl->os;
+	if(sctl->nodesc == 0) {
+		memmove(dctl->descstr, sctl->descstr, KNAMELEN);
+		dctl->nodesc = 0;
+	}
+}
 
+int
+pmcsetctl(u32int coreno, PmcCtl *pctl, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
 
+	if(coreno == m->machno)
+		return setctl(pctl, regno);
+
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+	ilock(&m->pmclock);
+	ctl2ctl(&p->PmcCtl, pctl);
+	p->ctlset |= PmcSet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	return 0;
+}
+
+int
+pmcgetctl(u32int coreno, PmcCtl *pctl, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
+
+	if(coreno == m->machno)
+		return getctl(pctl, regno);
+
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+
+	ilock(&m->pmclock);
+	p->ctlset |= PmcGet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	ilock(&m->pmclock);
+	memmove(pctl, &p->PmcCtl, sizeof(PmcCtl));
+	iunlock(&m->pmclock);
+	return 0;
+}
+
+void
+pmcupdate(Mach *m)
+{
+	PmcCtr *p;
+	int i, maxct, wk;
+
+	maxct = pmcnregs();
+	for (i = 0; i < maxct; i++) {
+		p = &m->pmc[i];
+		ilock(&m->pmclock);
+		if(p->ctrset & PmcSet)
+			setctr(p->ctr, i);
+		if(p->ctlset & PmcSet)
+			setctl(p, i);
+		p->ctr = getctr(i);
+		getctl(p, i);
+		p->ctrset = PmcIgn;
+		p->ctlset = PmcIgn;
+		wk = p->stale;
+		p->stale = 0;
+		iunlock(&m->pmclock);
+		if(wk)
+			wakeup(&p->r);
+	}
+}
+
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/k8/trap.c
--- a/src/9kron/k8/trap.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/k8/trap.c	Tue Jun 07 09:08:27 2011 +0000
@@ -9,6 +9,7 @@
 #include	"ureg.h"
 
 #include	"io.h"
+#include	"../port/pmc.h"
 
 extern int notify(Ureg*);
 
@@ -270,7 +271,7 @@
 		mp = m;
 	tos->core = mp->machno;	
 	tos->nixtype = mp->nixtype;
-
+	pmcupdate(m);	
 	/*
 	 * The process may change its core.
 	 * Be sure it has the right cyclefreq.
@@ -320,6 +321,8 @@
 	if(up != nil)
 		up->ntrap++;	/* stats only, races ok */
 
+	pmcupdate(m);
+
 	vno = ureg->type;
 	if(ctl = vctl[vno]){
 		if(ctl->isintr){
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/libc/port/confpmc.c
--- a/src/9kron/libc/port/confpmc.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/libc/port/confpmc.c	Tue Jun 07 09:08:27 2011 +0000
@@ -2,12 +2,14 @@
 #include <libc.h>
 
 int
-confpmc(int index, int mode, char *desc)
+confpmc(int core, int index, int mode, char *desc)
 {
 	int fd;
 	char name[32];
 	
-	snprint(name, sizeof(name), "/dev/ctr%2.2udctl", index);
+	if(core < 0)
+		core = getcore(nil);
+	snprint(name, sizeof(name), "/dev/core%4.4d/ctr%2.2dctl", core, index);
 
 	fd = open(name, OWRITE);
 	if (fd < 0)
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/libc/port/rdpmc.c
--- a/src/9kron/libc/port/rdpmc.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/libc/port/rdpmc.c	Tue Jun 07 09:08:27 2011 +0000
@@ -2,13 +2,16 @@
 #include <libc.h>
 
 
+
 uvlong
 rdpmc(int index)
 {
-	int fd, n;
+	int fd, n, core;
 	char name[16+2+1];	/* 0x0000000000000000\0 */
-	
-	snprint(name, sizeof(name), "/dev/ctr%2.2ud", index);
+
+	core = getcore(nil);
+
+	snprint(name, sizeof(name), "/dev/core%4.4d/ctr%2.2ud", core, index);
 
 	fd = open(name, OREAD);
 	if (fd < 0)
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/man/2/pmc
--- a/src/9kron/man/2/pmc	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/man/2/pmc	Tue Jun 07 09:08:27 2011 +0000
@@ -7,7 +7,7 @@
 .B #include <libc.h>
 .PP
 .B
-int confpmc(int index, int mode, char *desc)
+int confpmc(int core, int index, int mode, char *desc)
 .PP
 .B
 uvlong rdpmc(int index)
@@ -15,6 +15,8 @@
 .I Confpmc
 configures the counter
 .I index
+on core
+.I core
 setting the configuration according to the configuration string,
 .I desc
 whic is architecture dependant and the
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/man/3/pmc
--- a/src/9kron/man/3/pmc	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/man/3/pmc	Tue Jun 07 09:08:27 2011 +0000
@@ -5,20 +5,29 @@
 .nf
 .B bind '#ε' /dev
 
-.BI #ε/ctrdesc
-.BI #ε/ctr00
-.BI #ε/ctr00ctl
-.BI #ε/ctr01
-.BI #ε/ctr01ctl
+.BI #ε/core0000/ctrdesc
+.BI #ε/core0000/ctr00
+.BI #ε/core0000/ctr00ctl
+.BI #ε/core0000/ctr01
+.BI #ε/core0000/ctr01ctl
+ ...
+.BI #ε/core0001/ctr00
+.BI #ε/core0001/ctr00ctl
+.BI #ε/core0001/ctr01
+.BI #ε/core0001/ctr01ctl
  ...
 .fi
 .SH DESCRIPTION
 .PP
 The
 .I pmc
-device serves a one-level directory, giving
-access to the hardware counters on the processor
-the process reading or writing them is reading from.
+device serves a two-level directory, giving
+access to the hardware counters on the
+different cores.
+There is a directory per core,
+.B coreNNNN,
+containing files
+pertaining to that core.
 .PP
 The
 .B ctrNN
@@ -36,7 +45,7 @@
 .B os
 The counter only runs when the processor is in the kernel.
 .TP
-.BI set configuration
+.BI "set configuration
 Sets the counter configuration. The detail of what the configuration
 means is left to the specific implementation of the counter. Reading the
 .B ctrdesc
@@ -48,6 +57,11 @@
 any other way to access the counter from user space, like special
 instructions.
 .PP
+Any change to or from the counters or the configurations is seen
+as soon as possible. In the worst case, it is guaranteed that the values
+read or set are at least as fresh as the last time a process went in or out
+of the kernel on that core.
+.PP
 Configure the counter 0 to count L2 misses in the kernel.
 .EX
 	% bind '#ε' /dev
@@ -66,11 +80,11 @@
 		L2 ITLB miss
 		DRAM access
 		L3 miss
-	% echo reset > /dev/ctr00ctl
-	% echo kernel > /dev/ctr00ctl
-	% echo set L2 miss > /dev/ctr00ctl
-	% echo enable > /dev/ctr00ctl
-	% cat /dev/ctr00
+	% echo reset > /dev/core0000/ctr00ctl
+	% echo kernel > /dev/core0000/ctr00ctl
+	% echo set L2 miss > /dev/core0000/ctr00ctl
+	% echo enable > /dev/core0000/ctr00ctl
+	% cat /dev/core0000/ctr00
 		0x00000000000003e5
 	%
 .EE
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/port/devpmc.c
--- a/src/9kron/port/devpmc.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/port/devpmc.c	Tue Jun 07 09:08:27 2011 +0000
@@ -13,11 +13,14 @@
 
 enum{
 	Qdir		= 0,
+	Qgctl,
+	Qcore,
+
+	Qctr,
 	Qdata,
 	Qctl,
-	Qgctl,
 
-	PmcRdStr = 4*1024,
+	PmcCtlRdStr = 4*1024,
 };
 
 #define PMCTYPE(x)	(((unsigned)x)&0xffful)
@@ -26,22 +29,21 @@
 
 Dirtab *pmctab;
 static int npmctab;
+Dirtab *toptab;
+static int ntoptab;
 int pmcdebug;
 
 static void
-pmcinit(void)
+topdirinit(int ncores)
 {
-	int nr, i;
+	int i;
 	Dirtab *d;
 
-	nr = pmcnregs();
-		
-	npmctab = 2 + 2*nr;
-	pmctab = malloc(npmctab * sizeof(Dirtab));
-	if (pmctab == nil)
+	ntoptab = 1 + ncores;
+	toptab = malloc(ntoptab * sizeof(Dirtab));
+	if (toptab == nil)
 		return;
-	
-	d = pmctab;
+	d = toptab;
 	strncpy(d->name, ".", KNAMELEN);
 	mkqid(&d->qid, Qdir, 0, QTDIR);
 	d->perm = DMDIR|0555;
@@ -49,20 +51,78 @@
 	strncpy(d->name, "ctrdesc", KNAMELEN);
 	mkqid(&d->qid, Qgctl, 0, 0);
 	d->perm = 0444;
-	for (i = 2; i < nr + 2; i++) {
+	for (i = 2; i < ncores + 2; i++) {
+		d = &toptab[i];
+		snprint(d->name, KNAMELEN, "core%4.4ud", i - 2);
+		mkqid(&d->qid, PMCQID(i - 2, Qcore), 0, QTDIR);
+		d->perm = DMDIR|0555;
+	}
+
+}
+
+static void
+ctrdirinit(void)
+{
+	int nr, i;
+	Dirtab *d;
+
+	nr = pmcnregs();
+		
+	npmctab = 1 + 2*nr;
+	pmctab = malloc(npmctab * sizeof(Dirtab));
+	if (pmctab == nil){
+		free(toptab);
+		toptab = nil;
+		return;
+	}
+
+	d = pmctab;
+	strncpy(d->name, ".", KNAMELEN);
+	mkqid(&d->qid, Qctr, 0, QTDIR);
+	d->perm = DMDIR|0555;
+	for (i = 1; i < nr + 1; i++) {
 		d = &pmctab[i];
-		snprint(d->name, KNAMELEN, "ctr%2.2ud", i - 2);
-		mkqid(&d->qid, PMCQID(i - 2, Qdata), 0, 0);
+		snprint(d->name, KNAMELEN, "ctr%2.2ud", i - 1);
+		mkqid(&d->qid, PMCQID(i - 1, Qdata), 0, 0);
 		d->perm = 0600;
 
 		d = &pmctab[nr + i];
-		snprint(d->name, KNAMELEN, "ctr%2.2udctl", i - 2);
-		mkqid(&d->qid, PMCQID(i - 2, Qctl), 0, 0);
+		snprint(d->name, KNAMELEN, "ctr%2.2udctl", i - 1);
+		mkqid(&d->qid, PMCQID(i - 1, Qctl), 0, 0);
 		d->perm = 0600;
 	}	
 	
 }
 
+static void
+pmcnull(PmcCtl *p)
+{
+	memset(p, 0xff, sizeof(PmcCtl));
+	p->enab = PmcCtlNullval;
+	p->user = PmcCtlNullval;
+	p->os = PmcCtlNullval;
+	p->reset = PmcCtlNullval;
+	p->nodesc = 1;
+}
+
+static void
+pmcinit(void)
+{
+	int i, j, ncores, nr;
+	Mach *mp;
+
+	ncores = 0;
+	nr = pmcnregs();
+	for(i = 0; i < MACHMAX; i++) {
+		if((mp = sys->machptr[i]) != nil && mp->online != 0)
+			ncores++;
+		for(j = 0; j < nr; j++)
+			pmcnull(&mp->pmc[j]);
+	}
+	topdirinit(ncores);
+	ctrdirinit();
+}
+
 static Chan *
 pmcattach(char *spec)
 {
@@ -70,17 +130,63 @@
 		error(Enomem);
 	return devattach(L'ε', spec);
 }
+int
+pmcgen(Chan *c, char *name, Dirtab*, int, int s, Dir *dp)
+{
+	int t, i, n;
+	Dirtab *l, *d;
+
+	if(s == DEVDOTDOT){
+		devdir(c, (Qid){Qdir, 0, QTDIR}, "#ε", 0, eve, 0555, dp);
+		c->aux = nil;
+		return 1;
+	}
+	/* first, for directories, generate children */
+	switch((int)PMCTYPE(c->qid.path)){
+	case Qdir:
+		return devgen(c, name, toptab, ntoptab, s, dp);
+	case Qctr:
+		return devgen(c, name, pmctab, npmctab, s, dp);
+	case Qcore:
+		return devgen(c, name, pmctab, npmctab, s, dp);
+	default:
+		if(s != 0)
+			return -1;
+
+		t = PMCTYPE(c->qid.path);
+		if(t < Qctr){
+			i = t;
+			l = toptab;
+			n = ntoptab;
+		}else{
+			i = PMCID(t);
+			if (t == Qctl)
+				i += (npmctab - 1)/2;
+			l = pmctab;
+			n = npmctab;
+		}
+		if(i >=n)
+			return -1;
+
+		d = &l[i];
+		
+		devdir(c, d->qid, d->name, d->length, eve, d->perm, dp);
+		return 1;
+	}
+}
 
 static Walkqid*
 pmcwalk(Chan *c, Chan *nc, char **name, int nname)
 {
-	return devwalk(c, nc, name, nname, pmctab, npmctab, devgen);
+	if(PMCTYPE(c->qid.path) == Qcore)
+		c->aux = (void *)PMCID(c->qid.path);		/* core no */
+	return devwalk(c, nc, name, nname, nil, 0, pmcgen);
 }
 
 static long
 pmcstat(Chan *c, uchar *dp, long n)
 {
-	return devstat(c, dp, n, pmctab, npmctab, devgen);
+	return devstat(c, dp, n, nil, 0, pmcgen);
 }
 
 static Chan*
@@ -88,69 +194,56 @@
 {
 	if (!iseve())
 		error(Eperm);
-	return devopen(c, omode, pmctab, npmctab, devgen);
+	return devopen(c, omode, nil, 0, pmcgen);
 }
 
 static void
-pmcclose(Chan *c)
+pmcclose(Chan *)
 {
-	if (c->aux) {
-		free(c->aux);
-		c->aux = nil;
-	}
 }
 
 
-int
-pmcanyenab(void)
-{
-	int i;
-	Pmc p;
-
-	for (i = 0; i < pmcnregs(); i++) {
-		if (pmcgetctl(&p, i) < 0)
-			return -1;
-		if (p.enab)
-			return 1;
-	}
-
-	return 0;
-	
-}
 
 static long
 pmcread(Chan *c, void *a, long n, vlong offset)
 {
 	ulong type, id;
-	Pmc p;
+	PmcCtl p;
 	char *s;
 	u64int v;
+	u64int coreno;
 
-	if (c->qid.type == QTDIR)
-		return devdirread(c, a, n, pmctab, npmctab, devgen);
+	type = PMCTYPE(c->qid.path);
+	id = PMCID(c->qid.path);
 
-	s = malloc(PmcRdStr);
+	switch(type){
+	case Qcore:
+	case Qdir:
+	case Qctr:
+		return devdirread(c, a, n, nil, 0, pmcgen);
+	}
+
+	s = malloc(PmcCtlRdStr);
 	if(waserror()){
 		free(s);
 		nexterror();
 	}
-	type = PMCTYPE(c->qid.path);
-	id = PMCID(c->qid.path);
+
+	coreno = (u64int)c->aux;
+	p.coreno = coreno;
 	switch(type){
 	case Qdata:
-		if (up->ac != nil)
-			error("unimplemented yet");
-		v = pmcgetctr(id);
-		snprint(s, PmcRdStr, "%#ullx", v);
+		v = pmcgetctr(coreno, id);
+		snprint(s, PmcCtlRdStr, "%#ullx", v);
 		break;
 	case Qctl:
-		if (pmcgetctl(&p, id) < 0)
+		if (pmcgetctl(coreno, &p, id) < 0)
 			error("bad ctr");
-		if (pmcctlstr(s, PmcRdStr, &p) < 0)
+		if (pmcctlstr(s, PmcCtlRdStr, &p) < 0)
 			error("bad pmc");
 		break;
 	case Qgctl:
-		if (pmcdescstr(s, PmcRdStr) < 0)
+		if (pmcdescstr(s, PmcCtlRdStr) < 0)
 			error("bad pmc");
 		break;
 	default:
@@ -181,29 +274,49 @@
 	Debug, 		"debug",	0,
 };
 
-static void
-pmcnull(Pmc *p)
-{
-	memset(p, 0xff, sizeof(Pmc));
-	p->enab = PmcNullval;
-	p->user = PmcNullval;
-	p->os = PmcNullval;
-}
+typedef void (*APfunc)(void);
 
-typedef void (*APfunc)(void);
+typedef struct AcPmcArg AcPmcArg;
+struct AcPmcArg {
+	int regno;
+	int coreno;
+	PmcCtl;
+};
+
+typedef struct AcCtrArg AcCtrArg;
+struct AcCtrArg {
+	int regno;
+	int coreno;
+	u64int v;
+};
 
 void
 acpmcsetctl(void)
 {
-	Pmc p;
+	AcPmcArg p;
 	Mach *mp;
 
 	mp = up->ac;
-	memmove(&p, mp->icc->data, sizeof(Pmc));
+	memmove(&p, mp->icc->data, sizeof(AcPmcArg));
 	
-	mp->icc->rc = pmcsetctl(&p);
+	mp->icc->rc = pmcsetctl(p.coreno, &p, p.regno);
 	return;
 }
+
+void
+acpmcsetctr(void)
+{
+	AcCtrArg ctr;
+	Mach *mp;
+
+	mp = up->ac;
+	memmove(&ctr, mp->icc->data, sizeof(AcCtrArg));
+	
+	mp->icc->rc = pmcsetctr(ctr.coreno, ctr.v, ctr.regno);
+	return;
+}
+
+
 static long
 pmcwrite(Chan *c, void *a, long n, vlong)
 {
@@ -211,7 +324,10 @@
 	Cmdtab *ct;
 	ulong type;
 	char str[64];	/* 0x0000000000000000\0 */
-	Pmc p;
+	AcPmcArg p;
+	AcCtrArg ctr;
+	u64int coreno;
+	Mach *mp;
 
 	if (c->qid.type == QTDIR)
 		error(Eperm);
@@ -221,12 +337,25 @@
 		error(Ebadctl);
 
 	pmcnull(&p);
+	coreno = (u64int)c->aux;
+	p.coreno = coreno;
 	type = PMCTYPE(c->qid.path);
 	p.regno = PMCID(c->qid.path);
 	memmove(str, a, n);
 	str[n] = '\0';
+	mp = up->ac;
+
+	ctr.coreno = coreno;
+	ctr.regno = p.regno;
 	if (type == Qdata) {
-		pmcsetctr(atoi(str), p.regno);
+		/* I am a handler for a proc in the core, run an RPC*/
+		if (mp != nil && mp->machno == coreno) {
+			if (runac(mp, acpmcsetctr, 0, &ctr, sizeof(AcCtrArg)) < 0)
+				n = -1;
+		} else {
+		if (pmcsetctr(coreno, strtoull(str, 0, 0), p.regno) < 0)
+			n = -1;
+		}
 		return n;
 	}
 
@@ -234,6 +363,7 @@
 	/* TODO: should iterate through multiple lines */
 	if (strncmp(str, "set ", 4) == 0){
 		memmove(p.descstr, (char *)str + 4, n - 4);
+		p.descstr[n - 4] = '\0';
 		p.nodesc = 0;
 	} else {
 		cb = parsecmd(a, n);
@@ -268,11 +398,12 @@
 		free(cb);
 		poperror();
 	}
-	if (up->ac != nil) {
-		if (runac(up->ac, acpmcsetctl, 0, &p, sizeof(Pmc)) < 0)
+	/* I am a handler for a proc in the core, run an RPC*/
+	if (mp != nil && mp->machno == coreno) {
+		if (runac(mp, acpmcsetctl, 0, &p, sizeof(AcPmcArg)) < 0)
 			n = -1;
 	} else {
-		if (pmcsetctl(&p) < 0)
+		if (pmcsetctl(coreno, &p, p.regno) < 0)
 			n = -1;
 	}
 	return n;
diff -r 35d39df17077 -r ddaa629716d5 src/9kron/port/pmc.h
--- a/src/9kron/port/pmc.h	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron/port/pmc.h	Tue Jun 07 09:08:27 2011 +0000
@@ -1,34 +1,23 @@
 
 enum{
-	PmcNullval = 0xdead,
+	PmcCtlNullval = 0xdead,
 };
 
-typedef struct PmcCtrId PmcCtrId;
-struct PmcCtrId {
+typedef struct PmcCtlCtrId PmcCtlCtrId;
+
+
+struct PmcCtlCtrId {
 	char portdesc[KNAMELEN];
 	char archdesc[KNAMELEN];
 };
 
+int		pmcnregs(void);
+int		pmcsetctl(u32int coreno, PmcCtl *p, u32int regno);
+int		pmctrans(PmcCtl *p);
+int		pmcgetctl(u32int coreno, PmcCtl *p, u32int regno);
+int		pmcdescstr(char *str, int nstr);
+int		pmcctlstr(char *str, int nstr, PmcCtl *p);
+u64int	pmcgetctr(u32int coreno, u32int regno);
+int		pmcsetctr(u32int coreno, u64int v, u32int regno);
 
-typedef struct Pmc Pmc;
-struct Pmc{
-	u32int regno;
-	int enab;
-	int user;
-	int os;
-	int nodesc;
-	char descstr[KNAMELEN];
-	int reset;
-};
-
-int		pmcnregs(void);
-int		pmcsetctl(Pmc *p);
-int		pmctrans(Pmc *p);
-int		pmcgetctl(Pmc *p, u32int regno);
-int		pmcdescstr(char *str, int nstr);
-int		pmcctlstr(char *str, int nstr, Pmc *p);
-u64int	pmcgetctr(u32int regno);
-int		pmcsetctr(u64int v, u32int regno);
-
-int		pmcanyenab(void);
-
+void		pmcupdate(Mach *m);
diff -r 35d39df17077 -r ddaa629716d5 src/9kron2/k8/acore.c
--- a/src/9kron2/k8/acore.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron2/k8/acore.c	Tue Jun 07 09:08:27 2011 +0000
@@ -9,6 +9,7 @@
 #include "amd64.h"
 #include "ureg.h"
 #include "io.h"
+#include "../port/pmc.h"
 
 /*
  * NIX code run at the AC.
@@ -134,6 +135,9 @@
 	ACVctl *v;
 
 	n = nil;
+
+	pmcupdate(m);
+
 	if(u->type < nelem(acvctl)){
 		v = acvctl[u->type];
 		if(v != nil){
@@ -169,6 +173,7 @@
 	memmove(m->proc->dbgreg, u, sizeof *u);
 	m->icc->note = n;
 	fpuprocsave(m->proc);
+	pmcupdate(m);
 	mfence();
 	m->icc->fn = nil;
 	ready(m->proc);
diff -r 35d39df17077 -r ddaa629716d5 src/9kron2/k8/dat.h
--- a/src/9kron2/k8/dat.h	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron2/k8/dat.h	Tue Jun 07 09:08:27 2011 +0000
@@ -23,6 +23,8 @@
 typedef struct Ureg Ureg;
 typedef struct Vctl Vctl;
 typedef struct ACVctl ACVctl;
+typedef struct PmcCtr PmcCtr;
+typedef struct PmcCtl PmcCtl;
 
 #pragma incomplete Ureg
 
@@ -183,6 +185,35 @@
 };
 
 /*
+ * hw perf counters
+ */
+struct PmcCtl {
+	u32int coreno;
+	int enab;
+	int user;
+	int os;
+	int nodesc;
+	char descstr[KNAMELEN];
+	int reset;
+};
+
+struct PmcCtr{
+	int stale;
+	Rendez r;
+	u64int ctr;
+	int ctrset;
+	PmcCtl;
+	int ctlset;
+};
+
+enum {
+	PmcMaxCtrs = 4,
+	PmcIgn = 0,
+	PmcGet = 1,
+	PmcSet = 2,
+};
+
+/*
  * Per processor information.
  *
  * The offsets of the first few elements may be known
@@ -238,6 +269,9 @@
 	MFPU;
 	MCPU;
 
+	Lock pmclock;
+	PmcCtr pmc[PmcMaxCtrs];
+
 	NIX;
 };
 
diff -r 35d39df17077 -r ddaa629716d5 src/9kron2/k8/pmcio.c
--- a/src/9kron2/k8/pmcio.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron2/k8/pmcio.c	Tue Jun 07 09:08:27 2011 +0000
@@ -12,8 +12,6 @@
 #include	"../port/pmc.h"
 
 
-static QLock pmclck;
-
 /* non portable, for intel will be CPUID.0AH.EDX */
 
 enum {
@@ -23,6 +21,9 @@
 int
 pmcnregs(void)
 {
+	/* could run CPUID to see if there are registers,
+	 * PmcMaxCtrs
+ 	*/
 	return PeNreg;
 }
 
@@ -48,7 +49,7 @@
 	return cr4&Pce;
 }
 
-PmcCtrId pmcids[] = {
+PmcCtlCtrId pmcids[] = {
 	{"locked instr", "0x024 0x0"},
 	{"SMI intr", "0x02b 0"},
 	{"data access", "0x040 0x0"},
@@ -67,9 +68,9 @@
 };
 
 int
-pmctrans(Pmc *p)
+pmctrans(PmcCtl *p)
 {
-	PmcCtrId *pi;
+	PmcCtlCtrId *pi;
 
 	for (pi = &pmcids[0]; pi->portdesc[0] != '\0'; pi++){
 		if ( strncmp(p->descstr, pi->portdesc, strlen(pi->portdesc)) == 0){
@@ -80,13 +81,12 @@
 	return 1;
 }
 
-int
-pmcgetctl(Pmc *p, u32int regno)
+static int
+getctl(PmcCtl *p, u32int regno)
 {
 	u64int r, e, u;
 
 	r = rdmsr(regno + PerfEvtbase);
-	p->regno = regno;
 	p->enab = (r&PeCtEna) != 0;
 	p->user = (r&PeUsr) != 0;
 	p->os = (r&PeOS) != 0;
@@ -94,91 +94,117 @@
 	u = GetUMsk(r);
 	//TODO inverse translation
 	snprint(p->descstr, KNAMELEN, "%#ullx %#ullx", e, u);
+	p->nodesc = 0;
+	return 0;
+}
+
+int
+pmcanyenab(void)
+{
+	int i;
+	PmcCtl p;
+
+	for (i = 0; i < pmcnregs(); i++) {
+		if (getctl(&p, i) < 0)
+			return -1;
+		if (p.enab)
+			return 1;
+	}
+
 	return 0;
 }
 
 extern int pmcdebug;
 
-int
-pmcsetctl(Pmc *p)
+static int
+setctl(PmcCtl *p, int regno)
 {
 	u64int v, e, u;
 	char *toks[2];
+	char str[KNAMELEN];
 
-	if (p->regno >= pmcnregs())
+	if (regno >= pmcnregs())
 		error("invalid reg");
 
-	qlock(&pmclck);
-	v = rdmsr(p->regno + PerfEvtbase);
+	v = rdmsr(regno + PerfEvtbase);
 	v &= PeEvMskH|PeEvMskL|PeCtEna|PeOS|PeUsr|PeUnMsk;
-	if (p->enab != PmcNullval)
+	if (p->enab != PmcCtlNullval){
 		if (p->enab)
 			v |= PeCtEna;
 		else
 			v &= ~PeCtEna;
-	if (p->user != PmcNullval)
+		p->enab = PmcCtlNullval;
+	}
+	if (p->user != PmcCtlNullval){
 		if (p->user)
 			v |= PeUsr;
 		else
 			v &= ~PeUsr;
-	if (p->os != PmcNullval)
+		p->user = PmcCtlNullval;
+	}
+	if (p->os != PmcCtlNullval){
 		if (p->os)
 			v |= PeOS;
 		else
 			v &= ~PeOS;
-	if (pmctrans(p) < 0){
-		qunlock(&pmclck);
+		p->os = PmcCtlNullval;
+	}
+	if (pmctrans(p) < 0)
 		return -1;
-	}
 
-	if (!p->nodesc) {
-		if (tokenize(p->descstr, toks, 2) != 2)
+	if (p->nodesc == 0) {
+		memmove(str, p->descstr, KNAMELEN);
+		if (tokenize(str, toks, 2) != 2)
 			return -1;
 		e = atoi(toks[0]);
 		u = atoi(toks[1]);
 		v &= ~(PeEvMskL|PeEvMskH|PeUnMsk);
 		v |= SetEvMsk(v, e);
 		v |= SetUMsk(v, u);
+		p->nodesc = 1;
 	}
-	if (p->reset == 1){
+	if (p->reset != PmcCtlNullval && p->reset) {
 		v = 0;
-		wrmsr(p->regno+ PerfCtrbase, 0);
+		wrmsr(regno+ PerfCtrbase, 0);
+		p->reset = PmcCtlNullval;
 	}
-	wrmsr(p->regno+ PerfEvtbase, v);
+	wrmsr(regno+ PerfEvtbase, v);
 	pmcuserenab(pmcanyenab());
 	if (pmcdebug) {
-		v = rdmsr(p->regno+ PerfEvtbase);
-		print("conf pmc[%#ux]: %#llux\n", p->regno, v);
+		v = rdmsr(regno+ PerfEvtbase);
+		print("conf pmc[%#ux]: %#llux\n", regno, v);
 	}
-	qunlock(&pmclck);
 	return 0;
 }
 
 int
-pmcctlstr(char *str, int nstr, Pmc *p)
+pmcctlstr(char *str, int nstr, PmcCtl *p)
 {
 	int ns;
 
 	ns = 0;
-	if (p->enab)
+	if (p->enab && p->enab != PmcCtlNullval)
 		ns += snprint(str + ns, nstr - ns, "enable\n");
 	else
 		ns += snprint(str + ns, nstr - ns, "disable\n");
 		
-	if (p->user)
+	if (p->user && p->user != PmcCtlNullval)
 		ns += snprint(str + ns, nstr - ns, "user\n");
-	if (p->os)
+	if (p->os && p->user != PmcCtlNullval)
 		ns += snprint(str + ns, nstr - ns, "os\n");
 	
 	//TODO, inverse pmctrans?
-	ns += snprint(str + ns, nstr - ns, "%s\n", p->descstr);
+	if(!p->nodesc)
+		ns += snprint(str + ns, nstr - ns, "%s\n", p->descstr);
+	else 
+		ns += snprint(str + ns, nstr - ns, "no desc\n");
 	return ns;
 }
 
 int
 pmcdescstr(char *str, int nstr)
 {
-	PmcCtrId *pi;
+	PmcCtlCtrId *pi;
 	int ns;
 
 	ns = 0;
@@ -188,25 +214,159 @@
 	return ns;
 }
 
-u64int
-pmcgetctr(u32int regno)
+static u64int
+getctr(u32int regno)
 {
 	return rdmsr(regno + PerfCtrbase);
 }
 
-int
-pmcsetctr(u64int v, u32int regno)
+static int
+setctr(u64int v, u32int regno)
 {
 	wrmsr(regno + PerfCtrbase, v);
 	return 0;
 }
 
+int
+notstale(void *x)
+{
+	PmcCtr *p;
+	p = (PmcCtr *)x;
+	return !p->stale;
+}
 
+u64int
+pmcgetctr(u32int coreno, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
 
+	if(coreno == m->machno)
+		return getctr(regno);
 
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+	ilock(&m->pmclock);
+	p->ctrset |= PmcGet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	return p->ctr;
+}
 
+int
+pmcsetctr(u32int coreno, u64int v, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
 
+	if(coreno == m->machno)
+		return setctr(v, regno);
 
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+	ilock(&m->pmclock);
+	p->ctr = v;
+	p->ctrset |= PmcSet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	return 0;
+}
 
+static void
+ctl2ctl(PmcCtl *dctl, PmcCtl *sctl)
+{
+	if(sctl->enab != PmcCtlNullval)
+		dctl->enab = sctl->enab;
+	if(sctl->user != PmcCtlNullval)
+		dctl->user = sctl->user;
+	if(sctl->os != PmcCtlNullval)
+		dctl->os = sctl->os;
+	if(sctl->nodesc == 0) {
+		memmove(dctl->descstr, sctl->descstr, KNAMELEN);
+		dctl->nodesc = 0;
+	}
+}
 
+int
+pmcsetctl(u32int coreno, PmcCtl *pctl, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
 
+	if(coreno == m->machno)
+		return setctl(pctl, regno);
+
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+	ilock(&m->pmclock);
+	ctl2ctl(&p->PmcCtl, pctl);
+	p->ctlset |= PmcSet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	return 0;
+}
+
+int
+pmcgetctl(u32int coreno, PmcCtl *pctl, u32int regno)
+{
+	PmcCtr *p;
+	Mach *mp;
+
+	if(coreno == m->machno)
+		return getctl(pctl, regno);
+
+	mp = sys->machptr[coreno];
+	p = &mp->pmc[regno];
+
+	ilock(&m->pmclock);
+	p->ctlset |= PmcGet;
+	p->stale = 1;
+	iunlock(&m->pmclock);
+	if(mp->proc != nil || mp->nixtype != NIXAC){
+		apicipi(mp->apicno);
+		sleep(&p->r, notstale, p);
+	}
+	ilock(&m->pmclock);
+	memmove(pctl, &p->PmcCtl, sizeof(PmcCtl));
+	iunlock(&m->pmclock);
+	return 0;
+}
+
+void
+pmcupdate(Mach *m)
+{
+	PmcCtr *p;
+	int i, maxct, wk;
+
+	maxct = pmcnregs();
+	for (i = 0; i < maxct; i++) {
+		p = &m->pmc[i];
+		ilock(&m->pmclock);
+		if(p->ctrset & PmcSet)
+			setctr(p->ctr, i);
+		if(p->ctlset & PmcSet)
+			setctl(p, i);
+		p->ctr = getctr(i);
+		getctl(p, i);
+		p->ctrset = PmcIgn;
+		p->ctlset = PmcIgn;
+		wk = p->stale;
+		p->stale = 0;
+		iunlock(&m->pmclock);
+		if(wk)
+			wakeup(&p->r);
+	}
+}
+
diff -r 35d39df17077 -r ddaa629716d5 src/9kron2/k8/trap.c
--- a/src/9kron2/k8/trap.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron2/k8/trap.c	Tue Jun 07 09:08:27 2011 +0000
@@ -7,6 +7,7 @@
 
 #include	<tos.h>
 #include	"ureg.h"
+#include	"../port/pmc.h"
 
 #include	"io.h"
 
@@ -270,7 +271,7 @@
 		mp = m;
 	tos->core = mp->machno;	
 	tos->nixtype = mp->nixtype;
-
+	pmcupdate(m);	
 	/*
 	 * The process may change its core.
 	 * Be sure it has the right cyclefreq.
@@ -320,6 +321,8 @@
 	if(up != nil)
 		up->ntrap++;	/* stats only, races ok */
 
+	pmcupdate(m);
+
 	vno = ureg->type;
 	if(ctl = vctl[vno]){
 		if(ctl->isintr){
diff -r 35d39df17077 -r ddaa629716d5 src/9kron2/port/devpmc.c
--- a/src/9kron2/port/devpmc.c	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron2/port/devpmc.c	Tue Jun 07 09:08:27 2011 +0000
@@ -13,11 +13,14 @@
 
 enum{
 	Qdir		= 0,
+	Qgctl,
+	Qcore,
+
+	Qctr,
 	Qdata,
 	Qctl,
-	Qgctl,
 
-	PmcRdStr = 4*1024,
+	PmcCtlRdStr = 4*1024,
 };
 
 #define PMCTYPE(x)	(((unsigned)x)&0xffful)
@@ -26,22 +29,21 @@
 
 Dirtab *pmctab;
 static int npmctab;
+Dirtab *toptab;
+static int ntoptab;
 int pmcdebug;
 
 static void
-pmcinit(void)
+topdirinit(int ncores)
 {
-	int nr, i;
+	int i;
 	Dirtab *d;
 
-	nr = pmcnregs();
-		
-	npmctab = 2 + 2*nr;
-	pmctab = malloc(npmctab * sizeof(Dirtab));
-	if (pmctab == nil)
+	ntoptab = 1 + ncores;
+	toptab = malloc(ntoptab * sizeof(Dirtab));
+	if (toptab == nil)
 		return;
-	
-	d = pmctab;
+	d = toptab;
 	strncpy(d->name, ".", KNAMELEN);
 	mkqid(&d->qid, Qdir, 0, QTDIR);
 	d->perm = DMDIR|0555;
@@ -49,20 +51,78 @@
 	strncpy(d->name, "ctrdesc", KNAMELEN);
 	mkqid(&d->qid, Qgctl, 0, 0);
 	d->perm = 0444;
-	for (i = 2; i < nr + 2; i++) {
+	for (i = 2; i < ncores + 2; i++) {
+		d = &toptab[i];
+		snprint(d->name, KNAMELEN, "core%4.4ud", i - 2);
+		mkqid(&d->qid, PMCQID(i - 2, Qcore), 0, QTDIR);
+		d->perm = DMDIR|0555;
+	}
+
+}
+
+static void
+ctrdirinit(void)
+{
+	int nr, i;
+	Dirtab *d;
+
+	nr = pmcnregs();
+		
+	npmctab = 1 + 2*nr;
+	pmctab = malloc(npmctab * sizeof(Dirtab));
+	if (pmctab == nil){
+		free(toptab);
+		toptab = nil;
+		return;
+	}
+
+	d = pmctab;
+	strncpy(d->name, ".", KNAMELEN);
+	mkqid(&d->qid, Qctr, 0, QTDIR);
+	d->perm = DMDIR|0555;
+	for (i = 1; i < nr + 1; i++) {
 		d = &pmctab[i];
-		snprint(d->name, KNAMELEN, "ctr%2.2ud", i - 2);
-		mkqid(&d->qid, PMCQID(i - 2, Qdata), 0, 0);
+		snprint(d->name, KNAMELEN, "ctr%2.2ud", i - 1);
+		mkqid(&d->qid, PMCQID(i - 1, Qdata), 0, 0);
 		d->perm = 0600;
 
 		d = &pmctab[nr + i];
-		snprint(d->name, KNAMELEN, "ctr%2.2udctl", i - 2);
-		mkqid(&d->qid, PMCQID(i - 2, Qctl), 0, 0);
+		snprint(d->name, KNAMELEN, "ctr%2.2udctl", i - 1);
+		mkqid(&d->qid, PMCQID(i - 1, Qctl), 0, 0);
 		d->perm = 0600;
 	}	
 	
 }
 
+static void
+pmcnull(PmcCtl *p)
+{
+	memset(p, 0xff, sizeof(PmcCtl));
+	p->enab = PmcCtlNullval;
+	p->user = PmcCtlNullval;
+	p->os = PmcCtlNullval;
+	p->reset = PmcCtlNullval;
+	p->nodesc = 1;
+}
+
+static void
+pmcinit(void)
+{
+	int i, j, ncores, nr;
+	Mach *mp;
+
+	ncores = 0;
+	nr = pmcnregs();
+	for(i = 0; i < MACHMAX; i++) {
+		if((mp = sys->machptr[i]) != nil && mp->online != 0)
+			ncores++;
+		for(j = 0; j < nr; j++)
+			pmcnull(&mp->pmc[j]);
+	}
+	topdirinit(ncores);
+	ctrdirinit();
+}
+
 static Chan *
 pmcattach(char *spec)
 {
@@ -70,17 +130,63 @@
 		error(Enomem);
 	return devattach(L'ε', spec);
 }
+int
+pmcgen(Chan *c, char *name, Dirtab*, int, int s, Dir *dp)
+{
+	int t, i, n;
+	Dirtab *l, *d;
+
+	if(s == DEVDOTDOT){
+		devdir(c, (Qid){Qdir, 0, QTDIR}, "#ε", 0, eve, 0555, dp);
+		c->aux = nil;
+		return 1;
+	}
+	/* first, for directories, generate children */
+	switch((int)PMCTYPE(c->qid.path)){
+	case Qdir:
+		return devgen(c, name, toptab, ntoptab, s, dp);
+	case Qctr:
+		return devgen(c, name, pmctab, npmctab, s, dp);
+	case Qcore:
+		return devgen(c, name, pmctab, npmctab, s, dp);
+	default:
+		if(s != 0)
+			return -1;
+
+		t = PMCTYPE(c->qid.path);
+		if(t < Qctr){
+			i = t;
+			l = toptab;
+			n = ntoptab;
+		}else{
+			i = PMCID(t);
+			if (t == Qctl)
+				i += (npmctab - 1)/2;
+			l = pmctab;
+			n = npmctab;
+		}
+		if(i >=n)
+			return -1;
+
+		d = &l[i];
+		
+		devdir(c, d->qid, d->name, d->length, eve, d->perm, dp);
+		return 1;
+	}
+}
 
 static Walkqid*
 pmcwalk(Chan *c, Chan *nc, char **name, int nname)
 {
-	return devwalk(c, nc, name, nname, pmctab, npmctab, devgen);
+	if(PMCTYPE(c->qid.path) == Qcore)
+		c->aux = (void *)PMCID(c->qid.path);		/* core no */
+	return devwalk(c, nc, name, nname, nil, 0, pmcgen);
 }
 
 static long
 pmcstat(Chan *c, uchar *dp, long n)
 {
-	return devstat(c, dp, n, pmctab, npmctab, devgen);
+	return devstat(c, dp, n, nil, 0, pmcgen);
 }
 
 static Chan*
@@ -88,69 +194,56 @@
 {
 	if (!iseve())
 		error(Eperm);
-	return devopen(c, omode, pmctab, npmctab, devgen);
+	return devopen(c, omode, nil, 0, pmcgen);
 }
 
 static void
-pmcclose(Chan *c)
+pmcclose(Chan *)
 {
-	if (c->aux) {
-		free(c->aux);
-		c->aux = nil;
-	}
 }
 
 
-int
-pmcanyenab(void)
-{
-	int i;
-	Pmc p;
-
-	for (i = 0; i < pmcnregs(); i++) {
-		if (pmcgetctl(&p, i) < 0)
-			return -1;
-		if (p.enab)
-			return 1;
-	}
-
-	return 0;
-	
-}
 
 static long
 pmcread(Chan *c, void *a, long n, vlong offset)
 {
 	ulong type, id;
-	Pmc p;
+	PmcCtl p;
 	char *s;
 	u64int v;
+	u64int coreno;
 
-	if (c->qid.type == QTDIR)
-		return devdirread(c, a, n, pmctab, npmctab, devgen);
+	type = PMCTYPE(c->qid.path);
+	id = PMCID(c->qid.path);
 
-	s = malloc(PmcRdStr);
+	switch(type){
+	case Qcore:
+	case Qdir:
+	case Qctr:
+		return devdirread(c, a, n, nil, 0, pmcgen);
+	}
+
+	s = malloc(PmcCtlRdStr);
 	if(waserror()){
 		free(s);
 		nexterror();
 	}
-	type = PMCTYPE(c->qid.path);
-	id = PMCID(c->qid.path);
+
+	coreno = (u64int)c->aux;
+	p.coreno = coreno;
 	switch(type){
 	case Qdata:
-		if (up->ac != nil)
-			error("unimplemented yet");
-		v = pmcgetctr(id);
-		snprint(s, PmcRdStr, "%#ullx", v);
+		v = pmcgetctr(coreno, id);
+		snprint(s, PmcCtlRdStr, "%#ullx", v);
 		break;
 	case Qctl:
-		if (pmcgetctl(&p, id) < 0)
+		if (pmcgetctl(coreno, &p, id) < 0)
 			error("bad ctr");
-		if (pmcctlstr(s, PmcRdStr, &p) < 0)
+		if (pmcctlstr(s, PmcCtlRdStr, &p) < 0)
 			error("bad pmc");
 		break;
 	case Qgctl:
-		if (pmcdescstr(s, PmcRdStr) < 0)
+		if (pmcdescstr(s, PmcCtlRdStr) < 0)
 			error("bad pmc");
 		break;
 	default:
@@ -181,29 +274,49 @@
 	Debug, 		"debug",	0,
 };
 
-static void
-pmcnull(Pmc *p)
-{
-	memset(p, 0xff, sizeof(Pmc));
-	p->enab = PmcNullval;
-	p->user = PmcNullval;
-	p->os = PmcNullval;
-}
+typedef void (*APfunc)(void);
 
-typedef void (*APfunc)(void);
+typedef struct AcPmcArg AcPmcArg;
+struct AcPmcArg {
+	int regno;
+	int coreno;
+	PmcCtl;
+};
+
+typedef struct AcCtrArg AcCtrArg;
+struct AcCtrArg {
+	int regno;
+	int coreno;
+	u64int v;
+};
 
 void
 acpmcsetctl(void)
 {
-	Pmc p;
+	AcPmcArg p;
 	Mach *mp;
 
 	mp = up->ac;
-	memmove(&p, mp->icc->data, sizeof(Pmc));
+	memmove(&p, mp->icc->data, sizeof(AcPmcArg));
 	
-	mp->icc->rc = pmcsetctl(&p);
+	mp->icc->rc = pmcsetctl(p.coreno, &p, p.regno);
 	return;
 }
+
+void
+acpmcsetctr(void)
+{
+	AcCtrArg ctr;
+	Mach *mp;
+
+	mp = up->ac;
+	memmove(&ctr, mp->icc->data, sizeof(AcCtrArg));
+	
+	mp->icc->rc = pmcsetctr(ctr.coreno, ctr.v, ctr.regno);
+	return;
+}
+
+
 static long
 pmcwrite(Chan *c, void *a, long n, vlong)
 {
@@ -211,7 +324,10 @@
 	Cmdtab *ct;
 	ulong type;
 	char str[64];	/* 0x0000000000000000\0 */
-	Pmc p;
+	AcPmcArg p;
+	AcCtrArg ctr;
+	u64int coreno;
+	Mach *mp;
 
 	if (c->qid.type == QTDIR)
 		error(Eperm);
@@ -221,12 +337,25 @@
 		error(Ebadctl);
 
 	pmcnull(&p);
+	coreno = (u64int)c->aux;
+	p.coreno = coreno;
 	type = PMCTYPE(c->qid.path);
 	p.regno = PMCID(c->qid.path);
 	memmove(str, a, n);
 	str[n] = '\0';
+	mp = up->ac;
+
+	ctr.coreno = coreno;
+	ctr.regno = p.regno;
 	if (type == Qdata) {
-		pmcsetctr(atoi(str), p.regno);
+		/* I am a handler for a proc in the core, run an RPC*/
+		if (mp != nil && mp->machno == coreno) {
+			if (runac(mp, acpmcsetctr, 0, &ctr, sizeof(AcCtrArg)) < 0)
+				n = -1;
+		} else {
+		if (pmcsetctr(coreno, strtoull(str, 0, 0), p.regno) < 0)
+			n = -1;
+		}
 		return n;
 	}
 
@@ -234,6 +363,7 @@
 	/* TODO: should iterate through multiple lines */
 	if (strncmp(str, "set ", 4) == 0){
 		memmove(p.descstr, (char *)str + 4, n - 4);
+		p.descstr[n - 4] = '\0';
 		p.nodesc = 0;
 	} else {
 		cb = parsecmd(a, n);
@@ -268,11 +398,12 @@
 		free(cb);
 		poperror();
 	}
-	if (up->ac != nil) {
-		if (runac(up->ac, acpmcsetctl, 0, &p, sizeof(Pmc)) < 0)
+	/* I am a handler for a proc in the core, run an RPC*/
+	if (mp != nil && mp->machno == coreno) {
+		if (runac(mp, acpmcsetctl, 0, &p, sizeof(AcPmcArg)) < 0)
 			n = -1;
 	} else {
-		if (pmcsetctl(&p) < 0)
+		if (pmcsetctl(coreno, &p, p.regno) < 0)
 			n = -1;
 	}
 	return n;
diff -r 35d39df17077 -r ddaa629716d5 src/9kron2/port/pmc.h
--- a/src/9kron2/port/pmc.h	Tue Jun 07 08:15:53 2011 +0000
+++ b/src/9kron2/port/pmc.h	Tue Jun 07 09:08:27 2011 +0000
@@ -1,34 +1,23 @@
 
 enum{
-	PmcNullval = 0xdead,
+	PmcCtlNullval = 0xdead,
 };
 
-typedef struct PmcCtrId PmcCtrId;
-struct PmcCtrId {
+typedef struct PmcCtlCtrId PmcCtlCtrId;
+
+
+struct PmcCtlCtrId {
 	char portdesc[KNAMELEN];
 	char archdesc[KNAMELEN];
 };
 
+int		pmcnregs(void);
+int		pmcsetctl(u32int coreno, PmcCtl *p, u32int regno);
+int		pmctrans(PmcCtl *p);
+int		pmcgetctl(u32int coreno, PmcCtl *p, u32int regno);
+int		pmcdescstr(char *str, int nstr);
+int		pmcctlstr(char *str, int nstr, PmcCtl *p);
+u64int	pmcgetctr(u32int coreno, u32int regno);
+int		pmcsetctr(u32int coreno, u64int v, u32int regno);
 
-typedef struct Pmc Pmc;
-struct Pmc{
-	u32int regno;
-	int enab;
-	int user;
-	int os;
-	int nodesc;
-	char descstr[KNAMELEN];
-	int reset;
-};
-
-int		pmcnregs(void);
-int		pmcsetctl(Pmc *p);
-int		pmctrans(Pmc *p);
-int		pmcgetctl(Pmc *p, u32int regno);
-int		pmcdescstr(char *str, int nstr);
-int		pmcctlstr(char *str, int nstr, Pmc *p);
-u64int	pmcgetctr(u32int regno);
-int		pmcsetctr(u64int v, u32int regno);
-
-int		pmcanyenab(void);
-
+void		pmcupdate(Mach *m);