decouple lapics and apics from the sipi, mp, mpacpi.

ride-along fixes
- add bustype to ioapicintrinit's arguments
- thus, we can eliminate the public mpisabusno from mp, and make it internal
to the ioapic code.
- add gsitoapicid to translate from gsi to apic id.
- the devno for isa is no longer artificially multipled by 4.  (necessary for acpi irqs.)
- rename acpiinit() to devacpiinit().  the reasons for that will be clear in a later
patch
- add affinity to Vctl.  this is currently just an integer—the machno.  this is
not used yet, and could be expanded later to include other bits.
- panic instead of failing silently.

Reference: /n/atom/patch/applied2013/apicdecouple
Date: Fri Aug 23 03:32:17 CES 2013
Signed-off-by: quanstro@quanstro.net

--- /sys/src/nix/k10/io.h	Fri Aug 23 03:25:10 2013
+++ /sys/src/nix/k10/io.h	Fri Aug 23 03:25:10 2013
@@ -70,6 +70,7 @@
 	Vctl*	next;			/* handlers on this vector */
 
 	int	isintr;			/* interrupt or fault/trap */
+	int	affinity;			/* processor affinity (-1 for none) */
 
 	Vkey;				/* source-specific key; tbdf for pci */
 	void	(*f)(Ureg*, void*);	/* handler to call */
--- /sys/src/nix/k10/lapic.c	Fri Aug 23 03:25:11 2013
+++ /sys/src/nix/k10/lapic.c	Fri Aug 23 03:25:12 2013
@@ -74,7 +74,20 @@
 static u32int* lapicbase;
 static int lapmachno = 1;
 
-Apic	xlapic[Napic];
+static	Apic	xlapic[Napic];
+
+Apic*
+lapiclookup(uint id)
+{
+	Apic *a;
+
+	if(id > nelem(xlapic))
+		return nil;
+	a = xlapic + id;
+	if(a->useable)
+		return a;
+	return nil;
+}
 
 static u32int
 lapicrget(int r)
@@ -161,17 +174,17 @@
 	addarchfile("lapic", 0444, lapicread, nil);
 
 	if(lapicno >= Napic){
-		print("lapicinit%d: out of range\n", lapicno);
+		panic("lapicinit%d: out of range", lapicno);
 		return;
 	}
 	if((apic = &xlapic[lapicno])->useable){
-//		print("lapicinit%d: already initialised\n", lapicno);
+		print("lapicinit%d: already initialised\n", lapicno);
 		return;
 	}
 	if(lapicbase == nil){
 		adrmapck(pa, 1024, Aapic, Mfree);
 		if((lapicbase = vmap(pa, 1024)) == nil){
-			print("lapicinit%d: can't map lapicbase %#P\n", lapicno, pa);
+			panic("lapicinit%d: can't map lapicbase %#P", lapicno, pa);
 			return;
 		}
 		DBG("lapicinit%d: lapicbase %#P -> %#p\n", lapicno, pa, lapicbase);
@@ -236,10 +249,18 @@
 		panic("lapiconline: no lapic base");
 
 	if((apicno = ((lapicrget(Id)>>24) & 0xff)) >= Napic)
-		return 0;
+		panic("lapic: id too large %d", apicno);
+	if(apicno != m->apicno){
+		panic("lapic: %d != %d", m->apicno, apicno);
+		dfr = lapicrget(Id) & ~(0xff<<24);
+		dfr |= m->apicno<<24;
+		lapicrput(Id, dfr);
+		apicno = m->apicno;
+	}
 	apic = &xlapic[apicno];
 	if(!apic->useable || apic->addr != nil)
-		return 0;
+		panic("lapiconline: lapic%d: useable %d addr %#p",
+			apicno, apic->useable, apic->addr);
 
 	/*
 	 * Things that can only be done when on the processor
--- /sys/src/nix/k10/ioapic.c	Fri Aug 23 03:25:13 2013
+++ /sys/src/nix/k10/ioapic.c	Fri Aug 23 03:25:14 2013
@@ -13,6 +13,7 @@
 
 struct Rbus {
 	Rbus	*next;
+	int	bustype;
 	int	devno;
 	Rdt	*rdt;
 };
@@ -47,7 +48,55 @@
 static Lock idtnolock;
 static int idtno = IdtIOAPIC;
 
-Apic	xioapic[Napic];
+static	Apic	xioapic[Napic];
+static	int	isabusno = -1;
+
+/* BOTCH: no need for this concept; we've got the bustype */
+static void
+ioapicisabus(int busno)
+{
+	if(isabusno != -1){
+		if(busno == isabusno)
+			return;
+		print("ioapic: isabus redefined: %d ↛ %d\n", isabusno, busno);
+//////		return;
+	}
+	print("ioapic: isa busno %d\n", busno);
+	isabusno = busno;
+}
+
+Apic*
+ioapiclookup(uint id)
+{
+	Apic *a;
+
+	if(id > nelem(xioapic))
+		return nil;
+	a = xioapic + id;
+	if(a->useable)
+		return a;
+	return nil;
+}
+
+int
+gsitoapicid(int gsi, uint *intin)
+{
+	int i;
+	Apic *a;
+
+	for(i=0; i<Napic; i++){
+		a = xioapic + i;
+		if(!a->useable)
+			continue;
+		if(gsi >= a->ibase && gsi <= a->ibase+a->nrdt){
+			if(intin != nil)
+				*intin = gsi - a->ibase;
+			return a - xioapic;
+		}
+	}
+	print("gsitoapicid: no ioapic found for gsi %d\n", gsi);
+	return -1;
+}
 
 static void
 rtblget(Apic* apic, int sel, u32int* hi, u32int* lo)
@@ -86,7 +135,7 @@
 }
 
 void
-ioapicintrinit(int busno, int apicno, int intin, int devno, u32int lo)
+ioapicintrinit(int bustype, int busno, int apicno, int intin, int devno, u32int lo)
 {
 	Rbus *rbus;
 	Rdt *rdt;
@@ -94,19 +143,28 @@
 
 	if(busno >= Nbus || apicno >= Napic || nrdtarray >= Nrdt)
 		return;
+
+	if(bustype == BusISA)
+		ioapicisabus(busno);
+
 	apic = &xioapic[apicno];
 	if(!apic->useable || intin >= apic->nrdt)
-		return;
+		panic("ioapic: intrinit: usable %d: bus %d apic %d intin %d dev %d lo %.8ux\n",
+			apic->useable, busno, apicno, intin, devno, lo);
 
 	rdt = rdtlookup(apic, intin);
 	if(rdt == nil){
+		if(nrdtarray == nelem(rdtarray)){
+			print("ioapic: intrinit: rdtarray too small\n");
+			return;
+		}
 		rdt = &rdtarray[nrdtarray++];
 		rdt->apic = apic;
 		rdt->intin = intin;
 		rdt->lo = lo;
 	}else{
 		if(lo != rdt->lo){
-			print("mutiple irq botch bus %d %d/%d/%d lo %d vs %d\n",
+			print("mutiple irq botch bus %d %d/%d/%d lo %.8ux vs %.8ux\n",
 				busno, apicno, intin, devno, lo, rdt->lo);
 			return;
 		}
@@ -115,6 +173,7 @@
 	rdt->ref++;
 	rbus = malloc(sizeof *rbus);
 	rbus->rdt = rdt;
+	rbus->bustype = bustype;
 	rbus->devno = devno;
 	rbus->next = rdtbus[busno];
 	rdtbus[busno] = rbus;
@@ -282,9 +341,11 @@
 	ioapicdump();
 }
 
-static void
+static int
 ioapicintrdd(u32int* hi, u32int* lo)
 {
+	Apic *lapic;
+	Mach *mach;
 	static int i;
 
 	/*
@@ -294,16 +355,17 @@
 	 * using physical mode delivery.  Using the topology
 	 * (packages/cores/threads) could be helpful.
 	 */
-	for(;; i = (i+1) % nelem(xlapic)){
-		if(!xlapic[i].useable)
+	for(;; i = (i+1) % Napic){
+		if((lapic = lapiclookup(i)) == nil)
 			continue;
-		if(sys->machptr[xlapic[i].machno] == nil)
+		if((mach = sys->machptr[lapic->machno]) == nil)
 			continue;
-		if(sys->machptr[xlapic[i].machno]->online != 0)
+		if(mach->online)
 			break;
 	}
 	*hi = i++<<24;
 	*lo |= Pm|MTf;
+	return mach->machno;
 }
 
 int
@@ -341,7 +403,7 @@
 	vno = nextvec();
 
 	lo = IPlow | TMedge | vno;
-	ioapicintrdd(&hi, &lo);
+	v->affinity = ioapicintrdd(&hi, &lo);
 
 	if(lo & Lm)
 		lo |= MTlp;
@@ -373,12 +435,8 @@
 	Rbus *rbus;
 	Rdt *rdt;
 	u32int hi, lo;
-	int busno, devno, vecno;
+	int bustype, busno, devno, vecno;
 
-	/*
-	 * Bridge between old and unspecified new scheme,
-	 * the work in progress...
-	 */
 	if(v->tbdf == BUSUNKNOWN){
 		if(v->irq >= IrqLINT0 && v->irq <= MaxIrqLAPIC){
 			if(v->irq != IrqSPURIOUS)
@@ -392,15 +450,14 @@
 			 * Make a busno and devno using the
 			 * ISA bus number and the irq.
 			 */
-			extern int mpisabusno;
-
-			if(mpisabusno == -1)
+			if(isabusno == -1)
 				panic("no ISA bus allocated");
-			busno = mpisabusno;
-			devno = v->irq<<2;
+			busno = isabusno;
+			devno = v->irq;
+			bustype = BusISA;
 		}
 	}
-	else if(BUSTYPE(v->tbdf) == BusPCI){
+	else if((bustype = BUSTYPE(v->tbdf)) == BusPCI){
 		/*
 		 * PCI.
 		 * Make a devno from BUSDNO(tbdf) and pcidev->intp.
@@ -409,37 +466,35 @@
 
 		busno = BUSBNO(v->tbdf);
 		if((pcidev = pcimatchtbdf(v->tbdf)) == nil)
-			panic("no PCI dev for tbdf %#8.8ux", v->tbdf);
+			panic("no PCI dev for tbdf %T", v->tbdf);
 		if((vecno = intrenablemsi(v, pcidev)) != -1)
 			return vecno;
 		disablemsi(v, pcidev);
 		if((devno = pcicfgr8(pcidev, PciINTP)) == 0)
-			panic("no INTP for tbdf %#8.8ux", v->tbdf);
+			panic("no INTP for tbdf %T", v->tbdf);
 		devno = BUSDNO(v->tbdf)<<2|(devno-1);
-		DBG("ioapicintrenable: tbdf %#8.8ux busno %d devno %d\n",
+		DBG("ioapicintrenable: tbdf %T busno %d devno %d\n",
 			v->tbdf, busno, devno);
 	}
 	else{
 		SET(busno, devno);
-		panic("unknown tbdf %#8.8ux", v->tbdf);
+		panic("unknown tbdf %T", v->tbdf);
 	}
 
 	rdt = nil;
 	for(rbus = rdtbus[busno]; rbus != nil; rbus = rbus->next)
-		if(rbus->devno == devno){
+		if(rbus->devno == devno && rbus->bustype == bustype){
 			rdt = rbus->rdt;
 			break;
 		}
 	if(rdt == nil){
-		extern int mpisabusno;
-
 		/*
 		 * First crack in the smooth exterior of the new code:
 		 * some BIOS make an MPS table where the PCI devices are
 		 * just defaulted to ISA.
 		 * Rewrite this to be cleaner.
 		 */
-		if((busno = mpisabusno) == -1)
+		if((busno = isabusno) == -1)
 			return -1;
 		devno = v->irq<<2;
 		for(rbus = rdtbus[busno]; rbus != nil; rbus = rbus->next)
@@ -447,21 +502,14 @@
 				rdt = rbus->rdt;
 				break;
 			}
-		DBG("isa: tbdf %#8.8ux busno %d devno %d %#p\n",
+		DBG("isa: tbdf %T busno %d devno %d %#p\n",
 			v->tbdf, busno, devno, rdt);
 	}
 	if(rdt == nil)
 		return -1;
 
 	/*
-	 * Second crack:
-	 * what to do about devices that intrenable/intrdisable frequently?
-	 * 1) there is no ioapicdisable yet;
-	 * 2) it would be good to reuse freed vectors.
-	 * Oh bugger.
-	 */
-	/*
-	 * This is a low-frequency event so just lock
+	 * Assume this is a low-frequency event so just lock
 	 * the whole IOAPIC to initialise the RDT entry
 	 * rather than putting a Lock in each entry.
 	 */
@@ -476,12 +524,12 @@
 
 	rdt->enabled++;
 	lo = (rdt->lo & ~Im);
-	ioapicintrdd(&hi, &lo);
+	v->affinity = ioapicintrdd(&hi, &lo);
 	rtblput(rdt->apic, rdt->intin, hi, lo);
 	vecno = lo & 0xff;
 	unlock(rdt->apic);
 
-	DBG("busno %d devno %d hi %#8.8ux lo %#8.8ux vecno %d\n",
+	DBG("busno %d devno %d hi %#.8ux lo %#.8ux vecno %d\n",
 		busno, devno, hi, lo, vecno);
 	v->isr = lapicisr;
 	v->eoi = lapiceoi;
--- /sys/src/nix/k10/apic.h	Fri Aug 23 03:25:15 2013
+++ /sys/src/nix/k10/apic.h	Fri Aug 23 03:25:15 2013
@@ -39,7 +39,7 @@
 enum {
 	Nbus		= 256,			/* must be 256 */
 	Napic		= 254,			/* xAPIC architectural limit */
-	Nrdt		= 64,
+	Nrdt		= 128,
 };
 
 /*
@@ -77,17 +77,13 @@
 	Im		= 0x00010000,		/* Interrupt Mask */
 };
 
-extern	Apic	xlapic[Napic];
-extern	Apic	xioapic[Napic];
-extern	Mach	*xlapicmachptr[Napic];		/* maintained, but unused */
+void	apictimerenab(void);
+int	gsitoapicid(int, uint*);
+void	ioapicdump(void);
+void	ioapicintrinit(int, int, int, int, int, u32int);
+Apic*	ioapiclookup(uint);
+void	lapicdump(void);
+Apic*	lapiclookup(uint);
 
-#define l16get(p)	(((p)[1]<<8)|(p)[0])
-#define	l32get(p)	(((u32int)l16get(p+2)<<16)|l16get(p))
-#define	l64get(p)	(((u64int)l32get(p+4)<<32)|l32get(p))
-
-extern void lapicdump(void);
-extern void apictimerenab(void);
-extern void ioapicdump(void);
-
-extern int pcimsienable(Pcidev*, uvlong);
-extern int pcimsimask(Pcidev*, int);
+int	pcimsienable(Pcidev*, uvlong);
+int	pcimsimask(Pcidev*, int);
--- /sys/src/nix/k10/mp.c	Fri Aug 23 03:25:17 2013
+++ /sys/src/nix/k10/mp.c	Fri Aug 23 03:25:17 2013
@@ -3,9 +3,14 @@
 #include "mem.h"
 #include "dat.h"
 #include "fns.h"
+#include "io.h"
 
 #include "apic.h"
 
+#define l16get(p)	(((p)[1]<<8)|(p)[0])
+#define	l32get(p)	(((u32int)l16get(p+2)<<16)|l16get(p))
+#define	l64get(p)	(((u64int)l32get(p+4)<<32)|l32get(p))
+
 /*
  * MultiProcessor Specification Version 1.[14].
  */
@@ -46,7 +51,6 @@
 	{ "ISA   ", IPhigh, TMedge, },
 };
 static Mpbus* mpbus[Nbus];
-int mpisabusno = -1;
 
 static void
 mpintrprint(char* s, u8int* p)
@@ -93,8 +97,7 @@
 			mpintrprint("INTIN botch", p);
 			return 0;
 		case 3:				/* IOINTR */
-			apic = &xioapic[p[6]];
-			if(!apic->useable){
+			if((apic = ioapiclookup(p[6])) == nil){
 				mpintrprint("unuseable ioapic", p);
 				return 0;
 			}
@@ -104,8 +107,7 @@
 			}
 			break;
 		case 4:				/* LINTR */
-			apic = &xlapic[p[6]];
-			if(!apic->useable){
+			if((apic = lapiclookup(p[6])) == nil){
 				mpintrprint("unuseable lapic", p);
 				return 0;
 			}
@@ -113,6 +115,7 @@
 				mpintrprint("LOCAL INTIN out of range", p);
 				return 0;
 			}
+			USED(apic);
 			break;
 		}
 	}
@@ -176,9 +179,10 @@
 static int
 mpparse(PCMP* pcmp, int maxmach)
 {
-	u32int lo;
 	u8int *e, *p;
-	int nmach, devno, i, n;
+	int nmach, bustype, i, n;
+	u32int lo;
+	Apic *a;
 
 	nmach = 0;
 	p = pcmp->entries;
@@ -220,14 +224,6 @@
 		for(i = 0; i < nelem(mpbusdef); i++){
 			if(memcmp(p+2, mpbusdef[i].type, 6) != 0)
 				continue;
-			if(memcmp(p+2, "ISA   ", 6) == 0){
-				if(mpisabusno != -1){
-					print("mpparse: bus %d already have ISA bus %d\n",
-						p[1], mpisabusno);
-					continue;
-				}
-				mpisabusno = p[1];
-			}
 			mpbus[p[1]] = &mpbusdef[i];
 			break;
 		}
@@ -268,17 +264,13 @@
 		if(DBGFLG)
 			mpintrprint(nil, p);
 
-		/*
-		 * Always present the device number in the style
-		 * of a PCI Interrupt Assignment Entry. For the ISA
-		 * bus the IRQ is the device number but unencoded.
-		 * May need to handle other buses here in the future
-		 * (but unlikely).
-		 */
-		devno = p[5];
-		if(memcmp(mpbus[p[4]]->type, "PCI   ", 6) != 0)
-			devno <<= 2;
-		ioapicintrinit(p[4], p[6], p[7], devno, lo);
+		bustype = -1;
+		if(memcmp(mpbus[p[4]]->type, "PCI   ", 6) == 0)
+			bustype = BusPCI;
+		else if(memcmp(mpbus[p[4]]->type, "ISA   ", 6) == 0)
+			bustype = BusISA;
+		if(bustype != -1)
+			ioapicintrinit(bustype, p[4], p[6], p[7], p[5], lo);
 
 		p += 8;
 		break;
@@ -298,13 +290,15 @@
 		 */
 		if(p[6] == 0xff){
 			for(i = 0; i < Napic; i++){
-				if(!xlapic[i].useable || xlapic[i].addr != nil)
+				if((a = lapiclookup(i)) == nil || a->addr != nil)
 					continue;
-				xlapic[i].lvt[p[7]] = lo;
+				a->lvt[p[7]] = lo;
 			}
 		}
-		else
-			xlapic[p[6]].lvt[p[7]] = lo;
+		else{
+			if((a = lapiclookup(p[6])) != nil)
+				a->lvt[p[7]] = lo;
+		}
 		p += 8;
 		break;
 	}
--- /sys/src/nix/k10/sipi.c	Fri Aug 23 03:25:18 2013
+++ /sys/src/nix/k10/sipi.c	Fri Aug 23 03:25:18 2013
@@ -26,7 +26,7 @@
 	 */
 	sipipa = mmuphysaddr(SIPIHANDLER);
 	if((sipipa & (4*KiB - 1)) || sipipa > (1*MiB - 2*4*KiB))
-		return;
+		panic("sipi: invalid sipipa");
 	sipiptr = UINT2PTR(SIPIHANDLER);
 	memmove(sipiptr, sipihandler, sizeof(sipihandler));
 	DBG("sipiptr %#p sipipa %#P\n", sipiptr, sipipa);
@@ -41,8 +41,7 @@
 	 */
 	nproc = 0;
 	for(apicno = 0; apicno < Napic; apicno++){
-		apic = &xlapic[apicno];
-		if(!apic->useable || apic->addr || apic->machno == 0)
+		if((apic = lapiclookup(apicno)) == nil || apic->addr != 0 || apic->machno == 0)
 			continue;
 		nproc++;
 		if(nproc == MACHMAX){
@@ -58,7 +57,6 @@
 		alloc = mallocalign(MACHSTKSZ+4*PTSZ+4*KiB+MACHSZ, 4096, 0, 0);
 		if(alloc == nil)
 			continue;
-		memset(alloc, 0, MACHSTKSZ+4*PTSZ+4*KiB+MACHSZ);
 		p = alloc+MACHSTKSZ;
 
 		sipiptr[-1] = mmuphysaddr(PTR2UINT(p));
--- /sys/src/nix/k10/mpacpi.c	Fri Aug 23 03:25:19 2013
+++ /sys/src/nix/k10/mpacpi.c	Fri Aug 23 03:25:20 2013
@@ -28,30 +28,20 @@
 		already = "";
 		switch(st->type){
 		case ASlapic:
-			if(st->lapic.id > Napic || np == maxmach)
-				break;
-			apic = xlapic + st->lapic.id;
 			bp = (np++ == 0);
-			if(apic->useable){
+			if(lapiclookup(st->lapic.id) == nil)
 				already = "(mp)";
-				goto pr;
-			}
-			lapicinit(st->lapic.id, apics->lapicpa, bp);
-		pr:
+			else
+				lapicinit(st->lapic.id, apics->lapicpa, bp);
 			USED(already);
 			DBG("lapic: mach %d/%d lapicid %d %s\n", np-1, apic->machno, st->lapic.id, already);
 			break;
 		case ASioapic:
-			if(st->ioapic.id > Napic)
-				break;
-			apic = xioapic + st->ioapic.id;
-			if(apic->useable){
+			if((apic = ioapiclookup(st->ioapic.id)) != nil){
 				apic->ibase = st->ioapic.ibase;	/* gnarly */
 				already = "(mp)";
-				goto pr1;
-			}
-			ioapicinit(st->ioapic.id, st->ioapic.ibase, st->ioapic.addr);
-		pr1:
+			}else
+				ioapicinit(st->ioapic.id, st->ioapic.ibase, st->ioapic.addr);
 			print("ioapic: %d ", st->ioapic.id);
 			print("addr %p base %d %s\n", apic->paddr, apic->ibase, already);
 			break;
--- /sys/src/nix/k10/fns.h	Fri Aug 23 03:25:21 2013
+++ /sys/src/nix/k10/fns.h	Fri Aug 23 03:25:21 2013
@@ -1,5 +1,5 @@
 #include "../port/portfns.h"
-int	acpiinit(void);
+void	acpiinit(int);
 Dirtab*	addarchfile(char*, int, long(*)(Chan*, void*, long, vlong), long(*)(Chan*, void*, long, vlong));
 void	adrinit(void);
 void	apicipi(int);
@@ -19,6 +19,7 @@
 u32int	cpuid(u32int, u32int, u32int[4]);
 int	dbgprint(char*, ...);
 void	delay(int);
+int	devacpiinit(void);
 void	dumpmmu(Proc*);
 void	dumpmmuwalk(uintmem);
 void	dumpptepg(int, uintmem);
@@ -178,7 +179,6 @@
 void	lapicsipi(int, uintmem);
 
 void	ioapicinit(int, int, uintmem);
-void	ioapicintrinit(int, int, int, int, u32int);
 void	ioapiconline(void);
 
 /*
--- /sys/src/nix/k10/main.c	Fri Aug 23 03:25:22 2013
+++ /sys/src/nix/k10/main.c	Fri Aug 23 03:25:23 2013
@@ -169,7 +169,7 @@
 	 * to malloc not being initialized correctly, or the data segment is misaligned
 	 * (it's amazing how far you can get with things like that completely broken).
 	 */
-	acpiinit();
+	devacpiinit();
 
 	umeminit();
 	trapinit();
--- /sys/src/nix/k10/devacpi.c	Fri Aug 23 03:25:25 2013
+++ /sys/src/nix/k10/devacpi.c	Fri Aug 23 03:25:26 2013
@@ -1510,7 +1510,7 @@
 }
 
 int
-acpiinit(void)
+devacpiinit(void)
 {
 	if(fadt.smicmd == 0){
 		fmtinstall('G', Gfmt);
@@ -1530,7 +1530,7 @@
 	 * This was written for the stock kernel.
 	 * This code must use 64 registers to be acpi ready in nix.
 	 */
-	if(1 || acpiinit() < 0)
+	if(1 || devacpiinit() < 0)
 		error("no acpi");
 
 	/*