move the selection of interrupt mechanism to trap.c, and
move the all msi, msi-x and ioapic (isa and pci) functions to respective
respective files.  this should cleanly separate these four interrupt
mechanisms.  

the lapic interrupt mess is an exercize for the reader

Reference: /n/atom/patch/applied/nixintrclean
Date: Fri Sep 18 15:04:40 CES 2015
Signed-off-by: quanstro@quanstro.net

--- /sys/src/nix/k10/msi.c	Fri Sep 18 15:04:06 2015
+++ /sys/src/nix/k10/msi.c	Fri Sep 18 15:04:08 2015
@@ -59,8 +59,8 @@
 	return 0;
 }
 
-int
-pcimsienable(Vkey *v, uvlong vec)
+static int
+msienable(Vkey *v, u64int vec)
 {
 	char *s;
 	uint c, f, d, datao, lopri, dmode, logical;
@@ -71,14 +71,13 @@
 
 	if((p = v->p) == nil)
 		return -1;
+	if(blacklist(p) != 0)
+		return -1;
 	c = msicap(p);
 	if(c == 0)
 		return -1;
 
 	f = pcicfgr16(p, c + 2) & ~Mmesgmsk;
-
-	if(blacklist(p) != 0)
-		return -1;
 	datao = 8;
 	d = vec>>48;
 	lopri = (vec & 0x700) == MTlp;
@@ -103,8 +102,8 @@
 	return 0;
 }
 
-int
-pcimsimask(Vkey *v, int mask)
+static int
+msimask(Vkey *v, int mask)
 {
 	uint c, f;
 	Pcidev *p;
@@ -124,3 +123,33 @@
 	}
 	return 0;
 }
+
+static int
+msidisable(Vctl *v)
+{
+	return msimask(v, 1);
+}
+
+int
+intrenablemsi(Vctl* v)
+{
+	u32int lo, hi;
+	u64int msivec;
+
+	lo = IPlow | TMedge;
+	ioapicphysdd(v, &hi, &lo);
+
+	msivec = (u64int)hi<<32 | lo;
+	if(msienable(v, msivec) == -1){
+		msidisable(v);
+		return -1;
+	}
+	v->eoi = lapiceoi;
+	v->type = "msi";
+	v->mask = msimask;
+	v->disable = msidisable;
+
+	DBG("msiirq: %T: enabling %.16llux %s vno %d\n", v->p->tbdf, msivec, v->name, v->vno);
+	return v->vno;
+}
+
--- /sys/src/nix/k10/msix.c	Fri Sep 18 15:04:09 2015
+++ /sys/src/nix/k10/msix.c	Fri Sep 18 15:04:10 2015
@@ -65,7 +65,7 @@
 
 enum {
 	Debug		= 1,
-	Dflags		= Debug,
+	Dflags		= 0,
 };
 
 #define dprint(flag, ...) do{if(flag&Dflags){print(__VA_ARGS__);}}while(0)
@@ -155,7 +155,7 @@
 }
 
 int
-pcimsixenable(Vkey *v, uvlong vec)
+msixenable(Vkey *v, u64int vec)
 {
 	char *s;
 	u32int *tbl, lopri, logical, d, dmode, o, c;
@@ -194,7 +194,7 @@
 }
 
 int
-pcimsixmask(Vkey *v, int mask)
+msixmask(Vkey *v, int mask)
 {
 	u32int *tbl, o;
 	Msix *x;
@@ -216,12 +216,14 @@
 }
 
 int
-pcimsixdisable(Pcidev *p)
+msixdisable(Vctl *v)
 {
 	u32int *tbl, i, c;
 	Msix *x;
 
-	if((x = getmsix(p)) == nil)
+	if((v->flag&Vmsix) == 0)
+		return -1;
+	if((x = getmsix(v->p)) == nil)
 		return -1;
 	tbl = x->tbl;
 	if(tbl == nil)
@@ -233,18 +235,34 @@
 
 	c = x->c+Msgctl;
 	if(adec(&x->enable) == 0)
-		pcicfgw16(p, c, pcicfgr16(p, c) & ~(Msixen|Fnmask));
+		pcicfgw16(v->p, c, pcicfgr16(v->p, c) & ~(Msixen|Fnmask));
 	return 0;
 
 }
 
-/* debug crap */
-void
-msixpba(Pcidev *p)
+int
+intrenablemsix(Vctl* v)
 {
-	Msix *x;
+	u32int lo, hi;
+	u64int msivec;
 
-	if((x = getmsix(p)) == nil)
-		return;
-	print("%T: pba %.16llux\n", x->p->tbdf, x->pba[0]);
+	if((v->flag&Vmsix) == 0)
+		return -1;
+	lo = IPlow | TMedge;
+
+	ioapicphysdd(v, &hi, &lo);
+	msivec = (u64int)hi<<32 | lo;
+	if(msixenable(v, msivec) == -1){
+		msixdisable(v);
+		return -1;
+	}
+	v->eoi = lapiceoi;
+	v->type = "msi-x";
+	v->mask = msixmask;
+	v->disable = msixdisable;
+
+	DBG("msix: %T: enabling %.16llux %s vno %d affinity %d\n",
+		v->p->tbdf, msivec, v->name, v->vno, v->affinity);
+	return v->vno;
 }
+
--- /sys/src/nix/k10/apic.h	Fri Sep 18 15:04:12 2015
+++ /sys/src/nix/k10/apic.h	Fri Sep 18 15:04:13 2015
@@ -79,7 +79,6 @@
 int	gsitoapicid(int, uint*);
 void	ioapicdump(void);
 Ioapic*	ioapicinit(int, int, uintmem);
-int	ioapicintrdisable(int);
 int	ioapicintrenable(Vctl*);
 void	ioapicintrinit(int, int, int, int, int, u32int);
 Ioapic*	ioapiclookup(uint);
@@ -94,8 +93,3 @@
 void	lapicpri(int);
 void	lapicsetcolor(int, int);
 void	lapicsipi(int, uintmem);
-
-int	pcimsienable(Vkey*, uvlong);
-int	pcimsimask(Vkey*, int);
-int	pcimsixenable(Vkey*, uvlong);
-int	pcimsixmask(Vkey*, int);
--- /sys/src/nix/k10/ioapic.c	Fri Sep 18 15:04:15 2015
+++ /sys/src/nix/k10/ioapic.c	Fri Sep 18 15:04:17 2015
@@ -429,73 +429,22 @@
 	return 0;
 }
 
-static int
-msimask(Vkey *v, int mask)
-{
-	return pcimsimask(v, mask);
-}
-
-static int
-intrenablemsi(Vctl* v)
-{
-	u32int lo, hi;
-	u64int msivec;
-
-	lo = IPlow | TMedge;
-	ioapicphysdd(v, &hi, &lo);
-
-	msivec = (u64int)hi<<32 | lo;
-	if(pcimsienable(v, msivec) == -1)
-		return -1;
-	v->eoi = lapiceoi;
-	v->type = "msi";
-	v->mask = msimask;
-
-	DBG("msiirq: %T: enabling %.16llux %s vno %d\n", v->p->tbdf, msivec, v->name, v->vno);
-	return v->vno;
-}
-
 int
-disablemsi(Vctl *v)
-{
-	return pcimsimask(v, 1);
-}
-
-static int
-msixmask(Vkey *v, int mask)
+ioapicintrdisable(Vctl* v)
 {
-	return pcimsixmask(v, mask);
-}
-
-static int
-intrenablemsix(Vctl* v)
-{
-	u32int lo, hi;
-	u64int msivec;
-
-	if((v->flag&Vmsix) == 0)
-		return -1;
-	lo = IPlow | TMedge;
+	Rdt *rdt;
 
-	ioapicphysdd(v, &hi, &lo);
-	msivec = (u64int)hi<<32 | lo;
-	if(pcimsixenable(v, msivec) == -1)
+	if((rdt = rdtvecno[v->vno]) == nil){
+		panic("ioapicintrdisable: vecno %d has no rdt", v->vno);
 		return -1;
-	v->eoi = lapiceoi;
-	v->type = "msi-x";
-	v->mask = msixmask;
+	}
 
-	DBG("msix: %T: enabling %.16llux %s vno %d affinity %d\n",
-		v->p->tbdf, msivec, v->name, v->vno, v->affinity);
-	return v->vno;
-}
+	lock(rdt->apic);
+	if(adec(&rdt->enabled) == 0)
+		rtblput(rdt->apic, rdt->intin, 0, rdt->lo);
+	unlock(rdt->apic);
 
-int
-disablemsix(Vctl *v)
-{
-	if((v->flag&Vmsix) == 0)
-		return -1;
-	return pcimsixmask(v, 1);
+	return 0;
 }
 
 int
@@ -562,72 +511,35 @@
 	DBG("busno %d devno %d hi %#.8ux lo %#.8ux vno %d af %d\n",
 		busno, devno, hi, lo, v->vno, v->affinity);
 	v->eoi = lapiceoi;
+	v->disable = ioapicintrdisable;
 	v->type = "ioapic";
 
 	return v->vno;
 }
 
 int
-ioapicintrenable(Vctl* v)
+intrenableioapicisa(Vctl* v)
 {
-	int devno;
-	Pcidev *p;
-
-	if(v->tbdf == BUSUNKNOWN){
-		if(v->irq >= IrqLINT0 && v->irq <= MaxIrqLAPIC){
-			if(v->irq != IrqSPURIOUS)
-				v->isr = lapiceoi;
-			v->type = "lapic";
-			return v->irq;
-		}
-		else{
-			/*
-			 * Legacy ISA.
-			 * Make a busno and devno using the
-			 * ISA bus number and the irq.
-			 */
-			if(isabusno == -1)
-				panic("no ISA bus allocated");
-			return ioapicrdt(v, isabusno, v->irq, BusISA);
-		}
-	}
-	else if(BUSTYPE(v->tbdf) == BusPCI){
-		/* BOTCH msi is in the wrong place.  it has nothing to do with ioapics */
-		if((p = pcimatchtbdf(v->tbdf)) == nil)
-			panic("ioapic: no pci dev for tbdf %T", v->tbdf);
-		v->p = p;
-		if(intrenablemsix(v) != -1)
-			return v->vno;
-		disablemsix(v);
-		if(intrenablemsi(v) != -1)
-			return v->vno;
-		disablemsi(v);
-		if((devno = pcicfgr8(p, PciINTP)) == 0){
-			print("ioapicintrenable: no intp for %T\n", v->tbdf);
-			return -1;
-		}
-		return ioapicrdt(v, BUSBNO(v->tbdf), BUSDNO(v->tbdf)<<2|(devno-1), BusPCI);
-	}
-	else{
-		panic("unknown tbdf %T", v->tbdf);
+	/*
+	 * Legacy ISA.
+	 * Make a busno and devno using the
+	 * ISA bus number and the irq.
+	 */
+	if(isabusno == -1){
+		iprint("ioapicintrenable: no ISA bus allocated\n");
 		return -1;
 	}
+	return ioapicrdt(v, isabusno, v->irq, BusISA);
 }
 
 int
-ioapicintrdisable(int vecno)
+intrenableioapicpci(Vctl* v)
 {
-	Rdt *rdt;
+	int devno;
 
-	if((rdt = rdtvecno[vecno]) == nil){
-		panic("ioapicintrdisable: vecno %d has no rdt", vecno);
+	if((devno = pcicfgr8(v->p, PciINTP)) == 0){
+		iprint("ioapicintrenable: no intp for %T\n", v->tbdf);
 		return -1;
 	}
-
-	lock(rdt->apic);
-	if(adec(&rdt->enabled) == 0)
-		rtblput(rdt->apic, rdt->intin, 0, rdt->lo);
-	unlock(rdt->apic);
-
-	return 0;
+	return ioapicrdt(v, BUSBNO(v->tbdf), BUSDNO(v->tbdf)<<2|(devno-1), BusPCI);
 }
--- /sys/src/nix/k10/io.h	Fri Sep 18 15:04:19 2015
+++ /sys/src/nix/k10/io.h	Fri Sep 18 15:04:20 2015
@@ -1,10 +1,13 @@
 typedef	struct	Hwconf	Hwconf;
+typedef	struct	Msix	Msix;
 typedef	struct	Pciconf	Pciconf;
 typedef	struct	Pcidev	Pcidev;
 typedef	struct	Vctl	Vctl;
 typedef	struct	Vkey	Vkey;
 typedef	struct	Vtime	Vtime;
 
+#pragma incomplete Msix
+
 enum {
 	VectorNMI	= 2,		/* non-maskable interrupt */
 	VectorBPT	= 3,		/* breakpoint */
@@ -70,9 +73,6 @@
 	Vmsix		= 1<<0,			/* allow msi-x allocation (unfortunately this changes handlers */
 };
 
-typedef	struct	Msix	Msix;
-#pragma incomplete Msix
-
 struct Vkey {
 	int	tbdf;			/* pci: ioapic or msi sources */
 	Pcidev*	p;			/* pci: ioapic or msi(-x) sources */
@@ -101,6 +101,7 @@
 	int	(*isr)(int);		/* get isr bit for this irq */
 	int	(*eoi)(int);		/* eoi */
 	int	(*mask)(Vkey*, int);	/* interrupt enable returns masked vector */
+	int	(*disable)(Vctl*);		/* disable */
 	int	vno;
 
 	Vtime;
@@ -273,6 +274,11 @@
 void	pcisetmwi(Pcidev*);
 int	pcisetpms(Pcidev*, int);
 int	strtotbdf(char*, char**, int);
+
+int	intrenablemsix(Vctl*);
+int	intrenablemsi(Vctl*);
+int	intrenableioapicpci(Vctl*);
+int	intrenableioapicisa(Vctl*);
 
 #define PCIWINDOW	0
 #define PCIWADDR(va)	(PADDR(va)+PCIWINDOW)
--- /sys/src/nix/k10/trap.c	Fri Sep 18 15:04:22 2015
+++ /sys/src/nix/k10/trap.c	Fri Sep 18 15:04:23 2015
@@ -32,6 +32,45 @@
 	v->affinity = -1;
 }
 
+static int
+intrenablelapic(Vctl* v)
+{
+	if(v->irq != IrqSPURIOUS)
+		v->isr = lapiceoi;
+	v->type = "lapic";
+	return v->irq;
+}
+
+static int
+physintrenable(Vctl* v)
+{
+	Pcidev *p;
+
+	if(v->tbdf == BUSUNKNOWN){
+		/* gross hack ... move out to call site */
+		if(v->irq >= IrqLINT0 && v->irq <= MaxIrqLAPIC)
+			return intrenablelapic(v);
+		else
+			return intrenableioapicisa(v);
+	}
+	else if(BUSTYPE(v->tbdf) == BusPCI){
+		if((p = pcimatchtbdf(v->tbdf)) == nil){
+			iprint("ioapic: no pci dev for tbdf %T", v->tbdf);
+			return -1;
+		}
+		v->p = p;
+		if(intrenablemsix(v) != -1)
+			return v->vno;
+		if(intrenablemsi(v) != -1)
+			return v->vno;
+		return intrenableioapicpci(v);
+	}
+	else{
+		iprint("ioapicintrenable: unknown tbdf %T\n", v->tbdf);
+		return -1;
+	}
+}
+
 void*
 vintrenable(Vctl *v, char *name)
 {
@@ -45,7 +84,7 @@
 	v->name[KNAMELEN-1] = 0;
 
 	ilock(&vctllock);
-	vno = ioapicintrenable(v);
+	vno = physintrenable(v);
 	if(vno == -1){
 		iunlock(&vctllock);
 		print("vintrenable: %s: couldn't enable irq %d, %T\n",
@@ -126,12 +165,12 @@
 			break;
 	if(x != v)
 		panic("intrdisable: v %#p", v);
-	if(v->mask)
+	if(v->mask != nil)
 		v->mask(v, 1);
 	v->f(nil, v->a);
 	*ll = v->next;
-	if(strcmp(v->type, "ioapic") == 0)
-		ioapicintrdisable(v->vno);
+	if(v->disable != nil)
+		v->disable(v);
 	else
 		iprint("intrdisable: not disabling %s type %s\n", v->name, v->type);
 	iunlock(&vctllock);
@@ -358,7 +397,7 @@
 		postnote(up, 1, buf, NDebug);
 	}
 	else if(vno >= VectorPIC && vno != VectorSYSCALL){
-		static ulong last;
+		static Tick last;
 		static uvlong count[MACHMAX];
 
 		/* spurious interrupt. */