allow multiple msi-x vectors (untested), and clean up interface

Reference: /n/atom/patch/applied/msix-multivec
Date: Sat Sep  5 21:52:23 CES 2015
Signed-off-by: quanstro@quanstro.net

--- /sys/src/nix/k10/apic.h	Sat Sep  5 21:51:49 2015
+++ /sys/src/nix/k10/apic.h	Sat Sep  5 21:51:51 2015
@@ -100,7 +100,7 @@
 void	lapicsetcolor(int, int);
 void	lapicsipi(int, uintmem);
 
-int	pcimsienable(Pcidev*, uvlong);
-int	pcimsimask(Pcidev*, int);
-int	pcimsixenable(Pcidev*, uvlong);
-int	pcimsixmask(Pcidev*, int);
+int	pcimsienable(Vkey*, uvlong);
+int	pcimsimask(Vkey*, int);
+int	pcimsixenable(Vkey*, uvlong);
+int	pcimsixmask(Vkey*, int);
--- /sys/src/nix/k10/lapic.c	Sat Sep  5 21:51:53 2015
+++ /sys/src/nix/k10/lapic.c	Sat Sep  5 21:51:54 2015
@@ -4,8 +4,8 @@
 #include "dat.h"
 #include "fns.h"
 
-#include "apic.h"
 #include "io.h"
+#include "apic.h"
 #include "adr.h"
 
 enum {						/* Local APIC registers */
--- /sys/src/nix/k10/msi.c	Sat Sep  5 21:51:56 2015
+++ /sys/src/nix/k10/msi.c	Sat Sep  5 21:51:57 2015
@@ -60,14 +60,17 @@
 }
 
 int
-pcimsienable(Pcidev *p, uvlong vec)
+pcimsienable(Vkey *v, uvlong vec)
 {
 	char *s;
 	uint c, f, d, datao, lopri, dmode, logical;
+	Pcidev *p;
 
 	if(sys->nomsi)
 		return -1;
 
+	if((p = v->p) == nil)
+		return -1;
 	c = msicap(p);
 	if(c == 0)
 		return -1;
@@ -101,10 +104,13 @@
 }
 
 int
-pcimsimask(Pcidev *p, int mask)
+pcimsimask(Vkey *v, int mask)
 {
 	uint c, f;
+	Pcidev *p;
 
+	if((p = v->p) == nil)
+		return -1;
 	c = msicap(p);
 	if(c == 0)
 		return -1;
--- /sys/src/nix/k10/ioapic.c	Sat Sep  5 21:51:59 2015
+++ /sys/src/nix/k10/ioapic.c	Sat Sep  5 21:52:00 2015
@@ -4,8 +4,8 @@
 #include "dat.h"
 #include "fns.h"
 
-#include "apic.h"
 #include "io.h"
+#include "apic.h"
 #include "adr.h"
 
 typedef struct Rbus Rbus;
@@ -432,9 +432,7 @@
 static int
 msimask(Vkey *v, int mask)
 {
-	if(v->p == nil)
-		return -1;
-	return pcimsimask(v->p, mask);
+	return pcimsimask(v, mask);
 }
 
 static int
@@ -447,7 +445,7 @@
 	ioapicphysdd(v, &hi, &lo);
 
 	msivec = (u64int)hi<<32 | lo;
-	if(pcimsienable(v->p, msivec) == -1)
+	if(pcimsienable(v, msivec) == -1)
 		return -1;
 	v->eoi = lapiceoi;
 	v->type = "msi";
@@ -460,15 +458,13 @@
 int
 disablemsi(Vctl *v)
 {
-	if(v->p == nil)
-		return -1;
-	return pcimsimask(v->p, 1);
+	return pcimsimask(v, 1);
 }
 
 static int
 msixmask(Vkey *v, int mask)
 {
-	return pcimsixmask(v->p, mask);
+	return pcimsixmask(v, mask);
 }
 
 static int
@@ -483,7 +479,7 @@
 
 	ioapicphysdd(v, &hi, &lo);
 	msivec = (u64int)hi<<32 | lo;
-	if(pcimsixenable(v->p, msivec) == -1)
+	if(pcimsixenable(v, msivec) == -1)
 		return -1;
 	v->eoi = lapiceoi;
 	v->type = "msi-x";
@@ -499,7 +495,7 @@
 {
 	if((v->flag&Vmsix) == 0)
 		return -1;
-	return pcimsixmask(v->p, 1);
+	return pcimsixmask(v, 1);
 }
 
 int
--- /sys/src/nix/k10/sipi.c	Sat Sep  5 21:52:02 2015
+++ /sys/src/nix/k10/sipi.c	Sat Sep  5 21:52:03 2015
@@ -4,6 +4,7 @@
 #include "dat.h"
 #include "fns.h"
 
+#include "io.h"
 #include "apic.h"
 #include "sipi.h"
 
--- /sys/src/nix/k10/msix.c	Sat Sep  5 21:52:05 2015
+++ /sys/src/nix/k10/msix.c	Sat Sep  5 21:52:06 2015
@@ -70,16 +70,6 @@
 
 #define dprint(flag, ...) do{if(flag&Dflags){print(__VA_ARGS__);}}while(0)
 
-
-/*
- * first experiment.  not usable yet.
- * - need locking
- * - Msix structure should be passed in from driver
- * - no support for multiple vectors yet.
- */
-static Msix msixtab[5];
-static int nmsixtab;
-
 static void*
 birmap(Pcidev *p, u32int u, u32int *off)
 {
@@ -128,11 +118,7 @@
 	for(i = 0; i < x.nvec; i++)
 		x.tbl[4*i + Vecctl] |= Vecmask;
 
-	if(nmsixtab == nelem(msixtab)){
-		print("msix: not enough table entries");
-		return nil;
-	}
-	t = msixtab + nmsixtab++;
+	t = malloc(sizeof x);
 	*t = x;
 
 	return t;
@@ -141,13 +127,22 @@
 static Msix*
 getmsix(Pcidev *p)
 {
-	int i;
+	Msix *x;
+	static Lock l;
+
+	if(p->msix != nil)
+		return p->msix;
+
+	lock(&l);
+	if(p->msix != nil){
+		unlock(&l);
+		return p->msix;
+	}
+	x = msixinit(p);
+	unlock(&l);
 
-	for(i = 0; i < nmsixtab; i++)
-		if(msixtab[i].p == p)
-			return msixtab+i;
 	dprint(Debug, "msixinit %T from %#p\n", p->tbdf, getcallerpc(&p));
-	return msixinit(p);
+	return x;
 }
 
 static int
@@ -160,14 +155,17 @@
 }
 
 int
-pcimsixenable(Pcidev *p, uvlong vec)
+pcimsixenable(Vkey *v, uvlong vec)
 {
 	char *s;
 	u32int *tbl, lopri, logical, d, dmode, o, c;
 	Msix *x;
+	Pcidev *p;
 
 	if((s = getconf("*msix")) == nil || atoi(s) <= 0)
 		sys->nomsix = 1;
+	if((p = v->p) == nil)
+		return -1;
 	if(blacklist(p) == -1)
 		return -1;
 	if((x = getmsix(p)) == nil)
@@ -179,7 +177,7 @@
 	lopri = (vec & 0x700) == MTlp;
 	logical = (vec & Lm) != 0;
 	dmode = (vec >> 8) & 7;
-	o = 0;	/* wrong */
+	o = v->idx*4;
 	tbl[o + Msglo] = Msiabase | Msiadest * d | Msialowpri * lopri | Msialogical * logical;
 	tbl[o + Msghi] = 0;
 	tbl[o + Msgdata] = Msidassert | Msidlogical * logical| Msidmode * dmode | (uchar)vec;
@@ -196,20 +194,24 @@
 }
 
 int
-pcimsixmask(Pcidev *p, int mask)
+pcimsixmask(Vkey *v, int mask)
 {
-	u32int *tbl;
+	u32int *tbl, o;
 	Msix *x;
+	Pcidev *p;
 
+	if((p = v->p) == nil)
+		return -1;
 	if((x = getmsix(p)) == nil)
 		return -1;
 	tbl = x->tbl;
 	if(tbl == nil)
 		return -1;
+	o = v->idx*4;
 	if(mask)
-		tbl[Vecctl] |= Vecmask;
+		tbl[o + Vecctl] |= Vecmask;
 	else
-		tbl[Vecctl] &= ~Vecmask;
+		tbl[o + Vecctl] &= ~Vecmask;
 	return 0;
 }