msi-x: initial msi-x support the pci standard allows devices to support either msi or msi-x. a number of devices tend to support only msi-x. this patch is just enough to support a single msi-x vector per device. the driver must allocate its vector by hand and set Vmsix in the flags. this is due to concern about drivers changing behavior with msi-x vectors enable. (and may be revisted later.) this patch is designed for clarity. cleanup of the msi/msix interfaces is planned. Reference: /n/atom/patch/applied/msix Date: Sat Sep 5 20:56:14 CES 2015 Signed-off-by: quanstro@quanstro.net --- /sys/src/nix/k10/msix.c Thu Jan 1 00:00:00 1970 +++ /sys/src/nix/k10/msix.c Sat Sep 5 20:41:06 2015 @@ -0,0 +1,248 @@ +#include "u.h" +#include "../port/lib.h" +#include "mem.h" +#include "dat.h" +#include "fns.h" +#include "io.h" +#include "apic.h" + +typedef struct Msix Msix; +struct Msix { + Pcidev *p; + uchar *tblmap; + uchar *pbamap; + u32int *tbl; + u64int *pba; + + uint c; /* offset of cap in pci hdr */ + uint nvec; + + int enable; /* ainc'd count of enables */ +}; + +enum { + /* address */ + Msiabase = 0xfee00000u, + Msiadest = 1<<12, /* same as 63:56 of apic vector */ + Msiaedest = 1<<4, /* same as 55:48 of apic vector */ + Msialowpri = 1<<3, /* redirection hint */ + Msialogical = 1<<2, + + /* data */ + Msidlevel = 1<<15, + Msidassert = 1<<14, + Msidlogical = 1<<11, + Msidmode = 1<<8, /* 3 bits; delivery mode */ + Msidvector = 0xff<<0, +}; + +enum { + /* ptrs into cap structure (bytes in pci hdr) */ + Msgctl = 2, /* 16 bits */ + Tbloff = 4, /* qword aligned | bir */ + Pbaoff = 8, /* qword aligned | bir */ + + /* ptrs into tbl structure (so 4 dwords walk into a bar...) */ + Msglo = 0, + Msghi = 1, + Msgdata = 2, + Vecctl = 3, + Tblsz = 4, + + /* ptrs into pba structure */ + /* ... just a bucket 'o bits in qword hunks */ +}; + +enum { + /* Msgctl */ + Msixen = 1<<15, + Fnmask = 1<<14, + Vectblsz = (1<<11)-1, + + /* Vecctl */ + Vecmask = 1<<0, +}; + +enum { + Debug = 1, + Dflags = Debug, +}; + +#define dprint(flag, ...) do{if(flag&Dflags){print(__VA_ARGS__);}}while(0) + + +/* + * first experiment. not usable yet. + * - need locking + * - Msix structure should be passed in from driver + * - no support for multiple vectors yet. + */ +static Msix msixtab[5]; +static int nmsixtab; + +static void* +birmap(Pcidev *p, u32int u, u32int *off) +{ + u32int bir, sz; + uintmem io; + + bir = u&7; + if(bir > 5) + return nil; + *off = u&~7; + sz = p->mem[bir].size; + if(*off + 16 > sz){ + print("%T: offset too big %d>%d\n", p->tbdf, *off + 16, sz); + return nil; + } + io = p->mem[bir].bar & ~(uintmem)0xf; + dprint(Debug, "vmapoverlap(%#P, %#ux)\n", io, sz); + return vmapoverlap(io, sz); +} + +static Msix* +msixinit(Pcidev *p) +{ + u32int i, off; + Msix x, *t; + + memset(&x, 0, sizeof x); + x.p = p; + x.c = pcicap(p, PciCapMSIX); + if(x.c == -1) + return nil; + x.nvec = (pcicfgr16(p, x.c+Msgctl)&Vectblsz)+1; + + x.tblmap = birmap(p, pcicfgr32(p, x.c + Tbloff), &off); + if(x.tblmap == nil) + return nil; + x.tbl = (u32int*)(x.tblmap + off); + dprint(Debug, "%T: msixinit: x.tbl %.8ux %.8ux %.8ux %.8ux \n", p->tbdf, + x.tbl[0], x.tbl[1], x.tbl[2], x.tbl[3]); + + x.pbamap = birmap(p, pcicfgr32(p, x.c + Pbaoff), &off); + if(x.pbamap == nil) + return nil; + x.pba = (u64int*)(x.pbamap + off); + + for(i = 0; i < x.nvec; i++) + x.tbl[4*i + Vecctl] |= Vecmask; + + if(nmsixtab == nelem(msixtab)){ + print("msix: not enough table entries"); + return nil; + } + t = msixtab + nmsixtab++; + *t = x; + + return t; +} + +static Msix* +getmsix(Pcidev *p) +{ + int i; + + for(i = 0; i < nmsixtab; i++) + if(msixtab[i].p == p) + return msixtab+i; + dprint(Debug, "msixinit %T from %#p\n", p->tbdf, getcallerpc(&p)); + return msixinit(p); +} + +static int +blacklist(Pcidev *p) +{ + switch(p->vid<<16 | p->did){ + default: + return 0; + } +} + +int +pcimsixenable(Pcidev *p, uvlong vec) +{ + char *s; + u32int *tbl, lopri, logical, d, dmode, o, c; + Msix *x; + + if((s = getconf("*msix")) == nil || atoi(s) <= 0) + sys->nomsix = 1; + if(blacklist(p) == -1) + return -1; + if((x = getmsix(p)) == nil) + return -1; + tbl = x->tbl; + + d = vec>>56; + + lopri = (vec & 0x700) == MTlp; + logical = (vec & Lm) != 0; + dmode = (vec >> 8) & 7; + o = 0; /* wrong */ + tbl[o + Msglo] = Msiabase | Msiadest * d | Msialowpri * lopri | Msialogical * logical; + tbl[o + Msghi] = 0; + tbl[o + Msgdata] = Msidassert | Msidlogical * logical| Msidmode * dmode | (uchar)vec; + tbl[o + Vecctl] |= Vecmask; + sfence(); + c = x->c+Msgctl; + if(ainc(&x->enable) == 1){ + o = pcicfgr16(p, c) | Msixen; + pcicfgw16(p, c, o & ~Fnmask); + } + if(sys->nomsix) + return -1; + return 0; +} + +int +pcimsixmask(Pcidev *p, int mask) +{ + u32int *tbl; + Msix *x; + + if((x = getmsix(p)) == nil) + return -1; + tbl = x->tbl; + if(tbl == nil) + return -1; + if(mask) + tbl[Vecctl] |= Vecmask; + else + tbl[Vecctl] &= ~Vecmask; + return 0; +} + +int +pcimsixdisable(Pcidev *p) +{ + u32int *tbl, i, c; + Msix *x; + + if((x = getmsix(p)) == nil) + return -1; + tbl = x->tbl; + if(tbl == nil) + return -1; + + /* wrong: need a scheme to allocate bits */ + for(i = 0; i < x->nvec; i++) + x->tbl[4*i + Vecctl] |= Vecmask; + + c = x->c+Msgctl; + if(adec(&x->enable) == 0) + pcicfgw16(p, c, pcicfgr16(p, c) & ~(Msixen|Fnmask)); + return 0; + +} + +/* debug crap */ +void +msixpba(Pcidev *p) +{ + Msix *x; + + if((x = getmsix(p)) == nil) + return; + print("%T: pba %.16llux\n", x->p->tbdf, x->pba[0]); +} --- /sys/src/nix/k10/io.h Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/io.h Sat Sep 5 20:41:06 2015 @@ -66,8 +66,17 @@ IdtMAX = 255, }; +enum { + Vmsix = 1<<0, /* allow msi-x allocation (unfortunately this changes handlers */ +}; + +typedef struct Msix Msix; +#pragma incomplete Msix + struct Vkey { int tbdf; /* pci: ioapic or msi sources */ + Pcidev* p; /* pci: ioapic or msi(-x) sources */ + int idx; /* msi-x: interrupt index */ int irq; /* 8259-emulating sources */ }; @@ -79,7 +88,8 @@ struct Vctl { Vctl* next; /* handlers on this vector */ - int isintr; /* interrupt or fault/trap */ + uchar isintr; /* interrupt or fault/trap */ + uchar flag; int affinity; /* processor affinity (-1 for none) */ Vkey; /* source-specific key; tbdf for pci */ @@ -220,6 +230,8 @@ Pcidev* list; Pcidev* link; /* next device on this bno */ Pcidev* bridge; /* down a bus */ + + Msix* msix; /* pci: msi-x */ }; struct Pciconf { --- /sys/src/nix/k10/ioapic.c Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/ioapic.c Sat Sep 5 20:41:06 2015 @@ -468,6 +468,48 @@ return pcimsimask(p, 1); } +static int +msixmask(Vkey *v, int mask) +{ + Pcidev *p; + + p = pcimatchtbdf(v->tbdf); + if(p == nil) + return -1; + return pcimsixmask(p, mask); +} + +static int +intrenablemsix(Vctl* v, Pcidev *p) +{ + u32int lo, hi; + u64int msivec; + + if((v->flag&Vmsix) == 0 || p == nil) + return -1; + lo = IPlow | TMedge; + + ioapicphysdd(v, &hi, &lo); + msivec = (u64int)hi<<32 | lo; + if(pcimsixenable(p, msivec) == -1) + return -1; + v->eoi = lapiceoi; + v->type = "msi-x"; + v->mask = msixmask; + + print("msix: %T: enabling %.16llux %s vno %d affinity %d\n", + p->tbdf, msivec, v->name, v->vno, v->affinity); + return v->vno; +} + +int +disablemsix(Vctl *v, Pcidev *p) +{ + if((v->flag&Vmsix) == 0 || p == nil) + return -1; + return pcimsixmask(p, 1); +} + int ioapicrdt(Vctl *v, int busno, int devno, int bustype) { @@ -565,6 +607,10 @@ /* BOTCH msi is in the wrong place. it has nothing to do with ioapics */ if((p = pcimatchtbdf(v->tbdf)) == nil) panic("ioapic: no pci dev for tbdf %T", v->tbdf); + v->p = p; + if(intrenablemsix(v, p) != -1) + return v->vno; + disablemsix(v, p); if(intrenablemsi(v, p) != -1) return v->vno; disablemsi(v, p); --- /sys/src/nix/k10/apic.h Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/apic.h Sat Sep 5 20:41:06 2015 @@ -89,6 +89,7 @@ void ioapicintrinit(int, int, int, int, int, u32int); Apic* ioapiclookup(uint); void ioapiconline(void); +int ioapicphysdd(Vctl*, u32int*, u32int*); void lapicdump(void); int lapiceoi(int); void lapicinit(int, uintmem, int); @@ -101,3 +102,5 @@ int pcimsienable(Pcidev*, uvlong); int pcimsimask(Pcidev*, int); +int pcimsixenable(Pcidev*, uvlong); +int pcimsixmask(Pcidev*, int); --- /sys/src/nix/k10/cpu Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/cpu Sat Sep 5 20:41:06 2015 @@ -74,8 +74,8 @@ sdloop misc +dev -# mp mpacpi lapic ioapic msi pci sipi - acpi acpiio lapic ioapic msi pci sipi +# mp mpacpi lapic ioapic msi msix pci sipi + acpi acpiio lapic ioapic msi msix pci sipi nodraw boot cpu --- /sys/src/nix/k10/cpud Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/cpud Sat Sep 5 20:41:06 2015 @@ -73,8 +73,8 @@ sdloop misc +dev -# mp mpacpi lapic ioapic msi pci sipi - acpi acpiio lapic ioapic msi pci sipi +# mp mpacpi lapic ioapic msi msix pci sipi + acpi acpiio lapic ioapic msi msix pci sipi nodraw boot cpu --- /sys/src/nix/k10/cpuf Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/cpuf Sat Sep 5 20:41:06 2015 @@ -74,8 +74,8 @@ sdloop misc +dev -# mp mpacpi lapic ioapic msi pci sipi - acpi acpiio lapic ioapic msi pci sipi +# mp mpacpi lapic ioapic msi msix pci sipi + acpi acpiio lapic ioapic msi msix pci sipi nodraw boot cpu --- /sys/src/nix/k10/term Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/term Sat Sep 5 20:41:06 2015 @@ -86,8 +86,8 @@ sdloop misc +dev -# mp mpacpi lapic ioapic msi pci sipi - acpi acpiio lapic ioapic msi pci sipi +# mp mpacpi lapic ioapic msi msix pci sipi + acpi acpiio lapic ioapic msi msix pci sipi boot terminal il --- /sys/src/nix/k10/termd Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/termd Sat Sep 5 20:41:06 2015 @@ -84,8 +84,8 @@ sdloop misc +dev -# mp mpacpi lapic ioapic msi pci sipi - acpi acpiio lapic ioapic msi pci sipi +# mp mpacpi lapic ioapic msi msix pci sipi + acpi acpiio lapic ioapic msi msix pci sipi boot terminal il --- /sys/src/nix/k10/termf Sat Sep 5 20:41:06 2015 +++ /sys/src/nix/k10/termf Sat Sep 5 20:41:06 2015 @@ -84,8 +84,8 @@ sdloop misc +dev -# mp mpacpi lapic ioapic msi pci sipi - acpi acpiio lapic ioapic msi pci sipi +# mp mpacpi lapic ioapic msi msix pci sipi + acpi acpiio lapic ioapic msi msix pci sipi boot terminal il