add enough enclosure support to deal with simple expander topologies. Reference: /n/atom/patch/applied2013/sdmpt2enc Date: Wed Oct 23 18:39:49 CES 2013 Signed-off-by: quanstro@quanstro.net --- /sys/src/nix/k10/sdmpt2.c Wed Oct 23 18:39:18 2013 +++ /sys/src/nix/k10/sdmpt2.c Wed Oct 23 18:39:18 2013 @@ -106,7 +106,9 @@ EventAck = 0x08, ScsiIoRequest = 0x00, ScsiTaskManagement = 0x01, + ScsiEnclosureProcessor = 0x18, SasIoUnitControl = 0x1b, + SataPassthrough = 0x1c, Config = 0x04, }; @@ -139,7 +141,9 @@ }; enum { - Nunit = 16, + Nenclosure = 1, + Nunitperenc = 20, + Nunit = Nenclosure * Nunitperenc, /* should limit to 16 but i have more */ Offline = 0, Online, @@ -152,7 +156,11 @@ uchar link; u32int info; u16int flags; - uchar wwn[8]; + u64int wwn; + + int slot; + uint exph; + uint ench; }; typedef struct Req Req; @@ -272,6 +280,13 @@ */ switch(reply[0]>>24){ /* Function */ case ScsiIoRequest: + case SataPassthrough: + switch(status){ + case 0x0045: /* SCSI_DATA_UNDERRUN */ + case 0x0003: /* INVALID_SGL (user command with r/w bits mixed up) */ + return 0; + } + break; case ScsiTaskManagement: switch(status){ case 0x0040: /* SCSI_RECOVERED_ERROR */ @@ -301,7 +316,8 @@ } break; } - panic("ckstatus: %s: bad status 0x%04ux", ctlr->name, status); + panic("ckstatus: %s: fn %.2ux, bad status %#.04ux", + ctlr->name, reply[0]>>24, status); return -1; /* not reached */ } @@ -597,16 +613,23 @@ buf[6] = ctlr->reqfsz<<16; /* SystemRequestFrameSize */ buf[7] = ctlr->replyfreeq<<16; /* ReplyFreeQueueDepth */ buf[7] |= ctlr->replypostq; /* ReplyDescriptorPostQueueDepth */ - buf[10] = PCIWADDR(ctlr->req); /* SystemRequestFrameBaseAddress */ - buf[12] = PCIWADDR(ctlr->replypost); /* ReplyDescriptorPostQueueAddress */ - buf[14] = PCIWADDR(ctlr->replyfree); /* ReplyFreeQueueAddress */ + buf[10] = Pciwaddrl(ctlr->req); /* SystemRequestFrameBaseAddress */ + buf[11] = Pciwaddrh(ctlr->req); + buf[12] = Pciwaddrl(ctlr->replypost); /* ReplyDescriptorPostQueueAddress */ + buf[13] = Pciwaddrh(ctlr->replypost); + buf[14] = Pciwaddrl(ctlr->replyfree); /* ReplyFreeQueueAddress */ + buf[15] = Pciwaddrh(ctlr->replyfree); doorbell(ctlr, buf, 18); ckstatus(ctlr, buf); ckstate(ctlr, StateOperational); } -#define EVENT(x) (1U<<((x) % 32)) +static void +event(u32int *buf, u32int event) +{ + buf[event/32] &= ~(1 << event%32); +} static void eventnotify(Ctlr *ctlr) @@ -623,11 +646,9 @@ * host mapping, however the SAS_TOPOLOGY_CHANGE_LIST * event is merely sufficient. */ - buf[5] = ~EVENT(SasTopologyChangeList); - buf[6] = ~0; - buf[7] = ~0; - buf[8] = ~0; - buf[9] = ~0; + memset(buf+5, 0xff, 4*4); + event(buf+5, SasTopologyChangeList); + event(buf+5, SasEnclDeviceStatusChange); doorbell(ctlr, buf, 11); ckstatus(ctlr, buf); @@ -677,10 +698,10 @@ freereq(ctlr, r); } -static void -unitconfig(Ctlr *ctlr, Unit *u) +static int +devpage(Ctlr *ctlr, uint devh, u32int page[14]) { - u32int buf[7+2], page[14]; + u32int buf[7+2]; /* * Unit configuration is pulled from SAS Device Page 0. @@ -701,16 +722,28 @@ ckstatus(ctlr, buf); buf[0] |= 0x01; /* Action (READ_CURRENT) */ - buf[6] = 0x2<<28 | u->devh; /* PageAddress (HANDLE) */ - mksge32(buf+7, page, sizeof page, 0); /* PageBufferSGE */ + buf[6] = 0x2<<28 | devh; /* PageAddress (HANDLE) */ + mksge32(buf+7, page, 14*4, 0); /* PageBufferSGE */ doorbell(ctlr, buf, 7+2); if(!ckstatus(ctlr, buf)) - error(Ebadunit); + return -1; + return 0; +} + +static void +unitconfig(Ctlr *ctlr, Unit *u, uint devh) +{ + u32int page[14]; + if(devpage(ctlr, devh, page) == -1) + error(Ebadunit); + u->devh = devh; u->info = page[7]; /* DeviceInfo */ u->flags = page[8] & 0xffff; /* Flags */ - memmove(u->wwn, page+9, 8); /* DeviceName */ + u->wwn = getle((uchar*)(page+9), 8); + u->slot = page[2] & 0xffff; +// print("parent devh %ux phyno %d physport %d\n", page[5]&0xffff, page[5]>>16 & 0xff, page[8]>>24&0xff); } static void @@ -949,23 +982,70 @@ wakeup(r); } +/* + * temporary hack + */ +static int +port2unit(Ctlr *ctlr, uint devh, uint port, uint exph, uint ench) +{ + int unitidx, encidx; + u32int slot, page[14]; + + /* hack: ignore direct ports if controler has 1 port */ + if(ctlr->numports == 1) + encidx = ench - 2; + else + encidx = ench - 1; + if(encidx < 0 || encidx >= Nenclosure) + return -1; + + /* direct? */ + if(exph == 0 && ench == 1) + unitidx = port; + else{ + if(devpage(ctlr, devh, page) == -1) + return -1; + slot = page[2] & 0xffff; + if(slot >= Nunitperenc) + return -1; /* too many drives per enclosure */ + unitidx = encidx*Nunitperenc + slot; + } + + if(unitidx >= Nunit) + return -1; + + return unitidx; +} + +/* sleeze; try to detect when no direct connect */ +static int +anyconnected(u32int *data, int e) +{ + int i; + u32int *p, reason; + + for(i = 0; i < e; i++){ + p = data + i; + reason = *p>>24 & 0xf; + if(reason == 1 || reason == 2) + return 1; + } + return 0; +} + static void topoevent(Ctlr *ctlr, u32int *data) { - u32int *p, *e; - int i; + u32int *p; + int i, b, e, rate, reason, devh, unitno, exph, ench, num, start; Unit *u; - /* - * Unfortunately, devsd limits us to 16 devices, which - * essentially precludes support for SAS expanders and/or - * enclosures. A one-to-one mapping exists between a - * direct attached PHY and a device for simplicity. - */ - if(data[0]>>16 != 0x0) /* ExpanderDevHandle */ - return; - if((data[0] & 0xffff) != 0x1) /* EnclosureHandle */ - return; + exph = data[0] >> 16; + ench = data[0] & 0xffff; + start = data[2]>>8 & 0xff; + num = data[2]>>0 & 0xff; + print("%s: topoevent exp/enc %d/%d port %d-%d #phys %d\n", ctlr->name, + exph, ench, start, start+num, data[1]); /* * SAS topology changes are handled in two phases; we @@ -973,25 +1053,72 @@ * New units require additional configuration information * and missing units must have resources released. */ - p = data + 3; - e = p + (data[2] & 0xff); /* NumEntries */ - i = data[2]>>8 & 0xff; /* StartPhyNum */ - for(; p < e && i < Nunit; ++p, ++i){ - u = UNIT(ctlr, i); - switch(*p>>24 & 0xf){ /* PhyStatus (Reason Code) */ - case 0x1: - u->devh = *p & 0xffff; /* AttachedDevHandle */ - unitconfig(ctlr, u); + b = data[2]>>8 & 0xff; + e = data[2] & 0xff; /* NumEntries */ + + if(exph == 0 && ench == 1) + if(!anyconnected(data + 3, e)){ + print(" none connected\n"); + return; + } + + for(i = 0; i < e; i++){ + p = data + 3 + i; + rate = p[0]>>16 & 0xff; + devh = p[0] & 0xffff; + reason = *p>>24 & 0xf; + if(devh == 1 || devh == 0) + continue; /* ignore self; empty */ + unitno = port2unit(ctlr, devh, i, exph, ench); + if(unitno == -1 || unitno >= ctlr->sdev->nunit){ + print(" entry %d exph %d ench %d\n", i, exph, ench); + continue; + } + u = UNIT(ctlr, unitno); + switch(reason){ /* PhyStatus (Reason Code) */ + case 1: /* online */ + unitconfig(ctlr, u, devh); + u->exph = exph; + u->ench = ench; + print("unit %d slot %d:%d devh %.4ux rate %x stat %x online wwn %llux\n", + i+b, ench, u->slot, devh, rate, *p>>28, u->wwn); u->state = Online; break; - case 0x2: + case 2: /* offline */ + print("unit %d devh %.4ux rate %x offline\n", i+b, devh, rate); u->state = Offline; unitreset(ctlr, u); unitremove(ctlr, u); break; + default: + print("unit %d devh %.4ux rate %x ?? = %d\n", i+b, devh, rate, rate); + break; } - u->link = *p>>16 & 0xff; /* LinkRate */ + u->link = rate; /* LinkRate */ + } +} + +static void +encevent(Ctlr *, u32int *data) +{ + char *reason; + int s, n; + + switch(data[0]>>16 & 0xff){ + default: + reason = "*gok*"; + break; + case 1: + reason = "added"; + break; + case 2: + reason = "gone"; + break; } + s = data[3] >> 16; + n = data[3] & 0xffff; + print("mpt2: enc %d %s: port %d wwn %llux slots %d-%d phy %b\n", data[0]>>0 & 0xffff, reason, + data[0]>>24, (uvlong)data[2]<<32 | data[1], s, s+n, data[4]); } static void @@ -1008,8 +1135,12 @@ case SasTopologyChangeList: topoevent(ctlr, data); break; + case SasEnclDeviceStatusChange: + encevent(ctlr, data); + break; default: - panic("doevent: %s: bad event 0x%04ux", ctlr->name, event); + panic("sdmpt2: doevent: %s: bad event %#.4ux", ctlr->name, event); + break; } if(reply[1]>>16 & 0xff) /* AckRequired */ eventack(ctlr, event, context); @@ -1031,6 +1162,29 @@ } static void +satareply(Ctlr *ctlr, u16int smid, u32int *reply) +{ + Req *r; + SDreq *sr; + + r = REQ(ctlr, smid); + sr = r->sdreq; + if(ckstatus(ctlr, reply) == 1){ + if ((reply[3]>>8 & 0xff) == 0x00) /* SASStatus (SUCCESS) */ + sr->status = SDok; + else + sr->status = SDretry; + memmove(sr->data, reply+5, W2B(5)); /* StatusFIS */ + sr->rlen = W2B(5); + sr->dlen = reply[11]; /* TransferCount */ + } else + sr->status = SDretry; + + r->done++; + wakeup(r); +} + +static void addressreply(Ctlr *ctlr, u16int smid, u32int *reply) { uchar fn; @@ -1042,6 +1196,9 @@ case SasIoUnitControl: doreply(ctlr, smid, reply); break; + case SataPassthrough: + satareply(ctlr, smid, reply); + break; case ScsiIoRequest: /* * Address Replies to SCSI IO Requests always @@ -1058,7 +1215,7 @@ qevent(ctlr, reply); break; default: - panic("addressreply: %s: bad function 0x%02ux", ctlr->name, fn); + panic("addressreply: %s: bad function %#.02ux", ctlr->name, fn); } /* @@ -1211,18 +1368,38 @@ iocwrite(ctlr, HostInterruptMask, val); } +static int +ctlrunits(Ctlr *ctlr) +{ + char *s; + int units, n; + + /* start with 1:1 direct mapping */ + units = ctlr->numports; + + /* manually configured expander? */ + if((s = getconf("*mpt2expander")) != nil) + if((n = atoi(s)) > 0) + units = n; + + /* autodetected expander? (guess based on a single port) */ + if(units == 1 || units > Nunit) + units = Nunit; + + return units; +} + static SDev * mpt2pnp(void) { Pcidev *p; - SDev *sdev, *head, *tail; + SDev *sdev; int ioc; char name[8]; Ctlr *ctlr; static int iocs; p = nil; - head = tail = nil; while(p = pcimatch(p, 0x1000, 0)){ switch(p->did){ case 0x0070: /* LSISAS2004 */ @@ -1281,18 +1458,14 @@ } sdev->ifc = &sdmpt2ifc; sdev->ctlr = ctlr; - sdev->idno = 'M' + ioc; - sdev->nunit = Nunit; + sdev->idno = 'M'; + sdev->nunit = ctlrunits(ctlr); ctlr->sdev = sdev; - print("#S/sd%c: %s: mpt2 sas-2 with %d ports, %d max targets\n", - sdev->idno, name, ctlr->numports, ctlr->maxtargs); - if(head == nil) - head = sdev; - else - tail->next = sdev; - tail = sdev; + sdadddevs(sdev); + print("#S/%s: %s: mpt2 sas-2 with %d ports, %d max targets\n", + sdev->name, name, ctlr->numports, ctlr->maxtargs); } - return head; + return nil; } static void @@ -1407,6 +1580,14 @@ } } +/* hotpluggable. report all the ports (or slots) we've got */ +static int +mpt2verify(SDunit *u) +{ + scsiverify(u); + return 1; +} + static int mpt2enable(SDev *sdev) { @@ -1499,12 +1680,14 @@ u = UNIT(ctlr, unit->subno); getunitcaps(u, buf, sizeof buf); p = seprint(p, e, "capabilities %s\n", buf); - p = seprint(p, e, "wwn %llux\n", GBIT64(u->wwn)); + p = seprint(p, e, "wwn %.16llux\n", u->wwn); p = seprint(p, e, "type %s\n", unittype(u)); p = seprint(p, e, "link %s\n", unitlink(u)); p = seprint(p, e, "state %s\n", unitstate(u)); p = seprint(p, e, "geometry %llud %lud\n", unit->sectors, unit->secsize); + p = seprint(p, e, "slot %d:%d\n", u->ench, u->slot); + return p - o; } @@ -1526,7 +1709,7 @@ mpt2enable, /* enable */ mpt2disable, /* disable */ - scsiverify, /* verify */ + mpt2verify, /* verify */ scsionline, /* online */ mpt2rio, /* rio */ mpt2rctl, /* rctl */ --- /sys/src/9/pc/sdmpt2.c Wed Oct 23 18:39:18 2013 +++ /sys/src/9/pc/sdmpt2.c Wed Oct 23 18:39:18 2013 @@ -1,5 +1,6 @@ /* - * LSI Fusion-MPT SAS 2.0 SCSI Host Adapter + * LSI Fusion-MPT SAS 2.0 SCSI Host Adapter. + * not 64-bit clean */ #include "u.h" #include "../port/lib.h" @@ -7,10 +8,14 @@ #include "dat.h" #include "fns.h" #include "io.h" +#include "ureg.h" #include "../port/error.h" #include "../port/sd.h" +#define B2W(x) ((x) / BIT32SZ) +#define W2B(x) ((x) * BIT32SZ) + extern SDifc sdmpt2ifc; static char Ebadunit[] = "invalid unit"; @@ -101,7 +106,9 @@ EventAck = 0x08, ScsiIoRequest = 0x00, ScsiTaskManagement = 0x01, + ScsiEnclosureProcessor = 0x18, SasIoUnitControl = 0x1b, + SataPassthrough = 0x1c, Config = 0x04, }; @@ -133,19 +140,40 @@ HbdPhyEvent = 0x0024, }; -typedef struct Ctlr Ctlr; -typedef struct Req Req; -typedef struct Unit Unit; +enum { + Nenclosure = 1, + Nunitperenc = 20, + Nunit = Nenclosure * Nunitperenc, /* should limit to 16 but i have more */ + Offline = 0, + Online, +}; + +typedef struct Unit Unit; struct Unit { - uchar status; + int state; + u16int devh; uchar link; - ushort devh; - ulong info; - ushort flags; - uchar wwn[8]; + u32int info; + u16int flags; + u64int wwn; + + int slot; + uint exph; + uint ench; }; +typedef struct Req Req; +struct Req { + Rendez; + u16int smid; + u32int *req; + SDreq *sdreq; + int done; + Req *next; +}; + +typedef struct Ctlr Ctlr; struct Ctlr { Lock; int ioc; @@ -159,12 +187,12 @@ Lock doorlock; RWlock tasklock; - Unit unit[16]; + Unit unit[Nunit]; uchar numports; ushort maxtargs; - ulong *req; + u32int *req; Req *reqfree; Lock reqfreelock; Lock reqpostlock; @@ -172,9 +200,9 @@ ushort iocreqfsz; ushort reqfsz; - ulong *reply; - ulong *replyfree; - ulong *replypost; + u32int *reply; + u32int *replyfree; + u32int *replypost; uchar replyfsz; ushort replyq; ushort replyfreeq; @@ -186,31 +214,23 @@ Queue *eventq; }; -struct Req { - Rendez; - ushort smid; - ulong *req; - SDreq *sdreq; - int done; - Req *next; -}; - -static ulong +static u32int iocread(Ctlr *ctlr, int reg) { - return inl(ctlr->port+reg); + return inl(ctlr->port + reg); } static void -iocwrite(Ctlr *ctlr, int reg, ulong val) +iocwrite(Ctlr *ctlr, int reg, u32int val) { - outl(ctlr->port+reg, val); + outl(ctlr->port + reg, val); } -static ulong -iocwait(Ctlr *ctlr, int reg, ulong mask, int on) +static u32int +iocwait(Ctlr *ctlr, int reg, u32int mask, int on) { - ulong t, val; + ulong t; + u32int val; t = MACHP(0)->ticks; for(;;){ @@ -222,7 +242,7 @@ if(!(val & mask)) return val; if(TK2MS(MACHP(0)->ticks - t) > 10*1000) /* PG §3.7.1 */ - panic("iocwait: %s: wedge reg %#.2ux val %#.8ulx", + panic("iocwait: %s: wedge reg %#.2ux val %#.8ux", ctlr->name, reg, val); microdelay(500); } @@ -231,35 +251,42 @@ static void ckstate(Ctlr *ctlr, int state) { - ulong val; + u32int val; val = iocread(ctlr, Doorbell); if(val & StateMask != state) - panic("ckstate: %s: bad state %#.8ulx", ctlr->name, val); + panic("ckstate: %s: bad state %#.8ux", ctlr->name, val); } static int -ckstatus(Ctlr *ctlr, ulong *reply) +ckstatus(Ctlr *ctlr, u32int *reply) { ushort status; /* - * IOC Status is provided in every reply, which may be used - * to determine the success of a given function independent of - * message content. In the event an unexpected status is - * returned, a panic is issued. + * IOC Status is provided in every reply, which may + * be used to determine the success of a given function + * independent of message content. In the event an + * unexpected status is returned, a panic is issued. */ status = reply[3]>>16 & 0x7fff; /* IOCStatus */ if(status == 0x0000) /* SUCCESS */ return 1; /* - * Some functions support nominal failure modes; rather than - * panic, we allow the caller to determine the severity of the - * condition. + * Some functions support nominal failure modes; rather + * than panic, we allow the caller to determine the + * severity of the condition. */ switch(reply[0]>>24){ /* Function */ case ScsiIoRequest: + case SataPassthrough: + switch(status){ + case 0x0045: /* SCSI_DATA_UNDERRUN */ + case 0x0003: /* INVALID_SGL (user command with r/w bits mixed up) */ + return 0; + } + break; case ScsiTaskManagement: switch(status){ case 0x0040: /* SCSI_RECOVERED_ERROR */ @@ -289,23 +316,22 @@ } break; } - panic("ckstatus: %s: bad status %#.4ux", ctlr->name, status); + panic("ckstatus: %s: fn %.2ux, bad status %#.04ux", + ctlr->name, reply[0]>>24, status); return -1; /* not reached */ } static int -doorbell(Ctlr *ctlr, ulong *req, int nwords) +doorbell(Ctlr *ctlr, u32int *req, int nwords) { - ulong val; int i; - ushort *reply; + u16int *reply; ilock(&ctlr->doorlock); iocwait(ctlr, Doorbell, DoorbellUsed, 0); iocwrite(ctlr, HostInterruptStatus, 0); - val = Handshake | nwords<<16; - iocwrite(ctlr, Doorbell, val); + iocwrite(ctlr, Doorbell, Handshake | nwords<<16); iocwait(ctlr, HostInterruptStatus, Ioc2SysDbStatus, 1); iocwrite(ctlr, HostInterruptStatus, 0); @@ -319,17 +345,17 @@ /* * We do something sneaky here; replies are written back - * into the request buffer during handshake. Buffers must be - * sized to accomodate the larger of the two messages. + * into the request buffer during handshake. Buffers must + * be sized to accomodate the larger of the two messages. * * Doorbell reads yield 16 bits at a time; upper bits are - * considered reserved. The reply MsgLength is located in the - * first 32-bit word; a reply will always contain at least - * DefaultReplyLength words. + * considered reserved. The reply MsgLength is located + * in the first 32-bit word; a reply will always contain + * at least DefaultReplyLength words. */ - reply = (ushort *)req; + reply = (u16int *)req; nwords = DefaultReplyLength; - for(i = 0; i < nwords*2; ++i){ + for(i = 0; i < nwords * BIT16SZ; ++i){ iocwait(ctlr, HostInterruptStatus, Ioc2SysDbStatus, 1); reply[i] = iocread(ctlr, Doorbell); iocwrite(ctlr, HostInterruptStatus, 0); @@ -387,24 +413,6 @@ } static char * -unitstatus(Unit *u) -{ - switch(u->status & 0xf){ /* Reason Code */ - case 0x1: - case 0x5: - return "online"; - case 0x2: - return "missing"; - case 0x3: - return "linkchange"; - case 0x4: - return "nolinkchange"; - default: - return "unknown"; - } -} - -static char * unitlink(Unit *u) { switch(u->link>>4){ /* Current Link Rate */ @@ -421,22 +429,21 @@ } } -static int -unitonline(Unit *u) +static char * +unitstate(Unit *u) { - switch(u->status & 0xf){ /* Reason Code */ - case 0x1: - case 0x5: - return 1; + switch(u->state){ + case Online: + return "online"; default: - return 0; + return "offline"; } } #define REQ(ctlr, n) ((Req *)((ctlr)->req + (n)*(ctlr)->reqfsz + \ (ctlr)->iocreqfsz)) -static ulong * +static u32int * reallocreq(Ctlr *ctlr) { ushort n; @@ -448,9 +455,9 @@ * contiguously, aligned on a 16-byte boundary, and be a * multiple of 16 bytes in length. */ - n = (ctlr->iocreqfsz + ROUNDUP(sizeof(Req), 16))*BY2WD; - ctlr->reqfsz = n/BY2WD; - ctlr->req = mallocalign(ctlr->reqcredit*n, 16, 0, 0); + n = W2B(ctlr->iocreqfsz) + ROUNDUP(sizeof(Req), 16); + ctlr->reqfsz = B2W(n); + ctlr->req = mallocalign(ctlr->reqcredit * n, 16, 0, 0); if(ctlr->req == nil) print("reallocreq: %s: out of memory\n", ctlr->name); return ctlr->req; @@ -487,7 +494,7 @@ } static void -postreq(Ctlr *ctlr, Req *r, ulong *desc, int ms) +postreq(Ctlr *ctlr, Req *r, u32int *desc, int ms) { r->done = 0; @@ -500,13 +507,12 @@ ; tsleep(r, reqdone, r, ms); poperror(); - if(!r->done) error(Etimeout); } static void -postio(Ctlr *ctlr, Req *r, ulong *desc, int ms) +postio(Ctlr *ctlr, Req *r, u32int *desc, int ms) { rlock(&ctlr->tasklock); if(waserror()){ @@ -519,7 +525,7 @@ } static void -posttask(Ctlr *ctlr, Req *r, ulong *desc, int ms) +posttask(Ctlr *ctlr, Req *r, u32int *desc, int ms) { wlock(&ctlr->tasklock); if(waserror()){ @@ -531,8 +537,34 @@ wunlock(&ctlr->tasklock); } +/* need to fix assumptions of caller */ +static void +mksge64(u32int *sgl, void *data, int len, int write) +{ + int flags; + + flags = 1<<7; /* LastElement */ + flags |= 1<<6; /* EndOfBuffer */ + flags |= 1<<4; /* ElementType (Simple Element) */ + if(write) + flags |= 1<<2; /* Direction (Write) */ + flags |= 1<<1; /* AddressSize (64-bit) */ + flags |= 1<<0; /* EndOfList */ + + sgl[0] = flags<<24; /* Flags */ + sgl[0] |= len & 0xffffff; /* Length */ + if(data){ + sgl[1] = Pciwaddrl(data); /* Address */ + sgl[2] = Pciwaddrh(data); /* Address */ + } + else{ + sgl[1] = ~0; + sgl[2] = ~0; + } +} + static void -mksge(ulong *sgl, void *data, int len, int write) +mksge32(u32int *sgl, void *data, int len, int write) { int flags; @@ -555,9 +587,9 @@ static void iocfacts(Ctlr *ctlr) { - ulong buf[16]; + u32int buf[16]; - memset(buf, 0, 3*BY2WD); + memset(buf, 0, W2B(3)); buf[0] = IocFacts<<24; /* Function */ doorbell(ctlr, buf, 3); @@ -574,59 +606,64 @@ static void iocinit(Ctlr *ctlr) { - ulong buf[18]; + u32int buf[18]; - memset(buf, 0, 18*BY2WD); + memset(buf, 0, W2B(18)); buf[0] = IocInit<<24; /* Function */ buf[6] = ctlr->reqfsz<<16; /* SystemRequestFrameSize */ buf[7] = ctlr->replyfreeq<<16; /* ReplyFreeQueueDepth */ buf[7] |= ctlr->replypostq; /* ReplyDescriptorPostQueueDepth */ - buf[10] = PCIWADDR(ctlr->req); /* SystemRequestFrameBaseAddress */ - buf[12] = PCIWADDR(ctlr->replypost); /* ReplyDescriptorPostQueueAddress */ - buf[14] = PCIWADDR(ctlr->replyfree); /* ReplyFreeQueueAddress */ + buf[10] = Pciwaddrl(ctlr->req); /* SystemRequestFrameBaseAddress */ + buf[11] = Pciwaddrh(ctlr->req); + buf[12] = Pciwaddrl(ctlr->replypost); /* ReplyDescriptorPostQueueAddress */ + buf[13] = Pciwaddrh(ctlr->replypost); + buf[14] = Pciwaddrl(ctlr->replyfree); /* ReplyFreeQueueAddress */ + buf[15] = Pciwaddrh(ctlr->replyfree); doorbell(ctlr, buf, 18); ckstatus(ctlr, buf); ckstate(ctlr, StateOperational); } -#define EVENT(x) (1U<<((x) % 32)) +static void +event(u32int *buf, u32int event) +{ + buf[event/32] &= ~(1 << event%32); +} static void eventnotify(Ctlr *ctlr) { - ulong buf[11]; + u32int buf[11]; - memset(buf, 0, 11*BY2WD); + memset(buf, 0, W2B(11)); buf[0] = EventNotification<<24; /* Function */ /* * Event notifications are masked using the bit identified * by the value of the event; see MPI §8.4. PG §3.7.4 - * suggests a number of SAS events required for proper host - * mapping, however the SAS_TOPOLOGY_CHANGE_LIST event is - * merely sufficient. + * suggests a number of SAS events required for proper + * host mapping, however the SAS_TOPOLOGY_CHANGE_LIST + * event is merely sufficient. */ - buf[5] = ~EVENT(SasTopologyChangeList); - buf[6] = ~0; - buf[7] = ~0; - buf[8] = ~0; - buf[9] = ~0; + memset(buf+5, 0xff, 4*4); + event(buf+5, SasTopologyChangeList); + event(buf+5, SasEnclDeviceStatusChange); doorbell(ctlr, buf, 11); ckstatus(ctlr, buf); } static void -eventack(Ctlr *ctlr, ushort event, ulong context) +eventack(Ctlr *ctlr, u16int event, u32int context) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 5*BY2WD); + memset(r->req, 0, W2B(5)); r->req[0] = EventAck<<24; /* Function */ r->req[3] = event; /* Event */ r->req[4] = context; /* EventContext */ @@ -641,18 +678,18 @@ portenable(Ctlr *ctlr) { Req *r; - ulong desc[2]; + u32int desc[2]; /* * The Port Enable message is posted using the Request * Descriptor Post Queue for reliable delivery of events. - * Use of the System Doorbell will miss events on a - * uniprocessor. + * Use of the System Doorbell will miss events when used + * on a uniprocessor. */ r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 3*BY2WD); + memset(r->req, 0, W2B(3)); r->req[0] = PortEnable<<24; /* Function */ desc[0] = r->smid<<16 | 0x4<<1; /* Default Request */ @@ -661,52 +698,64 @@ freereq(ctlr, r); } -static void -unitconfig(Ctlr *ctlr, Unit *u) +static int +devpage(Ctlr *ctlr, uint devh, u32int page[14]) { - ulong buf[7+2], page[14]; + u32int buf[7+2]; /* - * Unit configuration is pulled from SAS Device Page 0. The - * DeviceInfo and Flags fields provide device interconnect and - * capabilities. We obtain configuration via the System - * Doorbell to simplify access to the page header. + * Unit configuration is pulled from SAS Device Page 0. + * The DeviceInfo and Flags fields provide device + * interconnect and capabilities. We obtain configuration + * via the System Doorbell to simplify access to the page + * header. */ - memset(buf, 0, 7*BY2WD); + memset(buf, 0, W2B(7)); buf[0] = Config<<24; /* Function */ buf[0] |= 0x00; /* Action (PAGE_HEADER) */ buf[1] = 0x12<<16; /* ExtPageType (SAS_DEVICE) */ buf[5] = 0xf<<24; /* PageType (Extended) */ buf[5] |= 0<<16; /* PageNumber */ - mksge(buf+7, nil, 0, 0); /* PageBufferSGE (empty) */ + mksge32(buf+7, nil, 0, 0); /* PageBufferSGE (empty) */ doorbell(ctlr, buf, 7+2); - if(!ckstatus(ctlr, buf)) - error(Ebadunit); + ckstatus(ctlr, buf); buf[0] |= 0x01; /* Action (READ_CURRENT) */ - buf[6] = 0x2<<28 | u->devh; /* PageAddress (HANDLE) */ - mksge(buf+7, page, sizeof page, 0); /* PageBufferSGE */ + buf[6] = 0x2<<28 | devh; /* PageAddress (HANDLE) */ + mksge32(buf+7, page, 14*4, 0); /* PageBufferSGE */ doorbell(ctlr, buf, 7+2); if(!ckstatus(ctlr, buf)) - error(Ebadunit); + return -1; + return 0; +} +static void +unitconfig(Ctlr *ctlr, Unit *u, uint devh) +{ + u32int page[14]; + + if(devpage(ctlr, devh, page) == -1) + error(Ebadunit); + u->devh = devh; u->info = page[7]; /* DeviceInfo */ u->flags = page[8] & 0xffff; /* Flags */ - memmove(u->wwn, page+9, 8); /* DeviceName */ + u->wwn = getle((uchar*)(page+9), 8); + u->slot = page[2] & 0xffff; +// print("parent devh %ux phyno %d physport %d\n", page[5]&0xffff, page[5]>>16 & 0xff, page[8]>>24&0xff); } static void unitremove(Ctlr *ctlr, Unit *u) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 11*BY2WD); + memset(r->req, 0, W2B(11)); r->req[0] = SasIoUnitControl<<24; /* Function */ r->req[0] |= 0x0d; /* Operation (REMOVE_DEVICE) */ r->req[1] = u->devh; /* DevHandle */ @@ -721,12 +770,12 @@ unitreset(Ctlr *ctlr, Unit *u) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 13*BY2WD); + memset(r->req, 0, W2B(13)); r->req[0] = ScsiTaskManagement<<24; /* Function */ r->req[0] |= u->devh; /* DevHandle */ r->req[1] = 0x03<<8; /* TaskType (Target Reset) */ @@ -741,12 +790,12 @@ scsiabort(Ctlr *ctlr, Unit *u, Req *s) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 13*BY2WD); + memset(r->req, 0, W2B(13)); r->req[0] = ScsiTaskManagement<<24; /* Function */ r->req[0] |= u->devh; /* DevHandle */ r->req[1] = 0x01<<8; /* TaskType (Abort Task) */ @@ -762,7 +811,7 @@ scsiio(Ctlr *ctlr, Unit *u, SDreq *sdreq) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil){ @@ -770,7 +819,7 @@ return; } r->sdreq = sdreq; - memset(r->req, 0, 24*BY2WD); + memset(r->req, 0, W2B(24)); r->req[0] = ScsiIoRequest<<24; /* Function */ r->req[0] |= u->devh; /* DevHandle */ r->req[3] = PCIWADDR(sdreq->sense); /* SenseBufferLowAddress */ @@ -784,7 +833,7 @@ r->req[15] = 0x2<<24; /* Data Direction (Read) */ memmove(r->req+16, sdreq->cmd, sdreq->clen); - mksge(r->req+24, sdreq->data, sdreq->dlen, sdreq->write); + mksge32(r->req+24, sdreq->data, sdreq->dlen, sdreq->write); desc[0] = r->smid<<16 | 0x0<<1; /* SCSI IO Request */ desc[1] = u->devh<<16; @@ -807,64 +856,65 @@ freereq(ctlr, r); } -#define REPLY(ctlr, n) ((ctlr)->reply + (n)*(ctlr)->replyfsz) +#define REPLY(ctlr, n) ((ctlr)->reply + (n)*1) #define REPLYPOST(ctlr, n) ((ctlr)->replypost + (n)*2) -static ulong * +static u32int * reallocreply(Ctlr *ctlr) { free(ctlr->reply); /* - * System Reply Message Frames are less disciplined; they do - * not have to be contiguous, must be aligned on a 4-byte - * boundary, and must be a multiple of 4 bytes in length. + * System Reply Message Frames are less disciplined; they + * do not have to be contiguous, must be aligned on a + * 4-byte boundary, and must be a multiple of 4 bytes in + * length. */ ctlr->replyq = ctlr->reqcredit + 32; /* PG §3.1.3 */ - ctlr->reply = mallocz(ctlr->replyq * ctlr->replyfsz*BY2WD, 0); + ctlr->reply = mallocz(ctlr->replyq * W2B(ctlr->replyfsz), 0); if(ctlr->reply == nil) print("reallocreply: %s: out of memory\n", ctlr->name); return ctlr->reply; } -static ulong * +static u32int * reallocreplyfree(Ctlr *ctlr) { free(ctlr->replyfree); /* * The Reply Free Queue must be allocated contiguously, - * aligned on a 16-byte boundary, and must have a depth that - * is a multiple of 16 bytes. + * aligned on a 16-byte boundary, and must have a depth + * that is a multiple of 16 bytes. */ ctlr->replyfreeq = ROUNDUP(ctlr->replyq + 1, 16); - ctlr->replyfree = mallocalign(ctlr->replyfreeq * BY2WD, 16, 0, 0); + ctlr->replyfree = mallocalign(ctlr->replyfreeq * W2B(1), 16, 0, 0); if(ctlr->replyfree == nil) print("reallocreplyfree: %s: out of memory\n", ctlr->name); return ctlr->replyfree; } -static ulong * +static u32int * reallocreplypost(Ctlr *ctlr) { free(ctlr->replypost); /* * The Reply Descriptor Post Queue must be allocated - * contiguously and aligned on a 16-byte boundary. The depth - * must not exceed MaxReplyDescriptorPostQueueDepth returned - * in the IOC Facts reply. + * contiguously and aligned on a 16-byte boundary. The + * depth must not exceed MaxReplyDescriptorPostQueueDepth + * returned in the IOC Facts reply. */ ctlr->replypostq = MIN(ctlr->replypostmax, ROUNDUP(ctlr->replyq + ctlr->reqcredit + 1, 16)); - ctlr->replypost = mallocalign(ctlr->replypostq * 2*BY2WD, 16, 0, 0); + ctlr->replypost = mallocalign(ctlr->replypostq * W2B(2), 16, 0, 0); if(ctlr->replypost == nil) print("reallocreplypost: %s: out of memory\n", ctlr->name); return ctlr->replypost; } static void -freereply(Ctlr *ctlr, ulong *reply) +freereply(Ctlr *ctlr, u32int *reply) { ctlr->replyfree[ctlr->replyfreep] = PCIWADDR(reply); ctlr->replyfreep++; @@ -872,10 +922,10 @@ ctlr->replyfreep = 0; } -static ulong * +static u32int * nextreplypost(Ctlr *ctlr) { - ulong *desc; + u32int *desc; desc = REPLYPOST(ctlr, ctlr->replypostp); if(desc[0] == 0xffffffff) @@ -885,7 +935,7 @@ } static void -freereplypost(Ctlr *ctlr, ulong *desc) +freereplypost(Ctlr *ctlr, u32int *desc) { desc[0] = 0xffffffff; desc[1] = 0xffffffff; @@ -895,7 +945,7 @@ } static void -scsiok(Ctlr *ctlr, ushort smid) +scsiok(Ctlr *ctlr, u16int smid) { Req *r; @@ -907,7 +957,7 @@ } static void -scsierror(Ctlr *ctlr, ushort smid, ulong *reply) +scsierror(Ctlr *ctlr, u16int smid, u32int *reply) { Req *r; @@ -922,7 +972,7 @@ } static void -doreply(Ctlr *ctlr, ushort smid, ulong *reply) +doreply(Ctlr *ctlr, u16int smid, u32int *reply) { Req *r; @@ -932,56 +982,151 @@ wakeup(r); } -static void -topoevent(Ctlr *ctlr, ulong *data) +/* + * temporary hack + */ +static int +port2unit(Ctlr *ctlr, uint devh, uint port, uint exph, uint ench) +{ + int unitidx, encidx; + u32int slot, page[14]; + + /* hack: ignore direct ports if controler has 1 port */ + if(ctlr->numports == 1) + encidx = ench - 2; + else + encidx = ench - 1; + if(encidx < 0 || encidx >= Nenclosure) + return -1; + + /* direct? */ + if(exph == 0 && ench == 1) + unitidx = port; + else{ + if(devpage(ctlr, devh, page) == -1) + return -1; + slot = page[2] & 0xffff; + if(slot >= Nunitperenc) + return -1; /* too many drives per enclosure */ + unitidx = encidx*Nunitperenc + slot; + } + + if(unitidx >= Nunit) + return -1; + + return unitidx; +} + +/* sleeze; try to detect when no direct connect */ +static int +anyconnected(u32int *data, int e) { - ulong *p, *e; int i; + u32int *p, reason; + + for(i = 0; i < e; i++){ + p = data + i; + reason = *p>>24 & 0xf; + if(reason == 1 || reason == 2) + return 1; + } + return 0; +} + +static void +topoevent(Ctlr *ctlr, u32int *data) +{ + u32int *p; + int i, b, e, rate, reason, devh, unitno, exph, ench, num, start; Unit *u; + exph = data[0] >> 16; + ench = data[0] & 0xffff; + start = data[2]>>8 & 0xff; + num = data[2]>>0 & 0xff; + print("%s: topoevent exp/enc %d/%d port %d-%d #phys %d\n", ctlr->name, + exph, ench, start, start+num, data[1]); + /* - * Unfortunately, devsd limits us to 16 devices, which - * essentially precludes support for SAS expanders and/or - * enclosures. A one-to-one mapping exists between a direct - * attached PHY and a device for simplicity. + * SAS topology changes are handled in two phases; we + * first obtain identifying information from event data. + * New units require additional configuration information + * and missing units must have resources released. */ - if(data[0]>>16 != 0x0) /* ExpanderDevHandle */ - return; - if((data[0] & 0xffff) != 0x1) /* EnclosureHandle */ + b = data[2]>>8 & 0xff; + e = data[2] & 0xff; /* NumEntries */ + + if(exph == 0 && ench == 1) + if(!anyconnected(data + 3, e)){ + print(" none connected\n"); return; + } - /* - * SAS topology changes are handled in two phases; we first - * obtain identifying information from event data. New units - * require additional configuration information and missing - * units must have resources released. - */ - p = data + 3; - e = p + (data[2] & 0xff); /* NumEntries */ - i = data[2]>>8 & 0xff; /* StartPhyNum */ - for(; p < e && i < nelem(ctlr->unit); ++p, ++i){ - u = UNIT(ctlr, i); - u->status = *p>>24 & 0xff; /* PhyStatus */ - u->link = *p>>16 & 0xff; /* LinkRate */ - switch(u->status & 0xf){ /* Reason Code */ - case 0x1: - u->devh = *p & 0xffff; /* AttachedDevHandle */ - unitconfig(ctlr, u); + for(i = 0; i < e; i++){ + p = data + 3 + i; + rate = p[0]>>16 & 0xff; + devh = p[0] & 0xffff; + reason = *p>>24 & 0xf; + if(devh == 1 || devh == 0) + continue; /* ignore self; empty */ + unitno = port2unit(ctlr, devh, i, exph, ench); + if(unitno == -1 || unitno >= ctlr->sdev->nunit){ + print(" entry %d exph %d ench %d\n", i, exph, ench); + continue; + } + u = UNIT(ctlr, unitno); + switch(reason){ /* PhyStatus (Reason Code) */ + case 1: /* online */ + unitconfig(ctlr, u, devh); + u->exph = exph; + u->ench = ench; + print("unit %d slot %d:%d devh %.4ux rate %x stat %x online wwn %llux\n", + i+b, ench, u->slot, devh, rate, *p>>28, u->wwn); + u->state = Online; break; - case 0x2: + case 2: /* offline */ + print("unit %d devh %.4ux rate %x offline\n", i+b, devh, rate); + u->state = Offline; unitreset(ctlr, u); unitremove(ctlr, u); break; + default: + print("unit %d devh %.4ux rate %x ?? = %d\n", i+b, devh, rate, rate); + break; } + u->link = rate; /* LinkRate */ + } +} + +static void +encevent(Ctlr *, u32int *data) +{ + char *reason; + int s, n; + + switch(data[0]>>16 & 0xff){ + default: + reason = "*gok*"; + break; + case 1: + reason = "added"; + break; + case 2: + reason = "gone"; + break; } + s = data[3] >> 16; + n = data[3] & 0xffff; + print("mpt2: enc %d %s: port %d wwn %llux slots %d-%d phy %b\n", data[0]>>0 & 0xffff, reason, + data[0]>>24, (uvlong)data[2]<<32 | data[1], s, s+n, data[4]); } static void -doevent(Ctlr *ctlr, ulong *reply) +doevent(Ctlr *ctlr, u32int *reply) { ushort event; - ulong context; - ulong *data; + u32int context; + u32int *data; event = reply[5] & 0xffff; /* Event */ context = reply[6]; /* EventContext */ @@ -990,20 +1135,24 @@ case SasTopologyChangeList: topoevent(ctlr, data); break; + case SasEnclDeviceStatusChange: + encevent(ctlr, data); + break; default: - panic("doevent: %s: bad event %#.4ux", ctlr->name, event); + panic("sdmpt2: doevent: %s: bad event %#.4ux", ctlr->name, event); + break; } if(reply[1]>>16 & 0xff) /* AckRequired */ eventack(ctlr, event, context); } static void -qevent(Ctlr *ctlr, ulong *reply) +qevent(Ctlr *ctlr, u32int *reply) { int n; Block *bp; - n = (reply[0]>>16 & 0xff)*BY2WD; /* MsgLength */ + n = W2B(reply[0]>>16 & 0xff); /* MsgLength */ bp = iallocb(n); if(bp == nil) panic("qevent: %s: out of memory", ctlr->name); @@ -1013,7 +1162,30 @@ } static void -addressreply(Ctlr *ctlr, ushort smid, ulong *reply) +satareply(Ctlr *ctlr, u16int smid, u32int *reply) +{ + Req *r; + SDreq *sr; + + r = REQ(ctlr, smid); + sr = r->sdreq; + if(ckstatus(ctlr, reply) == 1){ + if ((reply[3]>>8 & 0xff) == 0x00) /* SASStatus (SUCCESS) */ + sr->status = SDok; + else + sr->status = SDretry; + memmove(sr->data, reply+5, W2B(5)); /* StatusFIS */ + sr->rlen = W2B(5); + sr->dlen = reply[11]; /* TransferCount */ + } else + sr->status = SDretry; + + r->done++; + wakeup(r); +} + +static void +addressreply(Ctlr *ctlr, u16int smid, u32int *reply) { uchar fn; @@ -1024,6 +1196,9 @@ case SasIoUnitControl: doreply(ctlr, smid, reply); break; + case SataPassthrough: + satareply(ctlr, smid, reply); + break; case ScsiIoRequest: /* * Address Replies to SCSI IO Requests always @@ -1034,18 +1209,18 @@ case EventNotification: /* * We queue events for handling in a separate - * process to ensure sanity when the IOC requires - * synchronization via acknowledgement. + * process to ensure sanity when the IOC + * requires synchronization via acknowledgement. */ qevent(ctlr, reply); break; default: - panic("addressreply: %s: bad function %#.2ux", ctlr->name, fn); + panic("addressreply: %s: bad function %#.02ux", ctlr->name, fn); } /* - * To simplify handing a System Reply Message Frame back to - * the IOC, we update the ReplyFreeHostIndex register + * To simplify handing a System Reply Message Frame back + * to the IOC, we update the ReplyFreeHostIndex register * immediately. Unfortunately, this requires additional * coherence. */ @@ -1070,7 +1245,7 @@ static void resetctlr(Ctlr *ctlr) { - ulong val; + u32int val; iocwrite(ctlr, WriteSequence, 0); /* flush */ iocwrite(ctlr, WriteSequence, 0xf); @@ -1102,15 +1277,15 @@ { int i; Req *r; - ulong *p; + u32int *p; memset(ctlr->unit, 0, sizeof ctlr->unit); /* * Each time the controller is reset, an IOC Facts reponse - * may return different values. We err on the side of caution - * and reallocate resources prior to issuing an IOC Init - * request. + * may return different values. We err on the side of + * caution and reallocate resources prior to issuing an + * IOC Init request. */ iocfacts(ctlr); if(reallocreq(ctlr)) @@ -1125,23 +1300,25 @@ /* * Initialize System Request Message Frames and associated * structures. A SMID is written once to avoid headaches - * constructing messages in the I/O path. A SMID of 0 must be - * initialized and is considered reserved; it may not be + * constructing messages in the I/O path. A SMID of 0 must + * be initialized and is considered reserved; it may not be * placed on the free list or used by the host in any way. */ ctlr->reqfree = nil; - for(i = 1; i < ctlr->reqcredit; ++i){ - r = REQ(ctlr, i); - r->smid = i; - r->req = (ulong *)r - ctlr->iocreqfsz; - freereq(ctlr, r); - } + for(i = 0; i < ctlr->reqcredit; ++i) + if(i > 0){ + r = REQ(ctlr, i); + r->smid = i; + r->req = (u32int *)r - ctlr->iocreqfsz; + freereq(ctlr, r); + } /* * The Reply Descriptor Post Queue must be initialized with * the unused descriptor type for each entry. This is - * slightly reordered to take advantage of coherence required - * when updating the ReplyFreeHostIndex register below. + * slightly reordered to take advantage of coherence + * required when updating the ReplyFreeHostIndex register + * below. */ ctlr->replypostp = 0; for(i = 0; i < ctlr->replypostq; ++i){ @@ -1150,10 +1327,10 @@ } /* - * The Reply Free Queue is initialized with the lower 32 - * bits of the PADDR for each System Reply Frame. The - * ReplyFreeHostIndex register is initialized with the next - * (unused) entry. + * The Reply Free Queue is initialized with the lower + * 32 bits of the PADDR for each System Reply Frame. The + * ReplyFreeHostIndex register is initialized with the + * next (unused) entry. */ ctlr->replyfreep = 0; for(i = 0; i < ctlr->replyq; ++i){ @@ -1174,7 +1351,7 @@ static void enablectlr(Ctlr *ctlr) { - ulong val; + u32int val; val = iocread(ctlr, HostInterruptMask); val &= ~ReplyIntMask; @@ -1184,25 +1361,45 @@ static void disablectlr(Ctlr *ctlr) { - ulong val; + u32int val; val = iocread(ctlr, HostInterruptMask); val |= ReplyIntMask; iocwrite(ctlr, HostInterruptMask, val); } +static int +ctlrunits(Ctlr *ctlr) +{ + char *s; + int units, n; + + /* start with 1:1 direct mapping */ + units = ctlr->numports; + + /* manually configured expander? */ + if((s = getconf("*mpt2expander")) != nil) + if((n = atoi(s)) > 0) + units = n; + + /* autodetected expander? (guess based on a single port) */ + if(units == 1 || units > Nunit) + units = Nunit; + + return units; +} + static SDev * mpt2pnp(void) { Pcidev *p; - SDev *sdev, *head, *tail; + SDev *sdev; int ioc; char name[8]; Ctlr *ctlr; static int iocs; p = nil; - head = tail = nil; while(p = pcimatch(p, 0x1000, 0)){ switch(p->did){ case 0x0070: /* LSISAS2004 */ @@ -1226,22 +1423,22 @@ } ioc = iocs++; snprint(name, sizeof name, "ioc%d", ioc); - ctlr = malloc(sizeof *ctlr); + ctlr = malloc(sizeof(Ctlr)); if(ctlr == nil){ print("mpt2pnp: %s: out of memory\n", name); continue; } ctlr->ioc = ioc; - strncpy(ctlr->name, name, sizeof ctlr->name); - ctlr->port = p->mem[0].bar & ~1; - if(ioalloc(ctlr->port, p->mem[0].size, 0, "mpt2") < 0){ - print("mpt2pnp: %s: port %ux in use\n", name, ctlr->port); + strncpy(ctlr->name, name, sizeof(ctlr->name)); + ctlr->port = ioalloc(p->mem[0].bar & ~1, p->mem[0].size, 0, "mpt2"); + if(ctlr->port < 0){ + print("mpt2pnp: %s: ioalloc failed\n", name); freectlr(ctlr); continue; } pcisetbme(p); ctlr->pcidev = p; - ctlr->eventq = qopen(0, Qmsg|Qclosed, nil, nil); + ctlr->eventq = qopen(0, Qmsg | Qclosed, nil, nil); if(ctlr->eventq == nil){ print("mpt2pnp: %s: qopen failed\n", name); freectlr(ctlr); @@ -1253,7 +1450,7 @@ freectlr(ctlr); continue; } - sdev = malloc(sizeof *sdev); + sdev = malloc(sizeof(SDev)); if(sdev == nil){ print("mpt2pnp: %s: out of memory\n", name); freectlr(ctlr); @@ -1261,26 +1458,22 @@ } sdev->ifc = &sdmpt2ifc; sdev->ctlr = ctlr; - sdev->idno = 'M' + ioc; - sdev->nunit = nelem(ctlr->unit); + sdev->idno = 'M'; + sdev->nunit = ctlrunits(ctlr); ctlr->sdev = sdev; - print("#S/sd%c: %s: mpt2 sas-2 with %d ports, %d max targets\n", - sdev->idno, name, ctlr->numports, ctlr->maxtargs); - if(head == nil) - head = sdev; - else - tail->next = sdev; - tail = sdev; + sdadddevs(sdev); + print("#S/%s: %s: mpt2 sas-2 with %d ports, %d max targets\n", + sdev->name, name, ctlr->numports, ctlr->maxtargs); } - return head; + return nil; } static void mpt2interrupt(Ureg *, void *arg) { Ctlr *ctlr; - ulong val, *desc, *reply; - ushort smid; + u32int val, *desc, *reply; + u16int smid; ctlr = arg; ilock(ctlr); @@ -1289,7 +1482,10 @@ iunlock(ctlr); return; } - while(desc = nextreplypost(ctlr)){ + for(;;){ + desc = nextreplypost(ctlr); + if(desc == nil) + break; switch(desc[0] & 0xf){ /* ReplyFlags */ case ScsiIoSuccess: smid = desc[0]>>16; /* SMID */ @@ -1301,7 +1497,7 @@ addressreply(ctlr, smid, reply); break; default: - panic("mpt2interrupt: %s: bad reply %#.8ulx %#.8ulx", + panic("mpt2interrupt: %s: bad reply %#.8ux %#.8ux", ctlr->name, desc[0], desc[1]); } freereplypost(ctlr, desc); @@ -1352,13 +1548,13 @@ { Ctlr *ctlr; Block *bp; - ulong *reply; + u32int *reply; /* * For the unwary, a pending reset will first close the - * eventq, which will cause qbread to eventually error. The - * control structure below yields to the reset kproc until the - * eventq is reopened and sanity is restored. + * eventq, which will cause qbread to eventually error. + * The control structure below yields to the reset kproc + * until the eventq is reopened and sanity is restored. */ ctlr = arg; while(waserror()) @@ -1369,11 +1565,12 @@ runlock(&ctlr->resetlock); nexterror(); } - if(bp = qbread(ctlr->eventq, 0)) + bp = qbread(ctlr->eventq, 0); + if(bp) if(waserror()) wakeup(&ctlr->reset); else{ - reply = (ulong *)bp->rp; + reply = (u32int *)bp->rp; doevent(ctlr, reply); poperror(); } @@ -1383,6 +1580,14 @@ } } +/* hotpluggable. report all the ports (or slots) we've got */ +static int +mpt2verify(SDunit *u) +{ + scsiverify(u); + return 1; +} + static int mpt2enable(SDev *sdev) { @@ -1403,14 +1608,13 @@ kproc("mpt2reset", mpt2reset, ctlr); kproc("mpt2event", mpt2event, ctlr); } - if(waserror()) print("mpt2enable: %s: %s\n", ctlr->name, up->errstr); else{ portenable(ctlr); poperror(); } - return 0; + return 1; } static int @@ -1427,8 +1631,11 @@ iunlock(ctlr); snprint(name, sizeof name, "%s (%s)", sdev->name, sdev->ifc->name); - intrdisable(p->intl, mpt2interrupt, ctlr, p->tbdf, name); - return 0; + +// must fix intrdisable +// intrdisable(p->intl, mpt2interrupt, ctlr, p->tbdf, name); + USED(p); + return 1; } static int @@ -1444,7 +1651,7 @@ runlock(&ctlr->resetlock); return SDeio; } - if(!unitonline(u)) + if(u->state != Online) error(Ebadunit); if(waserror()){ wakeup(&ctlr->reset); @@ -1473,12 +1680,14 @@ u = UNIT(ctlr, unit->subno); getunitcaps(u, buf, sizeof buf); p = seprint(p, e, "capabilities %s\n", buf); - p = seprint(p, e, "wwn %ullx\n", getle(u->wwn, 8)); + p = seprint(p, e, "wwn %.16llux\n", u->wwn); p = seprint(p, e, "type %s\n", unittype(u)); - p = seprint(p, e, "status %s\n", unitstatus(u)); p = seprint(p, e, "link %s\n", unitlink(u)); - p = seprint(p, e, "geometry %ulld %uld\n", + p = seprint(p, e, "state %s\n", unitstate(u)); + p = seprint(p, e, "geometry %llud %lud\n", unit->sectors, unit->secsize); + p = seprint(p, e, "slot %d:%d\n", u->ench, u->slot); + return p - o; } @@ -1500,7 +1709,7 @@ mpt2enable, /* enable */ mpt2disable, /* disable */ - scsiverify, /* verify */ + mpt2verify, /* verify */ scsionline, /* online */ mpt2rio, /* rio */ mpt2rctl, /* rctl */ --- /sys/src/9/pcpae/sdmpt2.c Wed Oct 23 18:39:18 2013 +++ /sys/src/9/pcpae/sdmpt2.c Wed Oct 23 18:39:18 2013 @@ -1,5 +1,6 @@ /* - * LSI Fusion-MPT SAS 2.0 SCSI Host Adapter + * LSI Fusion-MPT SAS 2.0 SCSI Host Adapter. + * not 64-bit clean */ #include "u.h" #include "../port/lib.h" @@ -7,10 +8,14 @@ #include "dat.h" #include "fns.h" #include "io.h" +#include "ureg.h" #include "../port/error.h" #include "../port/sd.h" +#define B2W(x) ((x) / BIT32SZ) +#define W2B(x) ((x) * BIT32SZ) + extern SDifc sdmpt2ifc; static char Ebadunit[] = "invalid unit"; @@ -101,7 +106,9 @@ EventAck = 0x08, ScsiIoRequest = 0x00, ScsiTaskManagement = 0x01, + ScsiEnclosureProcessor = 0x18, SasIoUnitControl = 0x1b, + SataPassthrough = 0x1c, Config = 0x04, }; @@ -133,19 +140,40 @@ HbdPhyEvent = 0x0024, }; -typedef struct Ctlr Ctlr; -typedef struct Req Req; -typedef struct Unit Unit; +enum { + Nenclosure = 1, + Nunitperenc = 20, + Nunit = Nenclosure * Nunitperenc, /* should limit to 16 but i have more */ + Offline = 0, + Online, +}; + +typedef struct Unit Unit; struct Unit { - uchar status; + int state; + u16int devh; uchar link; - ushort devh; - ulong info; - ushort flags; - uchar wwn[8]; + u32int info; + u16int flags; + u64int wwn; + + int slot; + uint exph; + uint ench; }; +typedef struct Req Req; +struct Req { + Rendez; + u16int smid; + u32int *req; + SDreq *sdreq; + int done; + Req *next; +}; + +typedef struct Ctlr Ctlr; struct Ctlr { Lock; int ioc; @@ -159,12 +187,12 @@ Lock doorlock; RWlock tasklock; - Unit unit[16]; + Unit unit[Nunit]; uchar numports; ushort maxtargs; - ulong *req; + u32int *req; Req *reqfree; Lock reqfreelock; Lock reqpostlock; @@ -172,9 +200,9 @@ ushort iocreqfsz; ushort reqfsz; - ulong *reply; - ulong *replyfree; - ulong *replypost; + u32int *reply; + u32int *replyfree; + u32int *replypost; uchar replyfsz; ushort replyq; ushort replyfreeq; @@ -186,31 +214,23 @@ Queue *eventq; }; -struct Req { - Rendez; - ushort smid; - ulong *req; - SDreq *sdreq; - int done; - Req *next; -}; - -static ulong +static u32int iocread(Ctlr *ctlr, int reg) { - return inl(ctlr->port+reg); + return inl(ctlr->port + reg); } static void -iocwrite(Ctlr *ctlr, int reg, ulong val) +iocwrite(Ctlr *ctlr, int reg, u32int val) { - outl(ctlr->port+reg, val); + outl(ctlr->port + reg, val); } -static ulong -iocwait(Ctlr *ctlr, int reg, ulong mask, int on) +static u32int +iocwait(Ctlr *ctlr, int reg, u32int mask, int on) { - ulong t, val; + ulong t; + u32int val; t = MACHP(0)->ticks; for(;;){ @@ -222,7 +242,7 @@ if(!(val & mask)) return val; if(TK2MS(MACHP(0)->ticks - t) > 10*1000) /* PG §3.7.1 */ - panic("iocwait: %s: wedge reg %#.2ux val %#.8ulx", + panic("iocwait: %s: wedge reg %#.2ux val %#.8ux", ctlr->name, reg, val); microdelay(500); } @@ -231,35 +251,42 @@ static void ckstate(Ctlr *ctlr, int state) { - ulong val; + u32int val; val = iocread(ctlr, Doorbell); if(val & StateMask != state) - panic("ckstate: %s: bad state %#.8ulx", ctlr->name, val); + panic("ckstate: %s: bad state %#.8ux", ctlr->name, val); } static int -ckstatus(Ctlr *ctlr, ulong *reply) +ckstatus(Ctlr *ctlr, u32int *reply) { ushort status; /* - * IOC Status is provided in every reply, which may be used - * to determine the success of a given function independent of - * message content. In the event an unexpected status is - * returned, a panic is issued. + * IOC Status is provided in every reply, which may + * be used to determine the success of a given function + * independent of message content. In the event an + * unexpected status is returned, a panic is issued. */ status = reply[3]>>16 & 0x7fff; /* IOCStatus */ if(status == 0x0000) /* SUCCESS */ return 1; /* - * Some functions support nominal failure modes; rather than - * panic, we allow the caller to determine the severity of the - * condition. + * Some functions support nominal failure modes; rather + * than panic, we allow the caller to determine the + * severity of the condition. */ switch(reply[0]>>24){ /* Function */ case ScsiIoRequest: + case SataPassthrough: + switch(status){ + case 0x0045: /* SCSI_DATA_UNDERRUN */ + case 0x0003: /* INVALID_SGL (user command with r/w bits mixed up) */ + return 0; + } + break; case ScsiTaskManagement: switch(status){ case 0x0040: /* SCSI_RECOVERED_ERROR */ @@ -289,23 +316,22 @@ } break; } - panic("ckstatus: %s: bad status %#.4ux", ctlr->name, status); + panic("ckstatus: %s: fn %.2ux, bad status %#.04ux", + ctlr->name, reply[0]>>24, status); return -1; /* not reached */ } static int -doorbell(Ctlr *ctlr, ulong *req, int nwords) +doorbell(Ctlr *ctlr, u32int *req, int nwords) { - ulong val; int i; - ushort *reply; + u16int *reply; ilock(&ctlr->doorlock); iocwait(ctlr, Doorbell, DoorbellUsed, 0); iocwrite(ctlr, HostInterruptStatus, 0); - val = Handshake | nwords<<16; - iocwrite(ctlr, Doorbell, val); + iocwrite(ctlr, Doorbell, Handshake | nwords<<16); iocwait(ctlr, HostInterruptStatus, Ioc2SysDbStatus, 1); iocwrite(ctlr, HostInterruptStatus, 0); @@ -319,17 +345,17 @@ /* * We do something sneaky here; replies are written back - * into the request buffer during handshake. Buffers must be - * sized to accomodate the larger of the two messages. + * into the request buffer during handshake. Buffers must + * be sized to accomodate the larger of the two messages. * * Doorbell reads yield 16 bits at a time; upper bits are - * considered reserved. The reply MsgLength is located in the - * first 32-bit word; a reply will always contain at least - * DefaultReplyLength words. + * considered reserved. The reply MsgLength is located + * in the first 32-bit word; a reply will always contain + * at least DefaultReplyLength words. */ - reply = (ushort *)req; + reply = (u16int *)req; nwords = DefaultReplyLength; - for(i = 0; i < nwords*2; ++i){ + for(i = 0; i < nwords * BIT16SZ; ++i){ iocwait(ctlr, HostInterruptStatus, Ioc2SysDbStatus, 1); reply[i] = iocread(ctlr, Doorbell); iocwrite(ctlr, HostInterruptStatus, 0); @@ -387,24 +413,6 @@ } static char * -unitstatus(Unit *u) -{ - switch(u->status & 0xf){ /* Reason Code */ - case 0x1: - case 0x5: - return "online"; - case 0x2: - return "missing"; - case 0x3: - return "linkchange"; - case 0x4: - return "nolinkchange"; - default: - return "unknown"; - } -} - -static char * unitlink(Unit *u) { switch(u->link>>4){ /* Current Link Rate */ @@ -421,22 +429,21 @@ } } -static int -unitonline(Unit *u) +static char * +unitstate(Unit *u) { - switch(u->status & 0xf){ /* Reason Code */ - case 0x1: - case 0x5: - return 1; + switch(u->state){ + case Online: + return "online"; default: - return 0; + return "offline"; } } #define REQ(ctlr, n) ((Req *)((ctlr)->req + (n)*(ctlr)->reqfsz + \ (ctlr)->iocreqfsz)) -static ulong * +static u32int * reallocreq(Ctlr *ctlr) { ushort n; @@ -448,9 +455,9 @@ * contiguously, aligned on a 16-byte boundary, and be a * multiple of 16 bytes in length. */ - n = (ctlr->iocreqfsz + ROUNDUP(sizeof(Req), 16))*BY2WD; - ctlr->reqfsz = n/BY2WD; - ctlr->req = mallocalign(ctlr->reqcredit*n, 16, 0, 0); + n = W2B(ctlr->iocreqfsz) + ROUNDUP(sizeof(Req), 16); + ctlr->reqfsz = B2W(n); + ctlr->req = mallocalign(ctlr->reqcredit * n, 16, 0, 0); if(ctlr->req == nil) print("reallocreq: %s: out of memory\n", ctlr->name); return ctlr->req; @@ -487,7 +494,7 @@ } static void -postreq(Ctlr *ctlr, Req *r, ulong *desc, int ms) +postreq(Ctlr *ctlr, Req *r, u32int *desc, int ms) { r->done = 0; @@ -500,13 +507,12 @@ ; tsleep(r, reqdone, r, ms); poperror(); - if(!r->done) error(Etimeout); } static void -postio(Ctlr *ctlr, Req *r, ulong *desc, int ms) +postio(Ctlr *ctlr, Req *r, u32int *desc, int ms) { rlock(&ctlr->tasklock); if(waserror()){ @@ -519,7 +525,7 @@ } static void -posttask(Ctlr *ctlr, Req *r, ulong *desc, int ms) +posttask(Ctlr *ctlr, Req *r, u32int *desc, int ms) { wlock(&ctlr->tasklock); if(waserror()){ @@ -531,8 +537,34 @@ wunlock(&ctlr->tasklock); } +/* need to fix assumptions of caller */ +static void +mksge64(u32int *sgl, void *data, int len, int write) +{ + int flags; + + flags = 1<<7; /* LastElement */ + flags |= 1<<6; /* EndOfBuffer */ + flags |= 1<<4; /* ElementType (Simple Element) */ + if(write) + flags |= 1<<2; /* Direction (Write) */ + flags |= 1<<1; /* AddressSize (64-bit) */ + flags |= 1<<0; /* EndOfList */ + + sgl[0] = flags<<24; /* Flags */ + sgl[0] |= len & 0xffffff; /* Length */ + if(data){ + sgl[1] = Pciwaddrl(data); /* Address */ + sgl[2] = Pciwaddrh(data); /* Address */ + } + else{ + sgl[1] = ~0; + sgl[2] = ~0; + } +} + static void -mksge(ulong *sgl, void *data, int len, int write) +mksge32(u32int *sgl, void *data, int len, int write) { int flags; @@ -555,9 +587,9 @@ static void iocfacts(Ctlr *ctlr) { - ulong buf[16]; + u32int buf[16]; - memset(buf, 0, 3*BY2WD); + memset(buf, 0, W2B(3)); buf[0] = IocFacts<<24; /* Function */ doorbell(ctlr, buf, 3); @@ -574,59 +606,64 @@ static void iocinit(Ctlr *ctlr) { - ulong buf[18]; + u32int buf[18]; - memset(buf, 0, 18*BY2WD); + memset(buf, 0, W2B(18)); buf[0] = IocInit<<24; /* Function */ buf[6] = ctlr->reqfsz<<16; /* SystemRequestFrameSize */ buf[7] = ctlr->replyfreeq<<16; /* ReplyFreeQueueDepth */ buf[7] |= ctlr->replypostq; /* ReplyDescriptorPostQueueDepth */ - buf[10] = PCIWADDR(ctlr->req); /* SystemRequestFrameBaseAddress */ - buf[12] = PCIWADDR(ctlr->replypost); /* ReplyDescriptorPostQueueAddress */ - buf[14] = PCIWADDR(ctlr->replyfree); /* ReplyFreeQueueAddress */ + buf[10] = Pciwaddrl(ctlr->req); /* SystemRequestFrameBaseAddress */ + buf[11] = Pciwaddrh(ctlr->req); + buf[12] = Pciwaddrl(ctlr->replypost); /* ReplyDescriptorPostQueueAddress */ + buf[13] = Pciwaddrh(ctlr->replypost); + buf[14] = Pciwaddrl(ctlr->replyfree); /* ReplyFreeQueueAddress */ + buf[15] = Pciwaddrh(ctlr->replyfree); doorbell(ctlr, buf, 18); ckstatus(ctlr, buf); ckstate(ctlr, StateOperational); } -#define EVENT(x) (1U<<((x) % 32)) +static void +event(u32int *buf, u32int event) +{ + buf[event/32] &= ~(1 << event%32); +} static void eventnotify(Ctlr *ctlr) { - ulong buf[11]; + u32int buf[11]; - memset(buf, 0, 11*BY2WD); + memset(buf, 0, W2B(11)); buf[0] = EventNotification<<24; /* Function */ /* * Event notifications are masked using the bit identified * by the value of the event; see MPI §8.4. PG §3.7.4 - * suggests a number of SAS events required for proper host - * mapping, however the SAS_TOPOLOGY_CHANGE_LIST event is - * merely sufficient. + * suggests a number of SAS events required for proper + * host mapping, however the SAS_TOPOLOGY_CHANGE_LIST + * event is merely sufficient. */ - buf[5] = ~EVENT(SasTopologyChangeList); - buf[6] = ~0; - buf[7] = ~0; - buf[8] = ~0; - buf[9] = ~0; + memset(buf+5, 0xff, 4*4); + event(buf+5, SasTopologyChangeList); + event(buf+5, SasEnclDeviceStatusChange); doorbell(ctlr, buf, 11); ckstatus(ctlr, buf); } static void -eventack(Ctlr *ctlr, ushort event, ulong context) +eventack(Ctlr *ctlr, u16int event, u32int context) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 5*BY2WD); + memset(r->req, 0, W2B(5)); r->req[0] = EventAck<<24; /* Function */ r->req[3] = event; /* Event */ r->req[4] = context; /* EventContext */ @@ -641,18 +678,18 @@ portenable(Ctlr *ctlr) { Req *r; - ulong desc[2]; + u32int desc[2]; /* * The Port Enable message is posted using the Request * Descriptor Post Queue for reliable delivery of events. - * Use of the System Doorbell will miss events on a - * uniprocessor. + * Use of the System Doorbell will miss events when used + * on a uniprocessor. */ r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 3*BY2WD); + memset(r->req, 0, W2B(3)); r->req[0] = PortEnable<<24; /* Function */ desc[0] = r->smid<<16 | 0x4<<1; /* Default Request */ @@ -661,52 +698,64 @@ freereq(ctlr, r); } -static void -unitconfig(Ctlr *ctlr, Unit *u) +static int +devpage(Ctlr *ctlr, uint devh, u32int page[14]) { - ulong buf[7+2], page[14]; + u32int buf[7+2]; /* - * Unit configuration is pulled from SAS Device Page 0. The - * DeviceInfo and Flags fields provide device interconnect and - * capabilities. We obtain configuration via the System - * Doorbell to simplify access to the page header. + * Unit configuration is pulled from SAS Device Page 0. + * The DeviceInfo and Flags fields provide device + * interconnect and capabilities. We obtain configuration + * via the System Doorbell to simplify access to the page + * header. */ - memset(buf, 0, 7*BY2WD); + memset(buf, 0, W2B(7)); buf[0] = Config<<24; /* Function */ buf[0] |= 0x00; /* Action (PAGE_HEADER) */ buf[1] = 0x12<<16; /* ExtPageType (SAS_DEVICE) */ buf[5] = 0xf<<24; /* PageType (Extended) */ buf[5] |= 0<<16; /* PageNumber */ - mksge(buf+7, nil, 0, 0); /* PageBufferSGE (empty) */ + mksge32(buf+7, nil, 0, 0); /* PageBufferSGE (empty) */ doorbell(ctlr, buf, 7+2); - if(!ckstatus(ctlr, buf)) - error(Ebadunit); + ckstatus(ctlr, buf); buf[0] |= 0x01; /* Action (READ_CURRENT) */ - buf[6] = 0x2<<28 | u->devh; /* PageAddress (HANDLE) */ - mksge(buf+7, page, sizeof page, 0); /* PageBufferSGE */ + buf[6] = 0x2<<28 | devh; /* PageAddress (HANDLE) */ + mksge32(buf+7, page, 14*4, 0); /* PageBufferSGE */ doorbell(ctlr, buf, 7+2); if(!ckstatus(ctlr, buf)) - error(Ebadunit); + return -1; + return 0; +} +static void +unitconfig(Ctlr *ctlr, Unit *u, uint devh) +{ + u32int page[14]; + + if(devpage(ctlr, devh, page) == -1) + error(Ebadunit); + u->devh = devh; u->info = page[7]; /* DeviceInfo */ u->flags = page[8] & 0xffff; /* Flags */ - memmove(u->wwn, page+9, 8); /* DeviceName */ + u->wwn = getle((uchar*)(page+9), 8); + u->slot = page[2] & 0xffff; +// print("parent devh %ux phyno %d physport %d\n", page[5]&0xffff, page[5]>>16 & 0xff, page[8]>>24&0xff); } static void unitremove(Ctlr *ctlr, Unit *u) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 11*BY2WD); + memset(r->req, 0, W2B(11)); r->req[0] = SasIoUnitControl<<24; /* Function */ r->req[0] |= 0x0d; /* Operation (REMOVE_DEVICE) */ r->req[1] = u->devh; /* DevHandle */ @@ -721,12 +770,12 @@ unitreset(Ctlr *ctlr, Unit *u) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 13*BY2WD); + memset(r->req, 0, W2B(13)); r->req[0] = ScsiTaskManagement<<24; /* Function */ r->req[0] |= u->devh; /* DevHandle */ r->req[1] = 0x03<<8; /* TaskType (Target Reset) */ @@ -741,12 +790,12 @@ scsiabort(Ctlr *ctlr, Unit *u, Req *s) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil) error(Enoreqs); - memset(r->req, 0, 13*BY2WD); + memset(r->req, 0, W2B(13)); r->req[0] = ScsiTaskManagement<<24; /* Function */ r->req[0] |= u->devh; /* DevHandle */ r->req[1] = 0x01<<8; /* TaskType (Abort Task) */ @@ -762,7 +811,7 @@ scsiio(Ctlr *ctlr, Unit *u, SDreq *sdreq) { Req *r; - ulong desc[2]; + u32int desc[2]; r = nextreq(ctlr); if(r == nil){ @@ -770,7 +819,7 @@ return; } r->sdreq = sdreq; - memset(r->req, 0, 24*BY2WD); + memset(r->req, 0, W2B(24)); r->req[0] = ScsiIoRequest<<24; /* Function */ r->req[0] |= u->devh; /* DevHandle */ r->req[3] = PCIWADDR(sdreq->sense); /* SenseBufferLowAddress */ @@ -784,7 +833,7 @@ r->req[15] = 0x2<<24; /* Data Direction (Read) */ memmove(r->req+16, sdreq->cmd, sdreq->clen); - mksge(r->req+24, sdreq->data, sdreq->dlen, sdreq->write); + mksge32(r->req+24, sdreq->data, sdreq->dlen, sdreq->write); desc[0] = r->smid<<16 | 0x0<<1; /* SCSI IO Request */ desc[1] = u->devh<<16; @@ -807,64 +856,65 @@ freereq(ctlr, r); } -#define REPLY(ctlr, n) ((ctlr)->reply + (n)*(ctlr)->replyfsz) +#define REPLY(ctlr, n) ((ctlr)->reply + (n)*1) #define REPLYPOST(ctlr, n) ((ctlr)->replypost + (n)*2) -static ulong * +static u32int * reallocreply(Ctlr *ctlr) { free(ctlr->reply); /* - * System Reply Message Frames are less disciplined; they do - * not have to be contiguous, must be aligned on a 4-byte - * boundary, and must be a multiple of 4 bytes in length. + * System Reply Message Frames are less disciplined; they + * do not have to be contiguous, must be aligned on a + * 4-byte boundary, and must be a multiple of 4 bytes in + * length. */ ctlr->replyq = ctlr->reqcredit + 32; /* PG §3.1.3 */ - ctlr->reply = mallocz(ctlr->replyq * ctlr->replyfsz*BY2WD, 0); + ctlr->reply = mallocz(ctlr->replyq * W2B(ctlr->replyfsz), 0); if(ctlr->reply == nil) print("reallocreply: %s: out of memory\n", ctlr->name); return ctlr->reply; } -static ulong * +static u32int * reallocreplyfree(Ctlr *ctlr) { free(ctlr->replyfree); /* * The Reply Free Queue must be allocated contiguously, - * aligned on a 16-byte boundary, and must have a depth that - * is a multiple of 16 bytes. + * aligned on a 16-byte boundary, and must have a depth + * that is a multiple of 16 bytes. */ ctlr->replyfreeq = ROUNDUP(ctlr->replyq + 1, 16); - ctlr->replyfree = mallocalign(ctlr->replyfreeq * BY2WD, 16, 0, 0); + ctlr->replyfree = mallocalign(ctlr->replyfreeq * W2B(1), 16, 0, 0); if(ctlr->replyfree == nil) print("reallocreplyfree: %s: out of memory\n", ctlr->name); return ctlr->replyfree; } -static ulong * +static u32int * reallocreplypost(Ctlr *ctlr) { free(ctlr->replypost); /* * The Reply Descriptor Post Queue must be allocated - * contiguously and aligned on a 16-byte boundary. The depth - * must not exceed MaxReplyDescriptorPostQueueDepth returned - * in the IOC Facts reply. + * contiguously and aligned on a 16-byte boundary. The + * depth must not exceed MaxReplyDescriptorPostQueueDepth + * returned in the IOC Facts reply. */ ctlr->replypostq = MIN(ctlr->replypostmax, ROUNDUP(ctlr->replyq + ctlr->reqcredit + 1, 16)); - ctlr->replypost = mallocalign(ctlr->replypostq * 2*BY2WD, 16, 0, 0); + ctlr->replypost = mallocalign(ctlr->replypostq * W2B(2), 16, 0, 0); if(ctlr->replypost == nil) print("reallocreplypost: %s: out of memory\n", ctlr->name); return ctlr->replypost; } static void -freereply(Ctlr *ctlr, ulong *reply) +freereply(Ctlr *ctlr, u32int *reply) { ctlr->replyfree[ctlr->replyfreep] = PCIWADDR(reply); ctlr->replyfreep++; @@ -872,10 +922,10 @@ ctlr->replyfreep = 0; } -static ulong * +static u32int * nextreplypost(Ctlr *ctlr) { - ulong *desc; + u32int *desc; desc = REPLYPOST(ctlr, ctlr->replypostp); if(desc[0] == 0xffffffff) @@ -885,7 +935,7 @@ } static void -freereplypost(Ctlr *ctlr, ulong *desc) +freereplypost(Ctlr *ctlr, u32int *desc) { desc[0] = 0xffffffff; desc[1] = 0xffffffff; @@ -895,7 +945,7 @@ } static void -scsiok(Ctlr *ctlr, ushort smid) +scsiok(Ctlr *ctlr, u16int smid) { Req *r; @@ -907,7 +957,7 @@ } static void -scsierror(Ctlr *ctlr, ushort smid, ulong *reply) +scsierror(Ctlr *ctlr, u16int smid, u32int *reply) { Req *r; @@ -922,7 +972,7 @@ } static void -doreply(Ctlr *ctlr, ushort smid, ulong *reply) +doreply(Ctlr *ctlr, u16int smid, u32int *reply) { Req *r; @@ -932,56 +982,151 @@ wakeup(r); } -static void -topoevent(Ctlr *ctlr, ulong *data) +/* + * temporary hack + */ +static int +port2unit(Ctlr *ctlr, uint devh, uint port, uint exph, uint ench) +{ + int unitidx, encidx; + u32int slot, page[14]; + + /* hack: ignore direct ports if controler has 1 port */ + if(ctlr->numports == 1) + encidx = ench - 2; + else + encidx = ench - 1; + if(encidx < 0 || encidx >= Nenclosure) + return -1; + + /* direct? */ + if(exph == 0 && ench == 1) + unitidx = port; + else{ + if(devpage(ctlr, devh, page) == -1) + return -1; + slot = page[2] & 0xffff; + if(slot >= Nunitperenc) + return -1; /* too many drives per enclosure */ + unitidx = encidx*Nunitperenc + slot; + } + + if(unitidx >= Nunit) + return -1; + + return unitidx; +} + +/* sleeze; try to detect when no direct connect */ +static int +anyconnected(u32int *data, int e) { - ulong *p, *e; int i; + u32int *p, reason; + + for(i = 0; i < e; i++){ + p = data + i; + reason = *p>>24 & 0xf; + if(reason == 1 || reason == 2) + return 1; + } + return 0; +} + +static void +topoevent(Ctlr *ctlr, u32int *data) +{ + u32int *p; + int i, b, e, rate, reason, devh, unitno, exph, ench, num, start; Unit *u; + exph = data[0] >> 16; + ench = data[0] & 0xffff; + start = data[2]>>8 & 0xff; + num = data[2]>>0 & 0xff; + print("%s: topoevent exp/enc %d/%d port %d-%d #phys %d\n", ctlr->name, + exph, ench, start, start+num, data[1]); + /* - * Unfortunately, devsd limits us to 16 devices, which - * essentially precludes support for SAS expanders and/or - * enclosures. A one-to-one mapping exists between a direct - * attached PHY and a device for simplicity. + * SAS topology changes are handled in two phases; we + * first obtain identifying information from event data. + * New units require additional configuration information + * and missing units must have resources released. */ - if(data[0]>>16 != 0x0) /* ExpanderDevHandle */ - return; - if((data[0] & 0xffff) != 0x1) /* EnclosureHandle */ + b = data[2]>>8 & 0xff; + e = data[2] & 0xff; /* NumEntries */ + + if(exph == 0 && ench == 1) + if(!anyconnected(data + 3, e)){ + print(" none connected\n"); return; + } - /* - * SAS topology changes are handled in two phases; we first - * obtain identifying information from event data. New units - * require additional configuration information and missing - * units must have resources released. - */ - p = data + 3; - e = p + (data[2] & 0xff); /* NumEntries */ - i = data[2]>>8 & 0xff; /* StartPhyNum */ - for(; p < e && i < nelem(ctlr->unit); ++p, ++i){ - u = UNIT(ctlr, i); - u->status = *p>>24 & 0xff; /* PhyStatus */ - u->link = *p>>16 & 0xff; /* LinkRate */ - switch(u->status & 0xf){ /* Reason Code */ - case 0x1: - u->devh = *p & 0xffff; /* AttachedDevHandle */ - unitconfig(ctlr, u); + for(i = 0; i < e; i++){ + p = data + 3 + i; + rate = p[0]>>16 & 0xff; + devh = p[0] & 0xffff; + reason = *p>>24 & 0xf; + if(devh == 1 || devh == 0) + continue; /* ignore self; empty */ + unitno = port2unit(ctlr, devh, i, exph, ench); + if(unitno == -1 || unitno >= ctlr->sdev->nunit){ + print(" entry %d exph %d ench %d\n", i, exph, ench); + continue; + } + u = UNIT(ctlr, unitno); + switch(reason){ /* PhyStatus (Reason Code) */ + case 1: /* online */ + unitconfig(ctlr, u, devh); + u->exph = exph; + u->ench = ench; + print("unit %d slot %d:%d devh %.4ux rate %x stat %x online wwn %llux\n", + i+b, ench, u->slot, devh, rate, *p>>28, u->wwn); + u->state = Online; break; - case 0x2: + case 2: /* offline */ + print("unit %d devh %.4ux rate %x offline\n", i+b, devh, rate); + u->state = Offline; unitreset(ctlr, u); unitremove(ctlr, u); break; + default: + print("unit %d devh %.4ux rate %x ?? = %d\n", i+b, devh, rate, rate); + break; } + u->link = rate; /* LinkRate */ + } +} + +static void +encevent(Ctlr *, u32int *data) +{ + char *reason; + int s, n; + + switch(data[0]>>16 & 0xff){ + default: + reason = "*gok*"; + break; + case 1: + reason = "added"; + break; + case 2: + reason = "gone"; + break; } + s = data[3] >> 16; + n = data[3] & 0xffff; + print("mpt2: enc %d %s: port %d wwn %llux slots %d-%d phy %b\n", data[0]>>0 & 0xffff, reason, + data[0]>>24, (uvlong)data[2]<<32 | data[1], s, s+n, data[4]); } static void -doevent(Ctlr *ctlr, ulong *reply) +doevent(Ctlr *ctlr, u32int *reply) { ushort event; - ulong context; - ulong *data; + u32int context; + u32int *data; event = reply[5] & 0xffff; /* Event */ context = reply[6]; /* EventContext */ @@ -990,20 +1135,24 @@ case SasTopologyChangeList: topoevent(ctlr, data); break; + case SasEnclDeviceStatusChange: + encevent(ctlr, data); + break; default: - panic("doevent: %s: bad event %#.4ux", ctlr->name, event); + panic("sdmpt2: doevent: %s: bad event %#.4ux", ctlr->name, event); + break; } if(reply[1]>>16 & 0xff) /* AckRequired */ eventack(ctlr, event, context); } static void -qevent(Ctlr *ctlr, ulong *reply) +qevent(Ctlr *ctlr, u32int *reply) { int n; Block *bp; - n = (reply[0]>>16 & 0xff)*BY2WD; /* MsgLength */ + n = W2B(reply[0]>>16 & 0xff); /* MsgLength */ bp = iallocb(n); if(bp == nil) panic("qevent: %s: out of memory", ctlr->name); @@ -1013,7 +1162,30 @@ } static void -addressreply(Ctlr *ctlr, ushort smid, ulong *reply) +satareply(Ctlr *ctlr, u16int smid, u32int *reply) +{ + Req *r; + SDreq *sr; + + r = REQ(ctlr, smid); + sr = r->sdreq; + if(ckstatus(ctlr, reply) == 1){ + if ((reply[3]>>8 & 0xff) == 0x00) /* SASStatus (SUCCESS) */ + sr->status = SDok; + else + sr->status = SDretry; + memmove(sr->data, reply+5, W2B(5)); /* StatusFIS */ + sr->rlen = W2B(5); + sr->dlen = reply[11]; /* TransferCount */ + } else + sr->status = SDretry; + + r->done++; + wakeup(r); +} + +static void +addressreply(Ctlr *ctlr, u16int smid, u32int *reply) { uchar fn; @@ -1024,6 +1196,9 @@ case SasIoUnitControl: doreply(ctlr, smid, reply); break; + case SataPassthrough: + satareply(ctlr, smid, reply); + break; case ScsiIoRequest: /* * Address Replies to SCSI IO Requests always @@ -1034,18 +1209,18 @@ case EventNotification: /* * We queue events for handling in a separate - * process to ensure sanity when the IOC requires - * synchronization via acknowledgement. + * process to ensure sanity when the IOC + * requires synchronization via acknowledgement. */ qevent(ctlr, reply); break; default: - panic("addressreply: %s: bad function %#.2ux", ctlr->name, fn); + panic("addressreply: %s: bad function %#.02ux", ctlr->name, fn); } /* - * To simplify handing a System Reply Message Frame back to - * the IOC, we update the ReplyFreeHostIndex register + * To simplify handing a System Reply Message Frame back + * to the IOC, we update the ReplyFreeHostIndex register * immediately. Unfortunately, this requires additional * coherence. */ @@ -1070,7 +1245,7 @@ static void resetctlr(Ctlr *ctlr) { - ulong val; + u32int val; iocwrite(ctlr, WriteSequence, 0); /* flush */ iocwrite(ctlr, WriteSequence, 0xf); @@ -1102,15 +1277,15 @@ { int i; Req *r; - ulong *p; + u32int *p; memset(ctlr->unit, 0, sizeof ctlr->unit); /* * Each time the controller is reset, an IOC Facts reponse - * may return different values. We err on the side of caution - * and reallocate resources prior to issuing an IOC Init - * request. + * may return different values. We err on the side of + * caution and reallocate resources prior to issuing an + * IOC Init request. */ iocfacts(ctlr); if(reallocreq(ctlr)) @@ -1125,23 +1300,25 @@ /* * Initialize System Request Message Frames and associated * structures. A SMID is written once to avoid headaches - * constructing messages in the I/O path. A SMID of 0 must be - * initialized and is considered reserved; it may not be + * constructing messages in the I/O path. A SMID of 0 must + * be initialized and is considered reserved; it may not be * placed on the free list or used by the host in any way. */ ctlr->reqfree = nil; - for(i = 1; i < ctlr->reqcredit; ++i){ - r = REQ(ctlr, i); - r->smid = i; - r->req = (ulong *)r - ctlr->iocreqfsz; - freereq(ctlr, r); - } + for(i = 0; i < ctlr->reqcredit; ++i) + if(i > 0){ + r = REQ(ctlr, i); + r->smid = i; + r->req = (u32int *)r - ctlr->iocreqfsz; + freereq(ctlr, r); + } /* * The Reply Descriptor Post Queue must be initialized with * the unused descriptor type for each entry. This is - * slightly reordered to take advantage of coherence required - * when updating the ReplyFreeHostIndex register below. + * slightly reordered to take advantage of coherence + * required when updating the ReplyFreeHostIndex register + * below. */ ctlr->replypostp = 0; for(i = 0; i < ctlr->replypostq; ++i){ @@ -1150,10 +1327,10 @@ } /* - * The Reply Free Queue is initialized with the lower 32 - * bits of the PADDR for each System Reply Frame. The - * ReplyFreeHostIndex register is initialized with the next - * (unused) entry. + * The Reply Free Queue is initialized with the lower + * 32 bits of the PADDR for each System Reply Frame. The + * ReplyFreeHostIndex register is initialized with the + * next (unused) entry. */ ctlr->replyfreep = 0; for(i = 0; i < ctlr->replyq; ++i){ @@ -1174,7 +1351,7 @@ static void enablectlr(Ctlr *ctlr) { - ulong val; + u32int val; val = iocread(ctlr, HostInterruptMask); val &= ~ReplyIntMask; @@ -1184,25 +1361,45 @@ static void disablectlr(Ctlr *ctlr) { - ulong val; + u32int val; val = iocread(ctlr, HostInterruptMask); val |= ReplyIntMask; iocwrite(ctlr, HostInterruptMask, val); } +static int +ctlrunits(Ctlr *ctlr) +{ + char *s; + int units, n; + + /* start with 1:1 direct mapping */ + units = ctlr->numports; + + /* manually configured expander? */ + if((s = getconf("*mpt2expander")) != nil) + if((n = atoi(s)) > 0) + units = n; + + /* autodetected expander? (guess based on a single port) */ + if(units == 1 || units > Nunit) + units = Nunit; + + return units; +} + static SDev * mpt2pnp(void) { Pcidev *p; - SDev *sdev, *head, *tail; + SDev *sdev; int ioc; char name[8]; Ctlr *ctlr; static int iocs; p = nil; - head = tail = nil; while(p = pcimatch(p, 0x1000, 0)){ switch(p->did){ case 0x0070: /* LSISAS2004 */ @@ -1226,22 +1423,22 @@ } ioc = iocs++; snprint(name, sizeof name, "ioc%d", ioc); - ctlr = malloc(sizeof *ctlr); + ctlr = malloc(sizeof(Ctlr)); if(ctlr == nil){ print("mpt2pnp: %s: out of memory\n", name); continue; } ctlr->ioc = ioc; - strncpy(ctlr->name, name, sizeof ctlr->name); - ctlr->port = p->mem[0].bar & ~1; - if(ioalloc(ctlr->port, p->mem[0].size, 0, "mpt2") < 0){ - print("mpt2pnp: %s: port %ux in use\n", name, ctlr->port); + strncpy(ctlr->name, name, sizeof(ctlr->name)); + ctlr->port = ioalloc(p->mem[0].bar & ~1, p->mem[0].size, 0, "mpt2"); + if(ctlr->port < 0){ + print("mpt2pnp: %s: ioalloc failed\n", name); freectlr(ctlr); continue; } pcisetbme(p); ctlr->pcidev = p; - ctlr->eventq = qopen(0, Qmsg|Qclosed, nil, nil); + ctlr->eventq = qopen(0, Qmsg | Qclosed, nil, nil); if(ctlr->eventq == nil){ print("mpt2pnp: %s: qopen failed\n", name); freectlr(ctlr); @@ -1253,7 +1450,7 @@ freectlr(ctlr); continue; } - sdev = malloc(sizeof *sdev); + sdev = malloc(sizeof(SDev)); if(sdev == nil){ print("mpt2pnp: %s: out of memory\n", name); freectlr(ctlr); @@ -1261,26 +1458,22 @@ } sdev->ifc = &sdmpt2ifc; sdev->ctlr = ctlr; - sdev->idno = 'M' + ioc; - sdev->nunit = nelem(ctlr->unit); + sdev->idno = 'M'; + sdev->nunit = ctlrunits(ctlr); ctlr->sdev = sdev; - print("#S/sd%c: %s: mpt2 sas-2 with %d ports, %d max targets\n", - sdev->idno, name, ctlr->numports, ctlr->maxtargs); - if(head == nil) - head = sdev; - else - tail->next = sdev; - tail = sdev; + sdadddevs(sdev); + print("#S/%s: %s: mpt2 sas-2 with %d ports, %d max targets\n", + sdev->name, name, ctlr->numports, ctlr->maxtargs); } - return head; + return nil; } static void mpt2interrupt(Ureg *, void *arg) { Ctlr *ctlr; - ulong val, *desc, *reply; - ushort smid; + u32int val, *desc, *reply; + u16int smid; ctlr = arg; ilock(ctlr); @@ -1289,7 +1482,10 @@ iunlock(ctlr); return; } - while(desc = nextreplypost(ctlr)){ + for(;;){ + desc = nextreplypost(ctlr); + if(desc == nil) + break; switch(desc[0] & 0xf){ /* ReplyFlags */ case ScsiIoSuccess: smid = desc[0]>>16; /* SMID */ @@ -1301,7 +1497,7 @@ addressreply(ctlr, smid, reply); break; default: - panic("mpt2interrupt: %s: bad reply %#.8ulx %#.8ulx", + panic("mpt2interrupt: %s: bad reply %#.8ux %#.8ux", ctlr->name, desc[0], desc[1]); } freereplypost(ctlr, desc); @@ -1352,13 +1548,13 @@ { Ctlr *ctlr; Block *bp; - ulong *reply; + u32int *reply; /* * For the unwary, a pending reset will first close the - * eventq, which will cause qbread to eventually error. The - * control structure below yields to the reset kproc until the - * eventq is reopened and sanity is restored. + * eventq, which will cause qbread to eventually error. + * The control structure below yields to the reset kproc + * until the eventq is reopened and sanity is restored. */ ctlr = arg; while(waserror()) @@ -1369,11 +1565,12 @@ runlock(&ctlr->resetlock); nexterror(); } - if(bp = qbread(ctlr->eventq, 0)) + bp = qbread(ctlr->eventq, 0); + if(bp) if(waserror()) wakeup(&ctlr->reset); else{ - reply = (ulong *)bp->rp; + reply = (u32int *)bp->rp; doevent(ctlr, reply); poperror(); } @@ -1383,6 +1580,14 @@ } } +/* hotpluggable. report all the ports (or slots) we've got */ +static int +mpt2verify(SDunit *u) +{ + scsiverify(u); + return 1; +} + static int mpt2enable(SDev *sdev) { @@ -1403,14 +1608,13 @@ kproc("mpt2reset", mpt2reset, ctlr); kproc("mpt2event", mpt2event, ctlr); } - if(waserror()) print("mpt2enable: %s: %s\n", ctlr->name, up->errstr); else{ portenable(ctlr); poperror(); } - return 0; + return 1; } static int @@ -1427,8 +1631,11 @@ iunlock(ctlr); snprint(name, sizeof name, "%s (%s)", sdev->name, sdev->ifc->name); - intrdisable(p->intl, mpt2interrupt, ctlr, p->tbdf, name); - return 0; + +// must fix intrdisable +// intrdisable(p->intl, mpt2interrupt, ctlr, p->tbdf, name); + USED(p); + return 1; } static int @@ -1444,7 +1651,7 @@ runlock(&ctlr->resetlock); return SDeio; } - if(!unitonline(u)) + if(u->state != Online) error(Ebadunit); if(waserror()){ wakeup(&ctlr->reset); @@ -1473,12 +1680,14 @@ u = UNIT(ctlr, unit->subno); getunitcaps(u, buf, sizeof buf); p = seprint(p, e, "capabilities %s\n", buf); - p = seprint(p, e, "wwn %ullx\n", getle(u->wwn, 8)); + p = seprint(p, e, "wwn %.16llux\n", u->wwn); p = seprint(p, e, "type %s\n", unittype(u)); - p = seprint(p, e, "status %s\n", unitstatus(u)); p = seprint(p, e, "link %s\n", unitlink(u)); - p = seprint(p, e, "geometry %ulld %uld\n", + p = seprint(p, e, "state %s\n", unitstate(u)); + p = seprint(p, e, "geometry %llud %lud\n", unit->sectors, unit->secsize); + p = seprint(p, e, "slot %d:%d\n", u->ench, u->slot); + return p - o; } @@ -1500,7 +1709,7 @@ mpt2enable, /* enable */ mpt2disable, /* disable */ - scsiverify, /* verify */ + mpt2verify, /* verify */ scsionline, /* online */ mpt2rio, /* rio */ mpt2rctl, /* rctl */