prep for mcs locks for the 9 kernel. this is ment to bring the 9 kernel a little closer to the nix kernel. sadly, the pc kernel doesn't use the GS implementation of extern register yet, so silly hacks had to be made to mcslock.c. Notes: a few notes: - Mpl and Mreg have been imported from the nix kernel. Mpl is the type that splhi() returns. Mreg is a machine register. - sched'ing with locks is no longer allowed. even if we think it might hang the system. - procwired() now returns with the process wired. no messing about. NB: this is not compatable with the sources kernel, in the sense that a user program could depend on this behavior and be surprised when it doesn't work out with a sources kernel. - quanstro Reference: /n/atom/patch/applied/9mcsprep Date: Sat Jun 7 05:12:21 CES 2014 Signed-off-by: quanstro@quanstro.net Reviewed-by: quanstro --- /sys/src/9/alphapc/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/alphapc/dat.h Sat Jun 7 05:10:29 2014 @@ -5,6 +5,8 @@ typedef struct Label Label; typedef struct Lock Lock; typedef struct Mach Mach; +typedef int Mpl; +typedef u64int Mreg; typedef struct Notsave Notsave; typedef struct Page Page; typedef struct PCArch PCArch; --- /sys/src/9/bcm/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/bcm/dat.h Sat Jun 7 05:10:29 2014 @@ -21,6 +21,8 @@ typedef struct Memcache Memcache; typedef struct MMMU MMMU; typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct Page Page; typedef struct PhysUart PhysUart; --- /sys/src/9/bitsy/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/bitsy/dat.h Sat Jun 7 05:10:29 2014 @@ -1,25 +1,27 @@ -typedef struct Cisdat Cisdat; -typedef struct Conf Conf; +typedef struct Cisdat Cisdat; +typedef struct Conf Conf; typedef struct Confmem Confmem; -typedef struct FPU FPU; -typedef struct FPenv FPenv; -typedef struct FPsave FPsave; -typedef struct DevConf DevConf; -typedef struct Label Label; -typedef struct Lock Lock; -typedef struct MMU MMU; -typedef struct Mach Mach; -typedef struct Notsave Notsave; -typedef struct Page Page; -typedef struct PCMmap PCMmap; -typedef struct PCMslot PCMslot; +typedef struct FPU FPU; +typedef struct FPenv FPenv; +typedef struct FPsave FPsave; +typedef struct DevConf DevConf; +typedef struct Label Label; +typedef struct Lock Lock; +typedef struct MMU MMU; +typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; +typedef struct Notsave Notsave; +typedef struct Page Page; +typedef struct PCMmap PCMmap; +typedef struct PCMslot PCMslot; typedef struct PCMconftab PCMconftab; -typedef struct PhysUart PhysUart; -typedef struct PMMU PMMU; -typedef struct Proc Proc; -typedef struct Uart Uart; -typedef struct Ureg Ureg; -typedef struct Vctl Vctl; +typedef struct PhysUart PhysUart; +typedef struct PMMU PMMU; +typedef struct Proc Proc; +typedef struct Uart Uart; +typedef struct Ureg Ureg; +typedef struct Vctl Vctl; typedef long Tval; typedef ulong uintmem; --- /sys/src/9/kw/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/kw/dat.h Sat Jun 7 05:10:29 2014 @@ -7,6 +7,8 @@ typedef struct Memcache Memcache; typedef struct MMMU MMMU; typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct Page Page; typedef struct Pcidev Pcidev; --- /sys/src/9/mtx/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/mtx/dat.h Sat Jun 7 05:10:29 2014 @@ -5,6 +5,8 @@ typedef struct Label Label; typedef struct Lock Lock; typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct Page Page; typedef struct PCArch PCArch; --- /sys/src/9/omap/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/omap/dat.h Sat Jun 7 05:10:29 2014 @@ -33,7 +33,8 @@ typedef struct Memcache Memcache; typedef struct MMMU MMMU; typedef struct Mach Mach; -typedef u32int Mreg; /* Msr - bloody UART */ +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct Page Page; typedef struct PhysUart PhysUart; --- /sys/src/9/pc/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/pc/dat.h Sat Jun 7 05:10:29 2014 @@ -9,6 +9,8 @@ typedef struct Lock Lock; typedef struct MMU MMU; typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct PCArch PCArch; typedef struct Pcidev Pcidev; @@ -39,11 +41,11 @@ { ulong key; ulong sr; - ulong pc; + uintptr pc; Proc *p; Mach *m; ushort isilock; - long lockcycles; + uvlong lockcycles; }; struct Label --- /sys/src/9/pcpae/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/pcpae/dat.h Sat Jun 7 05:10:29 2014 @@ -5,8 +5,11 @@ typedef struct ISAConf ISAConf; typedef struct Label Label; typedef struct Lock Lock; +typedef struct LockEntry LockEntry; typedef struct MMU MMU; typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct PCArch PCArch; typedef struct Pcidev Pcidev; @@ -15,12 +18,12 @@ typedef struct Page Page; typedef struct PMMU PMMU; typedef struct Proc Proc; +typedef u64int PTE; typedef struct Segdesc Segdesc; typedef uvlong Tval; typedef struct Ureg Ureg; typedef struct Vctl Vctl; typedef u64int uintmem; -typedef u64int PTE; #pragma incomplete Pcidev #pragma incomplete Ureg @@ -194,6 +197,8 @@ Proc* readied; /* for runproc */ ulong schedticks; /* next forced context switch */ + LockEntry locks[8]; + int tlbfault; int tlbpurge; int pfault; @@ -203,6 +208,7 @@ int intr; int flushmmu; /* make current proc flush it's mmu state */ int ilockdepth; + uintptr ilockpc; Perf perf; /* performance counters */ ulong spuriousintr; --- /sys/src/9/ppc/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/ppc/dat.h Sat Jun 7 05:10:29 2014 @@ -6,6 +6,8 @@ typedef struct Label Label; typedef struct Lock Lock; typedef struct Mach Mach; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct PCArch PCArch; typedef struct PMMU PMMU; --- /sys/src/9/rb/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/rb/dat.h Sat Jun 7 05:10:29 2014 @@ -8,6 +8,8 @@ typedef struct Lock Lock; typedef struct Mach Mach; typedef struct MMU MMU; +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct Pcidev Pcidev; typedef struct PMMU PMMU; --- /sys/src/9/teg2/dat.h Sat Jun 7 05:10:29 2014 +++ /sys/src/9/teg2/dat.h Sat Jun 7 05:10:29 2014 @@ -33,7 +33,8 @@ typedef struct Memcache Memcache; typedef struct MMMU MMMU; typedef struct Mach Mach; -typedef u32int Mreg; /* Msr - bloody UART */ +typedef int Mpl; +typedef u32int Mreg; typedef struct Notsave Notsave; typedef struct Page Page; typedef struct Pcisiz Pcisiz; --- /sys/src/9/port/edf.c Sat Jun 7 05:10:29 2014 +++ /sys/src/9/port/edf.c Sat Jun 7 05:10:29 2014 @@ -93,7 +93,7 @@ return nil; ilock(&thelock); if((e = p->edf) && (e->flags & Admitted)){ - thelock.pc = getcallerpc(&p); + locksetpc(&thelock, getcallerpc(&p)); #ifdef EDFCYCLES edfcycles -= lcycles(); #endif --- /sys/src/9/port/mcslock.c Thu Jan 1 00:00:00 1970 +++ /sys/src/9/port/mcslock.c Sat Jun 7 05:10:29 2014 @@ -0,0 +1,220 @@ +#include "u.h" +#include "../port/lib.h" +#include "mem.h" +#include "dat.h" +#include "fns.h" + +#include "../port/edf.h" + +extern void cgascreenputr(Rune); +#define D(c) if(0)cgascreenputr((c)) + +static void +mcslock(Lock *lk, LockEntry *ql) +{ + LockEntry *pred; + + D('!'); + ql->next = nil; + ql->locked = 0; + pred = fasp(&lk->head, ql); + if(pred != nil){ + ql->locked = 1; + sfence(); /* ensure reader sees updated value */ + pred->next = ql; + while(monmwait(&ql->locked, 1) == 1) + {} + } +} + +static int +mcscanlock(Lock *lk, LockEntry *ql) +{ + D('?'); + ql->next = nil; + ql->locked = 0; + return casp(&lk->head, nil, ql); +} + +static LockEntry* +mcsunlock(Lock *lk, LockEntry *ql) +{ + D('#'); + if(ql->next != nil || !casp(&lk->head, ql, nil)){ + /* successor, wait for list to catch up */ + while(ql->next == nil) + {} + ql->next->locked = 0; + sfence(); + } + return ql; +} + +static LockEntry* +allocle(Lock *l, uintptr pc) +{ + LockEntry *a; + int i; + + a = &MACHP(m->machno)->locks[0]; + if(a->used != nil){ + i = nelem(m->locks)-1; + while(--i >= 0){ + a++; + if(a->used == nil) + break; + } + if(i < 0) + panic("allocle: need more m->locks"); + } + a->used = l; /* must be first, to claim against interrupts */ + a->pc = pc; + a->p = up; + a->m = MACHP(m->machno); + a->isilock = 0; + return a; +} + +static LockEntry* +findle(Lock *l) +{ + LockEntry *a; + + a = l->e; + if(a->used != l) + panic("findle"); + return a; +} + +int +lock(Lock *l) +{ + LockEntry *ql; + + if(up != nil) + up->nlocks++; + ql = allocle(l, getcallerpc(&l)); + mcslock(l, ql); + l->e = ql; + return 0; +} + +int +canlock(Lock *l) +{ + LockEntry *ql; + + if(up != nil) + up->nlocks++; + ql = allocle(l, getcallerpc(&l)); + if(mcscanlock(l, ql)){ + l->e = ql; + return 1; + } + ql->used = nil; + if(up != nil) + up->nlocks--; + return 0; +} + +void +unlock(Lock *l) +{ + LockEntry *ql; + + if(l->head == nil){ + print("unlock: not locked: pc %#p\n", getcallerpc(&l)); + return; + } + ql = findle(l); + if(ql->isilock) + panic("unlock of ilock: pc %#p", getcallerpc(&l)); + if(ql->p != up) + panic("unlock: up changed: pc %#p, acquired at pc %#p, lock p %#p, unlock up %#p", + getcallerpc(&l), ql->pc, ql->p, up); + mcsunlock(l, ql); + ql->used = nil; + if(up != nil && --up->nlocks == 0 && up->delaysched && islo()){ + /* + * Call sched if the need arose while locks were held + * But, don't do it from interrupt routines, hence the islo() test + */ + sched(); + } +} + +void +ilock(Lock *l) +{ + uintptr pc; + Mreg s; + LockEntry *ql; + + pc = getcallerpc(&l); + s = splhi(); + ql = allocle(l, pc); + ql->isilock = 1; + ql->sr = s; + /* the old taslock code would splx(s) to allow interrupts while waiting (if not nested) */ + mcslock(l, ql); + + l->e = ql; + m->ilockdepth++; + m->ilockpc = pc; + if(up != nil) + up->lastilock = l; +} + +void +iunlock(Lock *l) +{ + Mreg s; + LockEntry *ql; + + if(islo()) + panic("iunlock while lo: pc %#p\n", getcallerpc(&l)); + ql = findle(l); + if(!ql->isilock) + panic("iunlock of lock: pc %#p\n", getcallerpc(&l)); + if(ql->m != MACHP(m->machno)){ + panic("iunlock by mach%d, locked by mach%d: pc %#p\n", + m->machno, ql->m->machno, getcallerpc(&l)); + } + mcsunlock(l, ql); + s = ql->sr; + ql->used = nil; + m->ilockdepth--; + if(up != nil) + up->lastilock = nil; + splx(s); +} + +int +ownlock(Lock *l) +{ + int i; + + for(i = 0; i < nelem(m->locks); i++) + if(m->locks[i].used == l) + return 1; + return 0; +} + +uintptr +lockgetpc(Lock *l) +{ + LockEntry *ql; + + if(l != nil && (ql = l->e) != nil && ql->used == l) + return ql->pc; + return 0; +} + +void +locksetpc(Lock *l, uintptr pc) +{ + LockEntry *ql; + + if(l != nil && (ql = l->e) != nil && ql->used == l && ql->m == MACHP(m->machno)) + ql->pc = pc; +} --- /sys/src/9/port/qlock.c Sat Jun 7 05:10:29 2014 +++ /sys/src/9/port/qlock.c Sat Jun 7 05:10:29 2014 @@ -27,8 +27,6 @@ if(q < KADDR(4096)) print("qlock: nil qlock %#p\n", getcallerpc(&q)); - if(q->use.key == 0x55555555) - panic("qlock: q %#p, key 5*", q); lock(&q->use); rwstats.qlock++; if(!q->locked) { --- /sys/src/9/port/proc.c Sat Jun 7 05:10:29 2014 +++ /sys/src/9/port/proc.c Sat Jun 7 05:10:29 2014 @@ -90,7 +90,6 @@ up->qnext = procalloc.free; procalloc.free = up; - unlock(&palloc.avail[PGSHIFT]); unlock(&procalloc); break; @@ -123,20 +122,10 @@ * it is holding. This avoids dumb lock loops. * Don't delay if the process is Moribund. * It called sched to die. - * But do sched eventually. This avoids a missing unlock - * from hanging the entire kernel. - * But don't reschedule procs holding palloc or procalloc. - * Those are far too important to be holding while asleep. - * - * This test is not exact. There can still be a few instructions - * in the middle of taslock when a process holds a lock - * but Lock.p has not yet been initialized. */ - if(up->nlocks) - if(up->state != Moribund) - if(up->delaysched < 20 - || palloc.avail[PGSHIFT].Lock.p == up - || procalloc.Lock.p == up){ + if(up->nlocks && up->state != Moribund){ + if(up->delaysched > 1000000) + iprint("%s: delaysched: %#p %ld\n", up->text, lockgetpc(up->lastlock), up->delaysched); up->delaysched++; delayedscheds++; return; @@ -690,6 +679,11 @@ p->wired = MACHP(bm); p->mp = p->wired; + + if(m->machno != bm) + sched(); + if(m->machno != bm) + panic("procwired: %d != %d", m->machno, bm); } void @@ -701,11 +695,7 @@ pri = 0; p->basepri = pri; p->priority = pri; - if(fixed){ - p->fixedpri = 1; - } else { - p->fixedpri = 0; - } + p->fixedpri = fixed; } void --- /sys/src/9/port/portdat.h Sat Jun 7 05:10:30 2014 +++ /sys/src/9/port/portdat.h Sat Jun 7 05:10:30 2014 @@ -14,6 +14,7 @@ typedef struct Fgrp Fgrp; typedef struct Image Image; typedef struct Kbscan Kbscan; +typedef struct LockEntry LockEntry; typedef struct Log Log; typedef struct Logflag Logflag; typedef struct Mhead Mhead; @@ -72,6 +73,22 @@ { Lock; Proc *p; +}; + +struct LockEntry +{ + LockEntry* next; + uint locked; + Lock* used; + int isilock; + union { /* GAK */ + Mpl pl; + Mreg sr; + }; + /* for debugging */ + uintptr pc; + Proc* p; + Mach* m; }; struct QLock