This patch adds support for multiprocessor kernel debugging, documented in /sys/doc/acidmach.ms. Other sundry changes include: Each kernel has been updated to emit the full path on panic. This behavior can be changed by using the *nodumppath configuration option. This greatly reduces time spent debugging via ktrace. Support for *nodumpstack was extended to other kernels as well. libmach was updated to support stack generation past interrupt frames on i386. A number of small issues existed in the panic code path, which could lead to deadlock after an rdb session. These were corrected for each kernel and included minor cleanup of shutdown behavior in the event of panic. NMI handling for i386 was also reworked to support immediate debugging. Vestigal NMI setup was also removed in favor of the new interface. Notes: Sun Feb 23 01:39:33 EST 2014 geoff wow, lots of changes. Reference: /n/sources/patch/maybe/mp-debug Date: Mon Aug 12 08:35:32 CES 2013 Signed-off-by: sstallion@gmail.com Reviewed-by: geoff --- /sys/src/9/bcm/dat.h Mon Aug 12 08:25:58 2013 +++ /sys/src/9/bcm/dat.h Mon Aug 12 08:25:56 2013 @@ -224,7 +224,7 @@ Lock; int machs; /* bitmap of active CPUs */ int exiting; /* shutdown */ - int ispanic; /* shutdown in response to a panic */ + int panicking; /* panic */ }active; extern register Mach* m; /* R10 */ --- /sys/src/9/bcm/fns.h Mon Aug 12 08:26:03 2013 +++ /sys/src/9/bcm/fns.h Mon Aug 12 08:26:00 2013 @@ -103,6 +103,7 @@ extern void kexit(Ureg*); +#define _debug() #define getpgcolor(a) 0 #define kmapinval() #define countpagerefs(a, b) --- /sys/src/9/bcm/main.c Mon Aug 12 08:26:08 2013 +++ /sys/src/9/bcm/main.c Mon Aug 12 08:26:05 2013 @@ -239,6 +239,7 @@ confinit(); /* figures out amount of memory */ xinit(); uartconsinit(); + rdbinit(); screeninit(); print("\nPlan 9 from Bell Labs\n"); @@ -500,15 +501,11 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); active.machs &= ~(1<machno); active.exiting = 1; @@ -529,9 +526,10 @@ * exit kernel either on a panic or user request */ void -exit(int code) +exit(int ispanic) { - shutdown(code); + if(!ispanic) + shutdown(); splfhi(); archreboot(); } @@ -558,7 +556,7 @@ print("starting reboot..."); writeconf(); - shutdown(0); + shutdown(); /* * should be the only processor running now --- /sys/src/9/bcm/trap.c Mon Aug 12 08:26:14 2013 +++ /sys/src/9/bcm/trap.c Mon Aug 12 08:26:11 2013 @@ -480,13 +480,18 @@ uintptr l, i, v, estack; u32int *p; char *s; + extern char *kernfile; if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ iprint("dumpstack disabled\n"); return; } - iprint("ktrace /kernel/path %#.8lux %#.8lux %#.8lux # pc, sp, link\n", - ureg->pc, ureg->sp, ureg->r14); + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } + iprint("ktrace %s %#.8lux %#.8lux %#.8lux # pc, sp, link\n", + kernfile, ureg->pc, ureg->sp, ureg->r14); delay(2000); i = 0; if(up != nil && (uintptr)&l <= (uintptr)up->kstack+KSTACK) --- /sys/src/9/kw/dat.h Mon Aug 12 08:26:19 2013 +++ /sys/src/9/kw/dat.h Mon Aug 12 08:26:17 2013 @@ -193,7 +193,7 @@ Lock; int machs; /* bitmap of active CPUs */ int exiting; /* shutdown */ - int ispanic; /* shutdown in response to a panic */ + int panicking; /* panic */ }active; enum { --- /sys/src/9/kw/fns.h Mon Aug 12 08:26:24 2013 +++ /sys/src/9/kw/fns.h Mon Aug 12 08:26:22 2013 @@ -176,6 +176,7 @@ extern void kexit(Ureg*); +#define _debug() #define getpgcolor(a) 0 #define kmapinval() --- /sys/src/9/kw/main.c Mon Aug 12 08:26:31 2013 +++ /sys/src/9/kw/main.c Mon Aug 12 08:26:27 2013 @@ -370,15 +370,11 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); active.machs &= ~(1<machno); active.exiting = 1; @@ -399,9 +395,10 @@ * exit kernel either on a panic or user request */ void -exit(int code) +exit(int ispanic) { - shutdown(code); + if(!ispanic) + shutdown(); splhi(); archreboot(); } @@ -418,7 +415,7 @@ iprint("starting reboot..."); writeconf(); - shutdown(0); + shutdown(); /* * should be the only processor running now --- /sys/src/9/kw/trap.c Mon Aug 12 08:26:37 2013 +++ /sys/src/9/kw/trap.c Mon Aug 12 08:26:34 2013 @@ -567,9 +567,19 @@ { uintptr l, i, v, estack; u32int *p; + char *s; + extern char *kernfile; - iprint("ktrace /kernel/path %#.8lux %#.8lux %#.8lux # pc, sp, link\n", - ureg->pc, ureg->sp, ureg->r14); + if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ + iprint("dumpstack disabled\n"); + return; + } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } + iprint("ktrace %s %#.8lux %#.8lux %#.8lux # pc, sp, link\n", + kernfile, ureg->pc, ureg->sp, ureg->r14); delay(2000); i = 0; if(up != nil && (uintptr)&l <= (uintptr)up->kstack+KSTACK) --- /sys/src/9/mtx/dat.h Mon Aug 12 08:26:48 2013 +++ /sys/src/9/mtx/dat.h Mon Aug 12 08:26:43 2013 @@ -184,9 +184,9 @@ struct { Lock; - short machs; - short exiting; - short ispanic; + int machs; + int exiting; + int panicking; }active; /* --- /sys/src/9/mtx/fns.h Mon Aug 12 08:26:54 2013 +++ /sys/src/9/mtx/fns.h Mon Aug 12 08:26:52 2013 @@ -105,6 +105,7 @@ #define userureg(ur) (((ur)->status & MSR_PR) != 0) void watchreset(void); +#define _debug() #define waserror() (up->nerrlab++, setlabel(&up->errlab[up->nerrlab-1])) #define KADDR(a) ((void*)((ulong)(a)|KZERO)) #define PADDR(a) ((ulong)(a)&~KZERO) --- /sys/src/9/mtx/main.c Mon Aug 12 08:27:01 2013 +++ /sys/src/9/mtx/main.c Mon Aug 12 08:26:58 2013 @@ -18,6 +18,7 @@ machinit(); ioinit(); i8250console(); + rdbinit(); quotefmtinstall(); print("\nPlan 9\n"); confinit(); @@ -215,39 +216,33 @@ exit(0); } -void -exit(int ispanic) +static void +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); active.machs &= ~(1<machno); active.exiting = 1; unlock(&active); if(once) - print("cpu%d: exiting\n", m->machno); + iprint("cpu%d: exiting\n", m->machno); spllo(); for(ms = 5*1000; ms > 0; ms -= TK2MS(2)){ delay(TK2MS(2)); if(active.machs == 0 && consactive() == 0) break; } + delay(1000); +} - if(active.ispanic && m->machno == 0){ - if(cpuserver) - delay(10000); - else if(conf.monitor) - for(;;); - } - else - delay(1000); - +void +exit(int ispanic) +{ + if(!ispanic) + shutdown(); watchreset(); } --- /sys/src/9/mtx/trap.c Mon Aug 12 08:27:08 2013 +++ /sys/src/9/mtx/trap.c Mon Aug 12 08:27:04 2013 @@ -446,6 +446,8 @@ { ulong l, sl, el, v; int i; + char *s; + extern char *kernfile; l = (ulong)&l; if(up == 0){ @@ -462,14 +464,22 @@ } if(l > el || l < sl) return; - print("ktrace /kernel/path %.8lux %.8lux %.8lux\n", ureg->pc, ureg->sp, ureg->lr); + if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ + iprint("dumpstack disabled\n"); + return; + } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } + iprint("ktrace %s %.8lux %.8lux %.8lux\n", kernfile, ureg->pc, ureg->sp, ureg->lr); i = 0; for(; l < el; l += 4){ v = *(ulong*)l; if(KTZERO < v && v < (ulong)etext){ - print("%.8lux=%.8lux ", l, v); + iprint("%.8lux=%.8lux ", l, v); if(i++ == 4){ - print("\n"); + iprint("\n"); i = 0; } } --- /sys/src/9/omap/dat.h Mon Aug 12 08:27:18 2013 +++ /sys/src/9/omap/dat.h Mon Aug 12 08:27:13 2013 @@ -216,7 +216,7 @@ Lock; int machs; /* bitmap of active CPUs */ int exiting; /* shutdown */ - int ispanic; /* shutdown in response to a panic */ + int panicking; /* panic */ }active; extern register Mach* m; /* R10 */ --- /sys/src/9/omap/devcons.c Mon Aug 12 08:27:32 2013 +++ /sys/src/9/omap/devcons.c Mon Aug 12 08:27:24 2013 @@ -17,8 +17,6 @@ ulong kprintinuse; /* test and set whether /dev/kprint is open */ int iprintscreenputs = 1; -int panicking; - static struct { QLock; @@ -281,29 +279,23 @@ void panic(char *fmt, ...) { - int n, s; va_list arg; char buf[PRINTSIZE]; kprintoq = nil; /* don't try to write to /dev/kprint */ - if(panicking) + splhi(); + if(tas(&active.panicking) != 0) for(;;); - panicking = 1; + _debug(); - delay(20); - s = splhi(); strcpy(buf, "panic: "); va_start(arg, fmt); - n = vseprint(buf+strlen(buf), buf+sizeof(buf), fmt, arg) - buf; + vseprint(buf+strlen(buf), buf+sizeof(buf), fmt, arg); va_end(arg); iprint("%s\n", buf); if(consdebug) (*consdebug)(); - splx(s); - prflush(); - buf[n] = '\n'; -// putstrn(buf, n+1); // dumpstack(); exit(1); --- /sys/src/9/omap/fns.h Mon Aug 12 08:27:45 2013 +++ /sys/src/9/omap/fns.h Mon Aug 12 08:27:40 2013 @@ -163,6 +163,7 @@ extern void kexit(Ureg*); +#define _debug() #define getpgcolor(a) 0 #define kmapinval() --- /sys/src/9/omap/main.c Mon Aug 12 08:27:57 2013 +++ /sys/src/9/omap/main.c Mon Aug 12 08:27:53 2013 @@ -307,15 +307,11 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); active.machs &= ~(1<machno); active.exiting = 1; @@ -336,9 +332,10 @@ * exit kernel either on a panic or user request */ void -exit(int code) +exit(int ispanic) { - shutdown(code); + if(!ispanic) + shutdown(); splhi(); archreboot(); } @@ -387,7 +384,7 @@ print("starting reboot..."); writeconf(); - shutdown(0); + shutdown(); /* * should be the only processor running now --- /sys/src/9/omap/trap.c Mon Aug 12 08:28:06 2013 +++ /sys/src/9/omap/trap.c Mon Aug 12 08:28:02 2013 @@ -654,17 +654,22 @@ int x; uintptr l, v, i, estack; char *s; + extern char *kernfile; dumpregs(ureg); if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ iprint("dumpstack disabled\n"); return; } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } iprint("dumpstack\n"); x = 0; - x += iprint("ktrace /kernel/path %#.8lux %#.8lux %#.8lux # pc, sp, link\n", - ureg->pc, ureg->sp, ureg->r14); + x += iprint("ktrace %s %#.8lux %#.8lux %#.8lux # pc, sp, link\n", + kernfile, ureg->pc, ureg->sp, ureg->r14); delay(20); i = 0; if(up --- /sys/src/9/pc/apic.c Mon Aug 12 08:28:13 2013 +++ /sys/src/9/pc/apic.c Mon Aug 12 08:28:09 2013 @@ -447,3 +447,14 @@ else print("lapicnmidisable: no lapic\n"); } + +void +lapicnmibcast(void) +{ + if(lapicbase){ + lapicw(LapicICRHI, 0); + lapicw(LapicICRLO, LapicALLEXC|ApicNMI); + microdelay(200); + }else + print("lapicnmibcast: no lapic\n"); +} --- /sys/src/9/pc/archmp.c Mon Aug 12 08:28:22 2013 +++ /sys/src/9/pc/archmp.c Mon Aug 12 08:28:18 2013 @@ -18,6 +18,16 @@ lapicicrw(0, 0x000C0000|ApicINIT); } +static void +mpdebugothers(void) +{ + /* + * NMI all excluding self iff we have started other machs. + */ + if(conf.nmach > 1) + lapicnmibcast(); +} + static int identify(void); PCArch archmp = { @@ -31,6 +41,7 @@ .fastclock= i8253read, .timerset= lapictimerset, .resetothers= mpresetothers, +.debugothers= mpdebugothers, }; static int --- /sys/src/9/pc/dat.h Mon Aug 12 08:28:30 2013 +++ /sys/src/9/pc/dat.h Mon Aug 12 08:28:27 2013 @@ -212,6 +212,8 @@ { int machno; /* physical id of processor (KNOWN TO ASSEMBLY) */ ulong splpc; /* pc of last caller to splhi */ + Ureg* dbgreg; /* registers for debugging this processor */ + ulong dbgsp; /* sp for debugging this processor */ ulong* pdb; /* page directory base for this processor (va) */ Tss* tss; /* tss for this processor */ @@ -284,9 +286,9 @@ Lock; int machs; /* bitmap of active CPUs */ int exiting; /* shutdown */ - int ispanic; /* shutdown in response to a panic */ - int thunderbirdsarego; /* lets the added processors continue to schedinit */ + int panicking; /* panic */ int rebooting; /* just idle cpus > 0 */ + int thunderbirdsarego; /* lets the added processors continue to schedinit */ }active; /* @@ -312,6 +314,7 @@ void (*timerset)(uvlong); void (*resetothers)(void); /* put other cpus into reset */ + void (*debugothers)(void); }; /* cpuid instruction result register bits */ --- /sys/src/9/pc/l.s Mon Aug 12 08:28:38 2013 +++ /sys/src/9/pc/l.s Mon Aug 12 08:28:33 2013 @@ -1001,18 +1001,7 @@ MOVL $0, AX /* return 0 */ RET -/* - * Attempt at power saving. -rsc - */ TEXT halt(SB), $0 - CLI - CMPL nrdy(SB), $0 - JEQ _nothingready - STI - RET - -_nothingready: - STI HLT RET --- /sys/src/9/pc/main.c Mon Aug 12 08:28:52 2013 +++ /sys/src/9/pc/main.c Mon Aug 12 08:28:45 2013 @@ -97,6 +97,7 @@ options(); ioinit(); i8250console(); + rdbinit(); quotefmtinstall(); screeninit(); @@ -728,19 +729,15 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); /* * setting exiting will make hzclock() on each processor call exit(0), - * which calls shutdown(0) and arch->reset(), which on mp systems is + * which calls shutdown() and arch->reset(), which on mp systems is * mpshutdown, which idles non-bootstrap cpus and returns on bootstrap * processors (to permit a reboot). clearing our bit in machs avoids * calling exit(0) from hzclock() on this processor. @@ -759,17 +756,6 @@ if(active.machs == 0 && consactive() == 0) break; } - - if(active.ispanic){ - if(!cpuserver) - for(;;) - halt(); - if(getconf("*debug")) - delay(5*60*1000); - else - delay(10000); - }else - delay(1000); } void @@ -800,7 +786,7 @@ lock(&active); active.rebooting = 1; unlock(&active); - shutdown(0); + shutdown(); if(arch->resetothers) arch->resetothers(); delay(20); @@ -844,11 +830,18 @@ (*f)(PADDR(entry), PADDR(code), size); } +void +_debug(void) +{ + if(arch->debugothers) + arch->debugothers(); +} void exit(int ispanic) { - shutdown(ispanic); + if(!ispanic) + shutdown(); arch->reset(); } --- /sys/src/9/pc/mkfile Mon Aug 12 08:29:00 2013 +++ /sys/src/9/pc/mkfile Mon Aug 12 08:28:57 2013 @@ -61,6 +61,12 @@ $DEVS\ $PORT\ +ACID=\ + ../port/chan.c\ + ../port/proc.c\ + ../port/segment.c\ + trap.c\ + LIB=\ /$objtype/lib/libmemlayer.a\ /$objtype/lib/libmemdraw.a\ @@ -154,8 +160,8 @@ echo '};' } > $target -acid:V: - 8c -a -w -I. i8253.c>acid +acid: $ACID errstr.h + $CC -a -w -I. $ACID > acid %.checkether:VQ: for (i in ether*.c){ --- /sys/src/9/pc/mp.h Mon Aug 12 08:29:04 2013 +++ /sys/src/9/pc/mp.h Mon Aug 12 08:29:02 2013 @@ -223,6 +223,7 @@ extern int lapicisr(int); extern void lapicnmidisable(void); extern void lapicnmienable(void); +extern void lapicnmibcast(void); extern void lapiconline(void); extern void lapicspurious(Ureg*, void*); extern void lapicstartap(Apic*, int); --- /sys/src/9/pc/trap.c Mon Aug 12 08:29:13 2013 +++ /sys/src/9/pc/trap.c Mon Aug 12 08:29:09 2013 @@ -160,22 +160,6 @@ iunlock(&vctllock); } -static void -nmienable(void) -{ - int x; - - /* - * Hack: should be locked with NVRAM access. - */ - outb(0x70, 0x80); /* NMI latch clear */ - outb(0x70, 0); - - x = inb(0x61) & 0x07; /* Enable NMI */ - outb(0x61, 0x08|x); - outb(0x61, x); -} - /* * Minimal trap setup. Just enough so that we can panic * on traps (bugs) during kernel initialization. @@ -223,7 +207,6 @@ trapenable(VectorPF, fault386, 0, "fault386"); trapenable(Vector2F, doublefault, 0, "doublefault"); trapenable(Vector15, unexpected, 0, "unexpected"); - nmienable(); addarchfile("irqalloc", 0444, irqallocread, nil); trapinited = 1; @@ -413,16 +396,22 @@ kexit(ureg); return; } - else{ - if(vno == VectorNMI){ + else if(vno == VectorNMI){ + if(active.panicking){ /* - * Don't re-enable, it confuses the crash dumps. - nmienable(); + * Use of m->dbgsp avoids stack confusion + * caused by writing the address of the SP to + * the top of the stack. */ - iprint("cpu%d: NMI PC %#8.8lux\n", m->machno, ureg->pc); - while(m->machno != 0) - ; + m->dbgreg = ureg; + m->dbgsp = (ulong)&ureg->sp; + + for(;;) + halt(); } + panic("NMI"); + } + else{ dumpregs(ureg); if(!user){ ureg->sp = (ulong)&ureg->sp; @@ -518,15 +507,20 @@ extern ulong etext; int x; char *s; + extern char *kernfile; if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ iprint("dumpstack disabled\n"); return; } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } iprint("dumpstack\n"); x = 0; - x += iprint("ktrace /kernel/path %.8lux %.8lux <pc, ureg->sp); + x += iprint("ktrace %s %.8lux %.8lux <pc, ureg->sp); i = 0; if(up && (uintptr)&l >= (uintptr)up->kstack @@ -559,20 +553,6 @@ if(i) iprint("\n"); iprint("EOF\n"); - - if(ureg->trap != VectorNMI) - return; - - i = 0; - for(l = (uintptr)&l; l < estack; l += sizeof(uintptr)){ - iprint("%.8p ", *(uintptr*)l); - if(++i == 8){ - i = 0; - iprint("\n"); - } - } - if(i) - iprint("\n"); } void --- /sys/src/9/pcboot/dat.h Mon Aug 12 08:29:21 2013 +++ /sys/src/9/pcboot/dat.h Mon Aug 12 08:29:18 2013 @@ -284,9 +284,9 @@ Lock; int machs; /* bitmap of active CPUs */ int exiting; /* shutdown */ - int ispanic; /* shutdown in response to a panic */ - int thunderbirdsarego; /* lets the added processors continue to schedinit */ + int panicking; /* panic */ int rebooting; /* just idle cpus > 0 */ + int thunderbirdsarego; /* lets the added processors continue to schedinit */ }active; /* --- /sys/src/9/pcboot/fns.h Mon Aug 12 08:29:28 2013 +++ /sys/src/9/pcboot/fns.h Mon Aug 12 08:29:25 2013 @@ -62,3 +62,5 @@ /* libip */ int equivip4(uchar *, uchar *); int equivip6(uchar *, uchar *); + +#define _debug() --- /sys/src/9/pcboot/main.c Mon Aug 12 08:29:36 2013 +++ /sys/src/9/pcboot/main.c Mon Aug 12 08:29:33 2013 @@ -502,19 +502,15 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); /* * setting exiting will make hzclock() on each processor call exit(0), - * which calls shutdown(0) and arch->reset(), which on mp systems is + * which calls shutdown() and arch->reset(), which on mp systems is * mpshutdown, which idles non-bootstrap cpus and returns on bootstrap * processors (to permit a reboot). clearing our bit in machs avoids * calling exit(0) from hzclock() on this processor. @@ -533,17 +529,6 @@ if(active.machs == 0 && consactive() == 0) break; } - - if(active.ispanic){ - if(!cpuserver) - for(;;) - halt(); - if(getconf("*debug")) - delay(5*60*1000); - else - delay(10000); - }else - delay(1000); } void @@ -576,7 +561,7 @@ lock(&active); active.rebooting = 1; unlock(&active); - shutdown(0); + shutdown(); if(arch->resetothers) arch->resetothers(); delay(20); @@ -621,12 +606,11 @@ (*f)(PADDR(entry), PADDR(code), size); } - void exit(int ispanic) { - shutdown(ispanic); - spllo(); + if(!ispanic) + shutdown(); arch->reset(); } --- /sys/src/9/pcboot/trap.c Mon Aug 12 08:29:44 2013 +++ /sys/src/9/pcboot/trap.c Mon Aug 12 08:29:41 2013 @@ -503,6 +503,7 @@ extern ulong etext; int x; char *s; + extern char *kernfile; if (!Dumpstack) { print("no stack dump\n"); @@ -512,10 +513,14 @@ iprint("dumpstack disabled\n"); return; } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } iprint("dumpstack\n"); x = 0; - x += iprint("ktrace /kernel/path %.8lux %.8lux <pc, ureg->sp); + x += iprint("ktrace %s %.8lux %.8lux <pc, ureg->sp); i = 0; if(up && (uintptr)&l >= (uintptr)up->kstack @@ -548,20 +553,6 @@ if(i) iprint("\n"); iprint("EOF\n"); - - if(ureg->trap != VectorNMI) - return; - - i = 0; - for(l = (uintptr)&l; l < estack; l += sizeof(uintptr)){ - iprint("%.8p ", *(uintptr*)l); - if(++i == 8){ - i = 0; - iprint("\n"); - } - } - if(i) - iprint("\n"); } void --- /sys/src/9/port/devcons.c Mon Aug 12 08:29:52 2013 +++ /sys/src/9/port/devcons.c Mon Aug 12 08:29:48 2013 @@ -18,8 +18,6 @@ ulong kprintinuse; /* test and set whether /dev/kprint is open */ int iprintscreenputs = 1; -int panicking; - static struct { QLock; @@ -272,28 +270,23 @@ void panic(char *fmt, ...) { - int n, s; va_list arg; char buf[PRINTSIZE]; kprintoq = nil; /* don't try to write to /dev/kprint */ - if(panicking) + splhi(); + if(tas(&active.panicking) != 0) for(;;); - panicking = 1; + _debug(); - s = splhi(); strcpy(buf, "panic: "); va_start(arg, fmt); - n = vseprint(buf+strlen(buf), buf+sizeof(buf), fmt, arg) - buf; + vseprint(buf+strlen(buf), buf+sizeof(buf), fmt, arg); va_end(arg); iprint("%s\n", buf); if(consdebug) (*consdebug)(); - splx(s); - prflush(); - buf[n] = '\n'; - putstrn(buf, n+1); dumpstack(); exit(1); @@ -460,9 +453,7 @@ print("consdebug now %#p\n", consdebug); return; case 'D': - if(consdebug == nil) - consdebug = rdb; - consdebug(); + panic("consdebug"); return; case 'p': x = spllo(); --- /sys/src/9/port/edf.c Mon Aug 12 08:29:58 2013 +++ /sys/src/9/port/edf.c Mon Aug 12 08:29:55 2013 @@ -135,11 +135,10 @@ deadlineintr(Ureg*, Timer *t) { /* Proc reached deadline */ - extern int panicking; Proc *p; void (*pt)(Proc*, int, vlong); - if(panicking || active.exiting) + if(active.panicking || active.exiting) return; p = t->ta; @@ -211,10 +210,9 @@ releaseintr(Ureg*, Timer *t) { Proc *p; - extern int panicking; Schedq *rq; - if(panicking || active.exiting) + if(active.panicking || active.exiting) return; p = t->ta; --- /sys/src/9/port/mkdevc Mon Aug 12 08:30:02 2013 +++ /sys/src/9/port/mkdevc Mon Aug 12 08:30:00 2013 @@ -186,6 +186,7 @@ printf "char* conffile = \"%s/%s\";\n", pwd, ARGV[1]; printf "ulong kerndate = KERNDATE;\n"; + printf "char* kernfile = \"%s/9%s\";\n", pwd, ARGV[1]; exit }' $* --- /sys/src/9/port/portfns.h Mon Aug 12 08:30:08 2013 +++ /sys/src/9/port/portfns.h Mon Aug 12 08:30:05 2013 @@ -52,6 +52,7 @@ void cupdate(Chan*, uchar*, int, vlong); void cwrite(Chan*, uchar*, int, vlong); ulong dbgpc(Proc*); +void _debug(void); long decref(Ref*); int decrypt(void*, void*, int); void delay(int); @@ -285,6 +286,7 @@ void randominit(void); ulong randomread(void*, ulong); void rdb(void); +void rdbinit(void); void readn(Chan *, void *, long); int readnum(ulong, char*, ulong, ulong, int); int readstr(ulong, char*, ulong, char*); --- /sys/src/9/port/portmkfile Mon Aug 12 08:30:13 2013 +++ /sys/src/9/port/portmkfile Mon Aug 12 08:30:10 2013 @@ -30,12 +30,12 @@ mk 'CONF='$i clean:V: - rm -f *.[$OS] *.root.s *.rootc.c cfs.h fs.h init.h conf.h reboot.h reboot.list *.out *.m errstr.h + rm -f *.[$OS] *acid *.root.s *.rootc.c cfs.h fs.h init.h conf.h reboot.h reboot.list *.out *.m errstr.h for(i in $CONFLIST $CRAPLIST) mk $i.clean %.clean:V: - rm -f $stem.c [9bz]$stem [9bz]$stem.gz s[9bz]$stem boot$stem.* *.acid + rm -f $stem.c [9bz]$stem [9bz]$stem.gz s[9bz]$stem boot$stem.* nuke:V: clean rm -f ../boot/libboot.a$O *.elf *.rr --- /sys/src/9/port/rdb.c Mon Aug 12 08:30:17 2013 +++ /sys/src/9/port/rdb.c Thu Dec 5 21:53:16 2013 @@ -30,8 +30,10 @@ int i, c; for(;;){ - for(i=0; istatus & MSR_PR) != 0) #define waserror() (up->nerrlab++, setlabel(&up->errlab[up->nerrlab-1])) #define KADDR(a) ((void*)((ulong)(a)|KZERO)) --- /sys/src/9/ppc/main.c Mon Aug 12 08:30:42 2013 +++ /sys/src/9/ppc/main.c Mon Aug 12 08:30:39 2013 @@ -74,6 +74,7 @@ clockinit(); timerinit(); console(); + rdbinit(); quotefmtinstall(); printinit(); cpuidprint(); @@ -242,39 +243,32 @@ ready(p); } -void -exit(int ispanic) +static void +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); active.machs &= ~(1<machno); active.exiting = 1; unlock(&active); if(once) - print("cpu%d: exiting\n", m->machno); + iprint("cpu%d: exiting\n", m->machno); spllo(); for(ms = 5*1000; ms > 0; ms -= TK2MS(2)){ delay(TK2MS(2)); if(active.machs == 0 && consactive() == 0) break; } + delay(1000); +} - if(active.ispanic && m->machno == 0){ - if(cpuserver) - delay(10000); - else if(conf.monitor) - for(;;); - } - else - delay(1000); - +void +exit(int) +{ + shutdown(); } /* --- /sys/src/9/ppc/trap.c Mon Aug 12 08:30:52 2013 +++ /sys/src/9/ppc/trap.c Mon Aug 12 08:30:48 2013 @@ -505,6 +505,8 @@ { ulong l, sl, el, v; int i; + char *s; + extern char *kernfile; l = (ulong)&l; if(up == 0){ @@ -521,12 +523,20 @@ } if(l > el || l < sl) return; - print("ktrace /kernel/path %.8lux %.8lux %.8lux\n", ureg->pc, ureg->sp, ureg->lr); + if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ + iprint("dumpstack disabled\n"); + return; + } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } + iprint("ktrace %s %.8lux %.8lux %.8lux\n", kernfile, ureg->pc, ureg->sp, ureg->lr); i = 0; for(; l < el; l += 4){ v = *(ulong*)l; if(KTZERO < v && v < (ulong)etext){ - print("%.8lux=%.8lux ", l, v); + iprint("%.8lux=%.8lux ", l, v); if(i++ == 4){ print("\n"); i = 0; --- /sys/src/9/rb/dat.h Mon Aug 12 08:31:00 2013 +++ /sys/src/9/rb/dat.h Mon Aug 12 08:30:58 2013 @@ -213,7 +213,7 @@ Lock; long machs; /* bitmap of processors */ short exiting; - int ispanic; + ulong panicking; }active; extern KMap kpte[]; --- /sys/src/9/rb/fns.h Mon Aug 12 08:31:08 2013 +++ /sys/src/9/rb/fns.h Mon Aug 12 08:31:06 2013 @@ -138,6 +138,7 @@ #define PTR2UINT(p) ((uintptr)(p)) #define UINT2PTR(i) ((void*)(i)) +#define _debug() #define waserror() (up->nerrlab++, setlabel(&up->errlab[up->nerrlab-1])) #define KADDR(a) ((void*)((ulong)(a)|KSEG0)) --- /sys/src/9/rb/main.c Mon Aug 12 08:31:20 2013 +++ /sys/src/9/rb/main.c Mon Aug 12 08:31:15 2013 @@ -427,19 +427,15 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; ilock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); /* * setting exiting will make hzclock() on each processor call exit(0), - * which calls shutdown(0) and idles non-bootstrap cpus and returns + * which calls shutdown() and idles non-bootstrap cpus and returns * on bootstrap processors (to permit a reboot). clearing our bit * in machs avoids calling exit(0) from hzclock() on this processor. */ @@ -483,7 +479,7 @@ if (m->machno != 0) print("on cpu%d (not 0)!\n", m->machno); - shutdown(0); + shutdown(); /* * should be the only processor running now --- /sys/src/9/rb/trap.c Mon Aug 12 08:31:30 2013 +++ /sys/src/9/rb/trap.c Mon Aug 12 08:31:26 2013 @@ -582,14 +582,25 @@ static void _dumpstack(Ureg *ureg) { + char *s; ulong l, v, top, i; extern ulong etext; + extern char *kernfile; if(up == 0) return; + if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ + iprint("dumpstack disabled\n"); + return; + } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } + iprint("dumpstack\n"); - print("ktrace /kernel/path %.8lux %.8lux %.8lux\n", - ureg->pc, ureg->sp, ureg->r31); + print("ktrace %s %.8lux %.8lux %.8lux\n", + kernfile, ureg->pc, ureg->sp, ureg->r31); top = (ulong)up->kstack + KSTACK; i = 0; for(l=ureg->sp; l < top; l += BY2WD) { --- /sys/src/9/teg2/devcons.c Mon Aug 12 08:31:38 2013 +++ /sys/src/9/teg2/devcons.c Mon Aug 12 08:31:34 2013 @@ -18,8 +18,6 @@ ulong kprintinuse; /* test and set whether /dev/kprint is open */ int iprintscreenputs = 1; -int panicking; - static struct { QLock; @@ -282,33 +280,25 @@ void panic(char *fmt, ...) { - int n, s; va_list arg; char buf[PRINTSIZE]; kprintoq = nil; /* don't try to write to /dev/kprint */ - if(panicking) + splhi(); + if(tas(&active.panicking) != 0) for(;;); - panicking = 1; + _debug(); - s = splhi(); - delay(2000); - strcpy(buf, "\npanic: "); + strcpy(buf, "panic: "); va_start(arg, fmt); - n = vseprint(buf+strlen(buf), buf+sizeof(buf), fmt, arg) - buf; + vseprint(buf+strlen(buf), buf+sizeof(buf), fmt, arg); va_end(arg); iprint("%s\n", buf); if(consdebug) (*consdebug)(); - splx(s); - prflush(); - USED(n); -// buf[n] = '\n'; -// putstrn(buf, n+1); /* redundant */ // dumpstack(); - delay(2000); exit(1); } --- /sys/src/9/teg2/dat.h Mon Aug 12 08:31:47 2013 +++ /sys/src/9/teg2/dat.h Mon Aug 12 08:31:44 2013 @@ -249,7 +249,7 @@ int wfi; /* bitmap of CPUs in WFI state */ int stopped; /* bitmap of CPUs stopped */ int exiting; /* shutdown */ - int ispanic; /* shutdown in response to a panic */ + int panicking; /* panic */ int thunderbirdsarego; /* lets the added processors continue to schedinit */ }active; --- /sys/src/9/teg2/fns.h Mon Aug 12 08:31:59 2013 +++ /sys/src/9/teg2/fns.h Mon Aug 12 08:31:55 2013 @@ -219,6 +219,7 @@ extern void kexit(Ureg*); +#define _debug() #define getpgcolor(a) 0 #define kmapinval() --- /sys/src/9/teg2/main.c Mon Aug 12 08:32:14 2013 +++ /sys/src/9/teg2/main.c Mon Aug 12 08:32:06 2013 @@ -486,19 +486,15 @@ } static void -shutdown(int ispanic) +shutdown(void) { int ms, once; lock(&active); - if(ispanic) - active.ispanic = ispanic; - else if(m->machno == 0 && (active.machs & (1<machno)) == 0) - active.ispanic = 0; once = active.machs & (1<machno); /* * setting exiting will make hzclock() on each processor call exit(0), - * which calls shutdown(0) and idles non-bootstrap cpus and returns + * which calls shutdown() and idles non-bootstrap cpus and returns * on bootstrap processors (to permit a reboot). clearing our bit * in machs avoids calling exit(0) from hzclock() on this processor. */ @@ -527,9 +523,10 @@ * exit kernel either on a panic or user request */ void -exit(int code) +exit(int ispanic) { - shutdown(code); + if(!ispanic) + shutdown(); splhi(); if (m->machno == 0) archreboot(); @@ -606,7 +603,7 @@ for (want = 0, cpu = 1; cpu < navailcpus; cpu++) want |= 1 << cpu; active.stopped = 0; - shutdown(0); + shutdown(); for (ms = 15*1000; ms > 0 && active.stopped != want; ms -= 10) delay(10); delay(20); --- /sys/src/9/teg2/trap.c Mon Aug 12 08:32:34 2013 +++ /sys/src/9/teg2/trap.c Mon Aug 12 08:32:27 2013 @@ -949,18 +949,23 @@ int x; uintptr l, v, i, estack; char *s; + extern char *kernfile; dumpregs(ureg); if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){ iprint("dumpstack disabled\n"); return; } + if((s = getconf("*nodumppath")) != nil && strcmp(s, "0") != 0){ + if(s = strrchr(kernfile, '/')) + kernfile = ++s; + } delay(1000); iprint("dumpstack\n"); x = 0; - x += iprint("ktrace /kernel/path %#.8lux %#.8lux %#.8lux # pc, sp, link\n", - ureg->pc, ureg->sp, ureg->r14); + x += iprint("ktrace %s %#.8lux %#.8lux %#.8lux # pc, sp, link\n", + kernfile, ureg->pc, ureg->sp, ureg->r14); delay(20); i = 0; if(up --- /sys/src/libmach/8db.c Mon Aug 12 08:32:49 2013 +++ /sys/src/libmach/8db.c Mon Aug 12 08:32:42 2013 @@ -94,26 +94,23 @@ static int i386trace(Map *map, uvlong pc, uvlong sp, uvlong link, Tracer trace) { - int i; - uvlong osp; + int kernel, i; Symbol s, f; USED(link); i = 0; - osp = 0; while(findsym(pc, CTEXT, &s)) { - if (osp == sp) - break; - osp = sp; + kernel = pc & mach->ktmask; - if(strcmp(STARTSYM, s.name) == 0 || strcmp(PROFSYM, s.name) == 0) + if(!kernel && (strcmp(STARTSYM, s.name) == 0 || strcmp(PROFSYM, s.name) == 0)) break; if(pc != s.value) { /* not at first instruction */ if(findlocal(&s, FRAMENAME, &f) == 0) break; sp += f.value-mach->szaddr; - } + } else if(kernel && strcmp("forkret", s.name) == 0) + sp += 15 * mach->szaddr; /* pop interrupt frame */ if (geta(map, sp, &pc) < 0) break; @@ -123,6 +120,8 @@ (*trace)(map, pc, sp, &s); sp += mach->szaddr; + if(kernel && strcmp("forkret", s.name) == 0) + sp += 2 * mach->szaddr; /* pop iret cs, eflags */ if(++i > 1000) break; @@ -133,25 +132,35 @@ static uvlong i386frame(Map *map, uvlong addr, uvlong pc, uvlong sp, uvlong link) { + int kernel; Symbol s, f; USED(link); while (findsym(pc, CTEXT, &s)) { - if(strcmp(STARTSYM, s.name) == 0 || strcmp(PROFSYM, s.name) == 0) + kernel = pc & mach->ktmask; + + if(!kernel && (strcmp(STARTSYM, s.name) == 0 || strcmp(PROFSYM, s.name) == 0)) break; if(pc != s.value) { /* not first instruction */ if(findlocal(&s, FRAMENAME, &f) == 0) break; sp += f.value-mach->szaddr; - } + } else if(kernel && strcmp("forkret", s.name) == 0) + sp += 15 * mach->szaddr; /* pop interrupt frame */ if (s.value == addr) return sp; if (geta(map, sp, &pc) < 0) break; + + if(pc == 0) + break; + sp += mach->szaddr; + if(kernel && strcmp("forkret", s.name) == 0) + sp += 2 * mach->szaddr; /* pop iret cs, eflags */ } return 0; } --- /sys/lib/acid/mach Thu Jan 1 00:00:00 1970 +++ /sys/lib/acid/mach Mon Aug 12 08:32:53 2013 @@ -0,0 +1,173 @@ +// mp kernel support (386 only) +// +// usage: acid -k -l kernel -l mach +// + +MACHSIZE = BY2PG; + +defn machno() { + print(m->machno, "\n"); +} + +defn mach(mach) { + complex Mach mach; + + print(mach\X, " ", mach.machno, " proc "); + if mach.proc == 0 then + print(0\X, "\n"); + else + proc(mach.proc); +} + +defn machs() { + local i; + + i = 0; + loop 1, conf.nmach do { + mach(machp[i]); + i = i+1; + } +} + +defn machaddr(mach, addr) { + if addr < *m || addr >= *m+MACHSIZE then + return addr; + addr = addr & (MACHSIZE-1); + return mach+addr; +} + +defn machureg(mach) { + complex Mach mach; + + // dbgreg 0 is valid; current register state + return (Ureg)machaddr(mach, mach.dbgreg); +} + +defn machpc(mach) { + complex Mach mach; + local ureg; + + ureg = machureg(mach); + return ureg.pc; +} + +defn machsp(mach) { + complex Mach mach; + local ureg; + + if mach.dbgsp == 0 then { + ureg = machureg(mach); + return ureg.sp; + } else + return mach.dbgsp; +} + +defn machgpr(mach) { + complex Mach mach; + local ureg; + + ureg = machureg(mach); + print("AX\t", ureg.ax\X, " BX\t", ureg.bx\X, " CX\t", ureg.cx\X, " DX\t", ureg.dx\X, "\n"); + print("DI\t", ureg.di\X, " SI\t", ureg.si\X, " BP\t", ureg.bp\X, "\n"); +} + +defn machspr(mach) { + complex Mach mach; + local ureg, sp; + + ureg = machureg(mach); + sp = machsp(mach); + print("PC\t", ureg.pc\X, " ", fmt(ureg.pc, 'a'), " "); + pfl(ureg.pc); + print("SP\t", sp\X, " ECODE ", ureg.ecode\X, " EFLAG ", ureg.flags\X, "\n"); + print("CS\t", ureg.cs\X, " DS\t ", ureg.ds\X, " SS\t", ureg.ss\X, "\n"); + print("GS\t", ureg.gs\X, " FS\t ", ureg.fs\X, " ES\t", ureg.es\X, "\n"); + + print("TRAP\t", ureg.trap\X, " ", reason(ureg.trap), "\n"); +} + +defn machregs(mach) { + machspr(mach); + machgpr(mach); +} + +defn machstk(mach) { + _stk(machpc(mach), machaddr(mach, machsp(mach)), 0, 0); +} + +defn machlstk(mach) { + _stk(machpc(mach), machaddr(mach, machsp(mach)), 0, 1); +} + +defn machstacks() { + local i; + + i = 0; + loop 1, conf.nmach do { + print("=========================================================\n"); + mach(machp[i]); + machstk(machp[i]); + i = i+1; + } +} + +// a function of last resort... +defn machunwind(mach) { + complex Mach mach; + local sp, se; + + sp = machsp(mach); + if mach.proc != 0 then + se = sp | (KSTACK-1); + else + se = sp | (MACHSIZE-1); + dump(machaddr(mach, sp), (se-sp)/4, "a"); +} + +// kernel procstk assumes gotolabel; breaks on mp +defn procstk(p) { + complex Proc p; + + if p.mach != 0 then + _stk(machpc(p.mach), machaddr(p.mach, machsp(p.mach)), 0, 0); + else if p.state != 0 then + _stk(gotolabel, *(p.sched), 0, 0); +} + +defn proclstk(p) { + complex Proc p; + + if p.mach != 0 then + _stk(machpc(p.mach), machaddr(p.mach, machsp(p.mach)), 0, 1); + else if p.state != 0 then + _stk(gotolabel, *(p.sched), 0, 1); +} + +// builtin reason ignores status; breaks on mp +defn reason(status) { + if status == 0 then return "divide error"; + else if status == 1 then return "debug exception"; + else if status == 2 then return "nonmaskable interrupt"; + else if status == 3 then return "breakpoint"; + else if status == 4 then return "overflow"; + else if status == 5 then return "bounds check"; + else if status == 6 then return "invalid opcode"; + else if status == 7 then return "coprocessor not available"; + else if status == 8 then return "double fault"; + else if status == 9 then return "coprocessor segment overrun"; + else if status == 10 then return "invalid TSS"; + else if status == 11 then return "segment not present"; + else if status == 12 then return "stack exception"; + else if status == 13 then return "general protection violation"; + else if status == 14 then return "page fault"; + else if status == 16 then return "coprocessor error"; + else if status == 17 then return "alignment check"; + else if status == 18 then return "machine check"; + else if status == 19 then return "SIMD exception"; + else if status == 64 then return "system call"; + + // emulate i386except brokenness: + return "exception "+itoa(status); +} + +print("/sys/lib/acid/mach"); --- /sys/doc/acidmach.ms Thu Jan 1 00:00:00 1970 +++ /sys/doc/acidmach.ms Mon Aug 12 08:32:59 2013 @@ -0,0 +1,594 @@ +.TL +Multiprocessor Kernel Debugging Using Acid +.AU +Steven Stallion +stallion@coraid.com +.SH +Introduction +.LP +This document describes a method of debugging multiprocessor kernels +using +.I acid (1). +While in-situ debugging is yet to be fully realized, the mechanism +detailed herein provides superior postmortem visibility over previous +approaches. +It is expected that this body of work will result in a debugging +environment supportive of both in-situ and postmortem analysis in the +future. +This approach encourages use of existing tools and culminates in a new +.I acid +library, which provides a foundation for discovery and analysis of +multiprocessor related defects. +.LP +The reader should not consider this document an introductory text but +merely an exposition of completed work [1]. +.SH +Mechanism +.LP +A panicking kernel will issue a nonmaskable inter-processor interrupt +(IPI) to all processors, excluding the caller. +Upon receiving the trap, each +.CW Mach +records a pointer to the current +.CW Ureg +structure and the stack pointer to the +.CW dbgreg +and +.CW dbgsp +members, respectively. +After which, the processor enters a halt state with interrupts +disabled. +The debugging processor will maintain a value of zero for both members +as a hint to use current register state at debug time. +Use of a nonmaskable interrupt yields the ability to record state +regardless of processor context; interrupt processing no longer +interferes with stack generation. +.LP +If +.CW consdebug +is non-nil, +.CW panic +will branch to a debug routine, typically +.CW rdb . +The system is then prepared to accept remote debugging commands, +ostensibly via +.I rdbfs (4). +To simplify setting +.CW consdebug +at boot, a new configuration variable, named +.CW rdb +was introduced to obviate the need for the +.CW ^T^Td +control sequence (see +.I cons (3)). +.LP +To further simplify remote debugging, +.CW rdb +was modified to return if a serial break is received. +.I Rdbfs +was also updated to send a serial break upon receiving a kill +message, allowing +.I acid +to terminate the debugging session and transitively reboot the system +using the +.CW kill +builtin function. +.SH +Using the Library Functions +.LP +The +.CW mach +library provides a number of functions the user may employ to examine +multiprocessor state. +.CW Mach +was developed as a companion to the +.CW kernel +library. +As such, both libraries must be defined on the command line when +starting +.I acid . +Normal +.CW kernel +initialization rules apply; +.CW kinit +must be called to establish the proper mapping for the kernel prior to +calling functions defined in +.CW mach . +.LP +The following example attaches to a remote kernel with +.I rdbfs +and gathers basic information using the +.CW mach +library: +.P1 +% rdbfs /mnt/consoles/sys +attach /mnt/consoles/sys +% acid -k -l kernel -l mach -r 9pc +9pc:386 plan 9 boot image +/sys/lib/acid/port +/sys/lib/acid/386 +/sys/lib/acid/kernel +/sys/lib/acid/mach +acid: kinit() + rc("cd /sys/src/9/pc; mk proc.acid") + include("/sys/src/9/pc/proc.acid") +acid: rc("cd /sys/src/9/pc; mk proc.acid") +8c -FTVw -a -I. ../port/proc.c >proc.acid +acid: include("/sys/src/9/pc/proc.acid") +acid: machno() +0 +acid: machs() +0x80017000 0 up 0x00000000 +0x8003d000 1 up 0x00000000 +.P2 +.SH +Library Functions +.LP +The +.CW mach +library is located in the directory +.CW /sys/lib/acid . +As with other libraries, these functions may be overridden, +personalized, or added to by code defined in +.CW $home/lib/acid . +The implementation of these functions can be examined using the +.CW whatis +operator and then modified during debugging sessions. +.de Ip +.KS +.LP +.tl '\f2\\$1\fP\ \ \f(CW\\$2(\f2\\$3\f(CW)\f1''\\$4' +.IP +.. +.de Ex +.KE +.KS +.IP +.ft CW +.ta 4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +4n +.nf +.in +4n +.br +.. +.de Ee +.fi +.ft 1 +.br +.KE +.. +.\" +.\" +.\" +.Ip \f(CW{}\fB mach Mach "Print summary for Mach +.CW mach +prints a one line summary for the given +.I Mach . +The first printed column is the address of +.I Mach , +followed by the +.CW machno , +and ends with a summary of the currently scheduled process (see +.CW proc ). +.Ex +acid: mach(machp[1]) +0x8003d000 1 up 0x00000000 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machgpr Mach "Print general purpose registers for Mach +.CW machgpr +prints the general purpose registers for the given +.I Mach . +While +.CW machgpr +may be used interactively, this function is typically only called by +.CW machregs . +.Ex +acid: machgpr(machp[1]) +AX 0xc9121c18 BX 0x8025f34c CX 0x000000d8 DX 0x00000000 +DI 0x80279b44 SI 0x0005d203 BP 0x80016000 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machlstk Mach "Stack trace with local variables for Mach +.CW machlstk +produces a long format stack trace for the given +.I Mach , +similar to +.CW lstk . +Unlike +.CW lstk , +the +.CW : +operator should not be used to address variables on processors other +than the debugging +.CW Mach +(see +.CW machno ). +This is a limitation imposed by the current implementation of +.I acid +and +.I libmach . +.Ex +acid: machlstk(machp[1]) +runproc()+0x53 proc.c:531 + start=0xc9121c18 + p=0x80279948 + rq=0x8025f34c + i=0x5d203 +sched()+0x165 proc.c:164 + p=0x80279948 +schedinit()+0x90 proc.c:107 +squidboy(apic=0x8026b144)+0x96 mp.c:421 +0x80003091 ?file?:0 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machno "" "Display debugging Mach number +.CW machno +prints the number of the debugging +.CW Mach . +.Ex +acid: machno() +0 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machregs Mach "Print registers for Mach +.CW machregs +prints the contents of both the general and special purpose registers +for the given +.I Mach . +.CW machregs +calls +.CW machspr +then +.CW machgpr +to display the contents of the registers. +.Ex +acid: machregs(machp[1]) +PC 0x80196e10 runproc+0x53 proc.c:531 +SP 0x80016f80 ECODE 0x801006f8 EFLAG 0x00000202 +CS 0x00000010 DS 0x80010008 SS 0x0003b000 +GS 0x0000001b FS 0x0000001b ES 0x00000008 +TRAP 0x00000002 nonmaskable interrupt +AX 0xc9121c18 BX 0x8025f34c CX 0x000000d8 DX 0x00000000 +DI 0x80279b44 SI 0x0005d203 BP 0x80016000 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machs "" "Print summaries for all Machs +.CW machs +prints summaries for all +.CW Mach s +in the system. +.Ex +acid: machs() +0x80017000 0 up 0x00000000 +0x8003d000 1 up 0x00000000 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machspr Mach "Print special purpose registers for Mach +.CW machspr +prints the special purpose registers for the given +.I Mach . +While +.CW machspr +may be used interactively, this function is typically only called by +.CW machregs . +.Ex +acid: machspr(machp[1]) +PC 0x80196e10 runproc+0x53 proc.c:531 +SP 0x80016f80 ECODE 0x801006f8 EFLAG 0x00000202 +CS 0x00000010 DS 0x80010008 SS 0x0003b000 +GS 0x0000001b FS 0x0000001b ES 0x00000008 +TRAP 0x00000002 nonmaskable interrupt +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machstacks "" "Stack traces for all Machs +.CW machstacks +prints a stack trace for all +.CW Mach s +in the system, similar to +.CW stacks . +.Ex +acid: machstacks() +===================================================== +0x80017000 0 up 0x00000000 +runproc()+0x14d proc.c:530 +sched()+0x165 proc.c:164 +schedinit()+0x90 proc.c:107 +main()+0x158 main.c:130 +idle l.s:233 +===================================================== +0x8003d000 1 up 0x00000000 +runproc()+0x53 proc.c:531 +sched()+0x165 proc.c:164 +schedinit()+0x90 proc.c:107 +squidboy(apic=0x8026b144)+0x96 mp.c:421 +0x80003091 ?file?:0 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machstk Mach "Stack trace for Mach +.CW machstk +produces a short format stack trace for the given +.I Mach , +similar to +.CW stk . +Unlike +.CW stk , +the +.CW : +operator should not be used to address variables on processors other +than the current +.CW Mach +(see +.CW machno ). +This is a limitation imposed by the current implementation of +.I acid +and +.I libmach . +.Ex +acid: machstk(machp[1]) +runproc()+0x53 proc.c:531 +sched()+0x165 proc.c:164 +schedinit()+0x90 proc.c:107 +squidboy(apic=0x8026b144)+0x96 mp.c:421 +0x80003091 ?file?:0 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB machunwind Mach "Dump stack contents for Mach +.CW machunwind +dumps the contents of the stack for the given +.I Mach . +This is a function of last resort; it is primarily used to debug the +above functions. +.Ex +acid: machunwind(machp[1]) +\&... +0x8003dfd8: schedinit+0x90 +0x8003dfdc: 0x80279948 +0x8003dfe0: microdelay+0x3c +0x8003dfe4: 0x32e8 +0x8003dfe8: 0x0 +0x8003dfec: squidboy+0x96 +0x8003dff0: 0x64 +0x8003dff4: 0x0 +0x8003dff8: 0x80003091 +0x8003dffc: mplapic+0xb0 +.Ee +.SH +Support Functions +.LP +These functions provide utility to other library functions. +.\" +.\" +.\" +.Ip integer machaddr "Mach, integer" "Convert address for Mach +.CW machaddr +converts the given +.I integer +address to a global address. +If the address is mapped to processor-local memory, +.CW machaddr +will provide an alternative that can be addressed by any processor. +This function is idempotent; any address, regardless of mapping, may +be passed to this function. +.Ex +acid: MACHADDR = KZERO+0x16000; +acid: print(machaddr(machp[1], MACHADDR)\eX) +0x8003d000 +.Ee +.\" +.\" +.\" +.Ip integer machpc Mach "Find program counter for Mach +.CW machpc +provides the program counter for the given +.I Mach . +.Ex +acid: print(machpc(machp[1])\eX) +0x80196e10 +.Ee +.\" +.\" +.\" +.Ip integer machsp Mach "Find stack pointer for Mach +.CW machsp +provides the stack pointer for the given +.I Mach . +.Ex +acid: print(machsp(machp[1])\eX) +0x80016f80 +.Ee +.\" +.\" +.\" +.Ip Ureg machureg Mach "Find Ureg structure for Mach +.CW machureg +provides the +.I Ureg +structure for the given +.I Mach . +.Ex +acid: print(machureg(machp[1])\eX) +0x8003df3c +.Ee +.SH +Redefined Functions +.LP +A handful of functions defined in other modules are redefined to +augment behavior on multiprocessors. +This necessitates the +.CW mach +library be defined after augmented libraries on the command line. +.\" +.\" +.\" +.Ip \f(CW{}\fB proclstk Proc "Stack trace with local variables for Proc +.CW proclstk +produces a long format stack trace for the given +.I Proc , +similar to +.CW lstk . +Unlike +.CW lstk , +the +.CW : +operator should not be used to address variables unless the process is +scheduled on the current +.CW Mach +(see +.CW machno ). +This is a limitation imposed by the current implementation of +.I acid +and +.I libmach . +This function was added to supplement the redefined +.CW procstk +function below. +.Ex +acid: proclstk(0x8027ca08) +gotolabel(label=0x80016030)+0x0 l.s:1000 +sched()+0x160 proc.c:164 + p=0x286 +sysrendezvous(arg=0x8027cc64)+0x143 sysproc.c:836 + rendval=0x0 + tag=0x624bc + l=0x8b1ce87c + val=0x7c86d798 + p=0x8027d3c8 +syscall(ureg=0x8b20a1a4)+0x238 trap.c:726 + sp=0xcfffee9c + scallnr=0x22 + startns=0x0 + ret=0xffffffff + i=0x1 + stopns=0x0 + s=0x0 +_syscallintr()+0x18 plan9l.s:44 +0x8b20a1a4 ?file?:0 +.Ee +.\" +.\" +.\" +.Ip \f(CW{}\fB procstk Proc "Stack trace for Proc +.CW procstk +produces a short format stack trace for the given +.I Proc , +similar to +.CW stk . +Unlike +.CW stk , +the +.CW : +operator should not be used to address variables unless the process is +scheduled on the current +.CW Mach +(see +.CW machno ). +This is a limitation imposed by the current implementation of +.I acid +and +.I libmach . +This function overrides the definition in +.CW kernel . +This was necessary as +.CW kernel +assumes any given process terminates in a call to +.CW gotolabel . +This is not always the case on a multiprocessor where the process may +be actively scheduled at debug time. +.Ex +acid: procstk(0x8027ca08) +gotolabel(label=0x80016030)+0x0 l.s:1000 +sched()+0x160 proc.c:164 +sysrendezvous(arg=0x8027cc64)+0x143 sysproc.c:836 +syscall(ureg=0x8b20a1a4)+0x238 trap.c:726 +_syscallintr()+0x18 plan9l.s:44 +0x8b20a1a4 ?file?:0 +.Ee +.\" +.\" +.\" +.Ip string reason integer "Return cause of Mach stoppage +.CW reason +uses machine-dependent information to generate a string explaining why +a +.CW Mach +has stopped. +The +.I integer +argument is the value of an architecture dependent status register. +This function overrides the builtin definition. +This was necessary as the builtin function would discard the +.I integer +argument and always consult the status register on the debugging +.CW Mach . +.Ex +acid: print(reason(machureg(machp[1]).trap)) +nonmaskable interrupt +.Ee +.SH +Future Work +.LP +Only +.CW pc +kernels are supported. +.LP +In-situ debugging is not supported. +.LP +Floating point is not supported. +This is further complicated by support for XSAVE/XRESTOR and +YMM register state in +.CW pc +kernels. +.LP +.I Acid +and its constituent libraries assume uniprocessor, which causes +complications in kernel context. Use of redefined functions have +largely addressed these issues. +.LP +Users must be aware of processor-local address ranges. +A good understanding of kernel memory mapping is essential. +The addition of per-processor address translation to +.I acid +and +.I libmach +could ease this burden considerably. +.LP +The somewhat portable nature of the +.CW kernel +library is at odds with the +.CW pc -specific +.CW mach +library. +Once remaining portability issues are resolved, both libraries could +be merged. +.SH +Acknowledgements +.LP +Portions of this document use descriptions and formatting from the +``Acid Manual'', by Phil Winterbottom. +.SH +References +.IP [1] +P. Winterbottom, +``Acid: A Debugger Built from A Language'', +.I +USENIX Proc. of the Winter 1994 Conf., +.R +San Francisco, CA. --- /sys/doc/mkfile Mon Aug 12 08:33:05 2013 +++ /sys/doc/mkfile Mon Aug 12 08:33:03 2013 @@ -19,6 +19,7 @@ ape\ acidpaper\ acid\ + acidmach\ mk\ mkfiles\ asm\ --- /sys/man/8/plan9.ini Mon Aug 12 08:33:17 2013 +++ /sys/man/8/plan9.ini Mon Aug 12 08:33:10 2013 @@ -579,6 +579,8 @@ .LP would use COM1 at 19,200 baud with odd parity. +.SS \fLrdb=\fIvalue +This is used to enable remote kernel debugging at boot. .SS "PC CARD" .SS \fLpccard0=disabled\fP Disable probing for and automatic configuration of PC card controllers. @@ -809,6 +811,8 @@ .SS \fL*nopcirouting=\fP Disable pci routing during boot. May solve interrupt routing problems on certain machines. +.SS \fL*nodumppath=\fP +Disable printing the full kernel path on panic. .SS \fL*nodumpstack=\fP Disable printing a stack dump on panic. Useful if there is only a limited cga screen available,