From 5372588c6114144d8923fa61293908122cb4c3c5 Mon Sep 17 00:00:00 2001 From: aarzilli Date: Fri, 1 Sep 2017 15:34:13 +0200 Subject: [PATCH] proc: support cgo stacktraces When creating a stack trace we should switch between the goroutine stack and the system stack (where cgo code is executed) as appropriate to reconstruct the logical stacktrace. Goroutines that are currently executing on the system stack will have the SystemStack flag set, frames of the goroutine stack will have a negative FrameOffset (like always) and frames of the system stack will have a positive FrameOffset (which is actually just the CFA value for the frame). Updates #935 --- _fixtures/cgostacktest/hello.c | 23 ++++ _fixtures/cgostacktest/hello.h | 7 ++ _fixtures/cgostacktest/main.go | 30 +++++ pkg/dwarf/frame/table.go | 4 +- pkg/dwarf/op/regs.go | 5 + pkg/proc/arch.go | 39 +++++- pkg/proc/eval.go | 2 +- pkg/proc/proc.go | 17 +-- pkg/proc/proc_test.go | 216 ++++++++++++++++++++++++++++++++ pkg/proc/stack.go | 220 ++++++++++++++++++++++++++++----- pkg/proc/threads.go | 34 +++-- pkg/proc/variables.go | 5 +- service/debugger/debugger.go | 2 +- 13 files changed, 547 insertions(+), 57 deletions(-) create mode 100644 _fixtures/cgostacktest/hello.c create mode 100644 _fixtures/cgostacktest/hello.h create mode 100644 _fixtures/cgostacktest/main.go diff --git a/_fixtures/cgostacktest/hello.c b/_fixtures/cgostacktest/hello.c new file mode 100644 index 00000000..fd904a9e --- /dev/null +++ b/_fixtures/cgostacktest/hello.c @@ -0,0 +1,23 @@ +#include + +#include "_cgo_export.h" + +#define BREAKPOINT asm("int3;") + +void helloworld_pt2(int x) { + BREAKPOINT; + helloWorld(x+1); +} + +void helloworld(int x) { + helloworld_pt2(x+1); +} + +void helloworld_pt4(int x) { + BREAKPOINT; + helloWorld2(x+1); +} + +void helloworld_pt3(int x) { + helloworld_pt4(x+1); +} diff --git a/_fixtures/cgostacktest/hello.h b/_fixtures/cgostacktest/hello.h new file mode 100644 index 00000000..e59eb6e9 --- /dev/null +++ b/_fixtures/cgostacktest/hello.h @@ -0,0 +1,7 @@ +#ifndef __HELLO_H__ +#define __HELLO_H__ + +void helloworld(int); +void helloworld_pt3(int); + +#endif diff --git a/_fixtures/cgostacktest/main.go b/_fixtures/cgostacktest/main.go new file mode 100644 index 00000000..7b71c8ad --- /dev/null +++ b/_fixtures/cgostacktest/main.go @@ -0,0 +1,30 @@ +package main + +// #include +import "C" + +import ( + "fmt" + "runtime" +) + +func main() { + runtime.Breakpoint() + C.helloworld(2) +} + +//export helloWorld +func helloWorld(x C.int) { + helloWorldS(x) +} + +func helloWorldS(x C.int) { + runtime.Breakpoint() + C.helloworld_pt3(x - 1) +} + +//export helloWorld2 +func helloWorld2(x C.int) { + runtime.Breakpoint() + fmt.Printf("hello world %d\n", x) +} diff --git a/pkg/dwarf/frame/table.go b/pkg/dwarf/frame/table.go index 012a7020..5ac5fa38 100644 --- a/pkg/dwarf/frame/table.go +++ b/pkg/dwarf/frame/table.go @@ -74,8 +74,8 @@ const ( RuleExpression RuleValExpression RuleArchitectural - RuleCFA // Value is rule.Reg + rule.Offset - RuleRegOffset // Value is stored at address rule.Reg + rule.Offset + RuleCFA // Value is rule.Reg + rule.Offset + RuleFramePointer // Value is stored at address rule.Reg + rule.Offset, but only if it's less than the current CFA, otherwise same value ) const low_6_offset = 0x3f diff --git a/pkg/dwarf/op/regs.go b/pkg/dwarf/op/regs.go index 130a4d71..67c12fcf 100644 --- a/pkg/dwarf/op/regs.go +++ b/pkg/dwarf/op/regs.go @@ -14,6 +14,7 @@ type DwarfRegisters struct { ByteOrder binary.ByteOrder PCRegNum uint64 SPRegNum uint64 + BPRegNum uint64 } type DwarfRegister struct { @@ -61,6 +62,10 @@ func (regs *DwarfRegisters) SP() uint64 { return regs.Uint64Val(regs.SPRegNum) } +func (regs *DwarfRegisters) BP() uint64 { + return regs.Uint64Val(regs.BPRegNum) +} + // AddReg adds register idx to regs. func (regs *DwarfRegisters) AddReg(idx uint64, reg *DwarfRegister) { if idx >= uint64(len(regs.Regs)) { diff --git a/pkg/proc/arch.go b/pkg/proc/arch.go index e40c21c3..20d14c48 100644 --- a/pkg/proc/arch.go +++ b/pkg/proc/arch.go @@ -15,7 +15,7 @@ type Arch interface { BreakpointInstruction() []byte BreakpointSize() int DerefTLS() bool - FixFrameUnwindContext(*frame.FrameContext) *frame.FrameContext + FixFrameUnwindContext(fctxt *frame.FrameContext, pc uint64, bi *BinaryInfo) *frame.FrameContext RegSize(uint64) int RegistersToDwarfRegisters(Registers) op.DwarfRegisters GoroutineToDwarfRegisters(*G) op.DwarfRegisters @@ -29,6 +29,12 @@ type AMD64 struct { gStructOffset uint64 hardwareBreakpointUsage []bool goos string + + // crosscall2fn is the DIE of crosscall2, a function used by the go runtime + // to call C functions. This function in go 1.9 (and previous versions) had + // a bad frame descriptor which needs to be fixed to generate good stack + // traces. + crosscall2fn *Function } const ( @@ -75,9 +81,15 @@ func (a *AMD64) DerefTLS() bool { return a.goos == "windows" } +const ( + crosscall2SPOffsetBad = 0x8 + crosscall2SPOffsetWindows = 0x118 + crosscall2SPOffsetNonWindows = 0x58 +) + // FixFrameUnwindContext adds default architecture rules to fctxt or returns // the default frame unwind context if fctxt is nil. -func (a *AMD64) FixFrameUnwindContext(fctxt *frame.FrameContext) *frame.FrameContext { +func (a *AMD64) FixFrameUnwindContext(fctxt *frame.FrameContext, pc uint64, bi *BinaryInfo) *frame.FrameContext { if fctxt == nil { // When there's no frame descriptor entry use BP (the frame pointer) instead // - return register is [bp + a.PtrSize()] (i.e. [cfa-a.PtrSize()]) @@ -109,13 +121,30 @@ func (a *AMD64) FixFrameUnwindContext(fctxt *frame.FrameContext) *frame.FrameCon } } + if a.crosscall2fn == nil { + a.crosscall2fn = bi.LookupFunc["crosscall2"] + } + + if a.crosscall2fn != nil && pc >= a.crosscall2fn.Entry && pc < a.crosscall2fn.End { + rule := fctxt.CFA + if rule.Offset == crosscall2SPOffsetBad { + switch a.goos { + case "windows": + rule.Offset += crosscall2SPOffsetWindows + default: + rule.Offset += crosscall2SPOffsetNonWindows + } + } + fctxt.CFA = rule + } + // We assume that RBP is the frame pointer and we want to keep it updated, // so that we can use it to unwind the stack even when we encounter frames // without descriptor entries. // If there isn't a rule already we emit one. if fctxt.Regs[amd64DwarfBPRegNum].Rule == frame.RuleUndefined { fctxt.Regs[amd64DwarfBPRegNum] = frame.DWRule{ - Rule: frame.RuleRegOffset, + Rule: frame.RuleFramePointer, Reg: amd64DwarfBPRegNum, Offset: 0, } @@ -241,7 +270,7 @@ func (a *AMD64) RegistersToDwarfRegisters(regs Registers) op.DwarfRegisters { } } - return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum} + return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum, BPRegNum: amd64DwarfBPRegNum} } // GoroutineToDwarfRegisters extract the saved DWARF registers from a parked @@ -251,5 +280,5 @@ func (a *AMD64) GoroutineToDwarfRegisters(g *G) op.DwarfRegisters { dregs[amd64DwarfIPRegNum] = op.DwarfRegisterFromUint64(g.PC) dregs[amd64DwarfSPRegNum] = op.DwarfRegisterFromUint64(g.SP) dregs[amd64DwarfBPRegNum] = op.DwarfRegisterFromUint64(g.BP) - return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum} + return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum, BPRegNum: amd64DwarfBPRegNum} } diff --git a/pkg/proc/eval.go b/pkg/proc/eval.go index cafe3df7..e3147416 100644 --- a/pkg/proc/eval.go +++ b/pkg/proc/eval.go @@ -183,7 +183,7 @@ func (scope *EvalScope) evalAST(t ast.Expr) (*Variable, error) { } return scope.Gvar.clone(), nil } else if maybePkg.Name == "runtime" && node.Sel.Name == "frameoff" { - return newConstant(constant.MakeInt64(scope.Regs.CFA-int64(scope.StackHi)), scope.Mem), nil + return newConstant(constant.MakeInt64(scope.frameOffset), scope.Mem), nil } else if v, err := scope.packageVarAddr(maybePkg.Name + "." + node.Sel.Name); err == nil { return v, nil } diff --git a/pkg/proc/proc.go b/pkg/proc/proc.go index 8a5d20f1..baf1d049 100644 --- a/pkg/proc/proc.go +++ b/pkg/proc/proc.go @@ -286,7 +286,7 @@ func StepOut(dbp Process) error { } sameGCond := SameGoroutineCondition(selg) - retFrameCond := andFrameoffCondition(sameGCond, retframe.Regs.CFA-int64(retframe.StackHi)) + retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset()) var deferpc uint64 = 0 if filepath.Ext(topframe.Current.File) == ".go" { @@ -358,7 +358,7 @@ func GoroutinesInfo(dbp Process) ([]*G, error) { } var ( - threadg = map[int]Thread{} + threadg = map[int]*G{} allg []*G rdr = dbp.BinInfo().DwarfReader() ) @@ -370,7 +370,7 @@ func GoroutinesInfo(dbp Process) ([]*G, error) { } g, _ := GetG(th) if g != nil { - threadg[g.ID] = th + threadg[g.ID] = g } } @@ -410,14 +410,15 @@ func GoroutinesInfo(dbp Process) ([]*G, error) { if err != nil { return nil, err } - if thread, allocated := threadg[g.ID]; allocated { - loc, err := thread.Location() + if thg, allocated := threadg[g.ID]; allocated { + loc, err := thg.Thread.Location() if err != nil { return nil, err } - g.Thread = thread + g.Thread = thg.Thread // Prefer actual thread location information. g.CurrentLoc = *loc + g.SystemStack = thg.SystemStack } if g.Status != Gdead { allg = append(allg, g) @@ -481,10 +482,10 @@ func ConvertEvalScope(dbp Process, gid, frame int) (*EvalScope, error) { return nil, fmt.Errorf("Frame %d does not exist in goroutine %d", frame, gid) } - return &EvalScope{locs[frame].Current.PC, locs[frame].Regs, thread, g.variable, dbp.BinInfo(), locs[frame].StackHi}, nil + return &EvalScope{locs[frame].Current.PC, locs[frame].Regs, thread, g.variable, dbp.BinInfo(), locs[frame].FrameOffset()}, nil } // FrameToScope returns a new EvalScope for this frame func FrameToScope(p Process, frame Stackframe) *EvalScope { - return &EvalScope{frame.Current.PC, frame.Regs, p.CurrentThread(), nil, p.BinInfo(), frame.StackHi} + return &EvalScope{frame.Current.PC, frame.Regs, p.CurrentThread(), nil, p.BinInfo(), frame.FrameOffset()} } diff --git a/pkg/proc/proc_test.go b/pkg/proc/proc_test.go index c8ca7d63..0213ac42 100644 --- a/pkg/proc/proc_test.go +++ b/pkg/proc/proc_test.go @@ -3121,3 +3121,219 @@ func TestIssue844(t *testing.T) { } }) } + +func logStacktrace(t *testing.T, frames []proc.Stackframe) { + for j := range frames { + name := "?" + if frames[j].Current.Fn != nil { + name = frames[j].Current.Fn.Name + } + + t.Logf("\t%#x %#x %#x %s at %s:%d\n", frames[j].Call.PC, frames[j].FrameOffset(), frames[j].FramePointerOffset(), name, filepath.Base(frames[j].Call.File), frames[j].Call.Line) + } +} + +// stacktraceCheck checks that all the functions listed in tc appear in +// frames in the same order. +// Checks that all the functions in tc starting with "C." or with "!" are in +// a systemstack frame. +// Returns a slice m where m[i] is the index in frames of the function tc[i] +// or nil if any check fails. +func stacktraceCheck(t *testing.T, tc []string, frames []proc.Stackframe) []int { + m := make([]int, len(tc)) + i, j := 0, 0 + for i < len(tc) { + tcname := tc[i] + tcsystem := strings.HasPrefix(tcname, "C.") + if tcname[0] == '!' { + tcsystem = true + tcname = tcname[1:] + } + for j < len(frames) { + name := "?" + if frames[j].Current.Fn != nil { + name = frames[j].Current.Fn.Name + } + if name == tcname { + m[i] = j + if tcsystem != frames[j].SystemStack { + t.Logf("system stack check failed for frame %d (expected %v got %v)", j, tcsystem, frames[j].SystemStack) + t.Logf("expected: %v\n", tc) + return nil + } + break + } + + j++ + } + if j >= len(frames) { + t.Logf("couldn't find frame %d %s", i, tc) + t.Logf("expected: %v\n", tc) + return nil + } + + i++ + } + return m +} + +func frameInFile(frame proc.Stackframe, file string) bool { + for _, loc := range []proc.Location{frame.Current, frame.Call} { + if !strings.HasSuffix(loc.File, "/"+file) && !strings.HasSuffix(loc.File, "\\"+file) { + return false + } + if loc.Line <= 0 { + return false + } + } + return true +} + +func TestCgoStacktrace(t *testing.T) { + if runtime.GOOS == "windows" { + ver, _ := goversion.Parse(runtime.Version()) + if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 9, -1, 0, 0, ""}) { + t.Skip("disabled on windows with go before version 1.9") + } + } + if runtime.GOOS == "darwin" { + ver, _ := goversion.Parse(runtime.Version()) + if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 8, -1, 0, 0, ""}) { + t.Skip("disabled on macOS with go before version 1.8") + } + } + + // Tests that: + // a) we correctly identify the goroutine while we are executing cgo code + // b) that we can stitch together the system stack (where cgo code + // executes) and the normal goroutine stack + + // Each test case describes how the stack trace should appear after a + // continue. The first function on each test case is the topmost function + // that should be found on the stack, the actual stack trace can have more + // frame than those listed here but all the frames listed must appear in + // the specified order. + testCases := [][]string{ + []string{"main.main"}, + []string{"C.helloworld_pt2", "C.helloworld", "main.main"}, + []string{"main.helloWorldS", "main.helloWorld", "C.helloworld_pt2", "C.helloworld", "main.main"}, + []string{"C.helloworld_pt4", "C.helloworld_pt3", "main.helloWorldS", "main.helloWorld", "C.helloworld_pt2", "C.helloworld", "main.main"}, + []string{"main.helloWorld2", "C.helloworld_pt4", "C.helloworld_pt3", "main.helloWorldS", "main.helloWorld", "C.helloworld_pt2", "C.helloworld", "main.main"}} + + var gid int + + frameOffs := map[string]int64{} + framePointerOffs := map[string]int64{} + + withTestProcess("cgostacktest/", t, func(p proc.Process, fixture protest.Fixture) { + for itidx, tc := range testCases { + assertNoError(proc.Continue(p), t, fmt.Sprintf("Continue at iteration step %d", itidx)) + + g, err := proc.GetG(p.CurrentThread()) + assertNoError(err, t, fmt.Sprintf("GetG at iteration step %d", itidx)) + + if itidx == 0 { + gid = g.ID + } else { + if gid != g.ID { + t.Fatalf("wrong goroutine id at iteration step %d (expected %d got %d)", itidx, gid, g.ID) + } + } + + frames, err := g.Stacktrace(100) + assertNoError(err, t, fmt.Sprintf("Stacktrace at iteration step %d", itidx)) + + t.Logf("iteration step %d", itidx) + logStacktrace(t, frames) + + m := stacktraceCheck(t, tc, frames) + mismatch := (m == nil) + + for i, j := range m { + if strings.HasPrefix(tc[i], "C.hellow") { + if !frameInFile(frames[j], "hello.c") { + t.Logf("position in %q is %s:%d (call %s:%d)", tc[i], frames[j].Current.File, frames[j].Current.Line, frames[j].Call.File, frames[j].Call.Line) + mismatch = true + break + } + } + if frameOff, ok := frameOffs[tc[i]]; ok { + if frameOff != frames[j].FrameOffset() { + t.Logf("frame %s offset mismatch", tc[i]) + } + if framePointerOffs[tc[i]] != frames[j].FramePointerOffset() { + t.Logf("frame %s pointer offset mismatch", tc[i]) + } + } else { + frameOffs[tc[i]] = frames[j].FrameOffset() + framePointerOffs[tc[i]] = frames[j].FramePointerOffset() + } + } + + // also check that ThreadStacktrace produces the same list of frames + threadFrames, err := proc.ThreadStacktrace(p.CurrentThread(), 100) + assertNoError(err, t, fmt.Sprintf("ThreadStacktrace at iteration step %d", itidx)) + + if len(threadFrames) != len(frames) { + mismatch = true + } else { + for j := range frames { + if frames[j].Current.File != threadFrames[j].Current.File || frames[j].Current.Line != threadFrames[j].Current.Line { + t.Logf("stack mismatch between goroutine stacktrace and thread stacktrace") + t.Logf("thread stacktrace:") + logStacktrace(t, threadFrames) + mismatch = true + break + } + } + } + if mismatch { + t.Fatal("see previous loglines") + } + } + }) +} + +func TestCgoSources(t *testing.T) { + if runtime.GOOS == "windows" { + ver, _ := goversion.Parse(runtime.Version()) + if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 9, -1, 0, 0, ""}) { + t.Skip("disabled on windows with go before version 1.9") + } + } + + withTestProcess("cgostacktest/", t, func(p proc.Process, fixture protest.Fixture) { + sources := p.BinInfo().Sources + for _, needle := range []string{"main.go", "hello.c"} { + found := false + for _, k := range sources { + if strings.HasSuffix(k, "/"+needle) || strings.HasSuffix(k, "\\"+needle) { + found = true + break + } + } + if !found { + t.Errorf("File %s not found", needle) + } + } + }) +} + +func TestSystemstackStacktrace(t *testing.T) { + // check that we can follow a stack switch initiated by runtime.systemstack() + withTestProcess("panic", t, func(p proc.Process, fixture protest.Fixture) { + _, err := setFunctionBreakpoint(p, "runtime.startpanic_m") + assertNoError(err, t, "setFunctionBreakpoint()") + assertNoError(proc.Continue(p), t, "first continue") + assertNoError(proc.Continue(p), t, "second continue") + g, err := proc.GetG(p.CurrentThread()) + assertNoError(err, t, "GetG") + frames, err := g.Stacktrace(100) + assertNoError(err, t, "stacktrace") + logStacktrace(t, frames) + m := stacktraceCheck(t, []string{"!runtime.startpanic_m", "!runtime.systemstack", "runtime.startpanic", "main.main"}, frames) + if m == nil { + t.Fatal("see previous loglines") + } + }) +} diff --git a/pkg/proc/stack.go b/pkg/proc/stack.go index d60807f5..249ca2b8 100644 --- a/pkg/proc/stack.go +++ b/pkg/proc/stack.go @@ -12,8 +12,6 @@ import ( // This code is partly adaped from runtime.gentraceback in // $GOROOT/src/runtime/traceback.go -const runtimeStackBarrier = "runtime.stackBarrier" - // NoReturnAddr is returned when return address // could not be found during stack trace. type NoReturnAddr struct { @@ -33,24 +31,50 @@ type Stackframe struct { // Frame registers. Regs op.DwarfRegisters // High address of the stack. - StackHi uint64 + stackHi uint64 // Return address for this stack frame (as read from the stack frame itself). Ret uint64 // Address to the memory location containing the return address addrret uint64 // Err is set if an error occoured during stacktrace Err error + // SystemStack is true if this frame belongs to a system stack. + SystemStack bool +} + +// FrameOffset returns the address of the stack frame, absolute for system +// stack frames or as an offset from stackhi for goroutine stacks (a +// negative value). +func (frame *Stackframe) FrameOffset() int64 { + if frame.SystemStack { + return frame.Regs.CFA + } + return frame.Regs.CFA - int64(frame.stackHi) +} + +// FramePointerOffset returns the value of the frame pointer, absolute for +// system stack frames or as an offset from stackhi for goroutine stacks (a +// negative value). +func (frame *Stackframe) FramePointerOffset() int64 { + if frame.SystemStack { + return int64(frame.Regs.BP()) + } + return int64(frame.Regs.BP()) - int64(frame.stackHi) } // ThreadStacktrace returns the stack trace for thread. // Note the locations in the array are return addresses not call addresses. func ThreadStacktrace(thread Thread, depth int) ([]Stackframe, error) { - regs, err := thread.Registers(true) - if err != nil { - return nil, err + g, _ := GetG(thread) + if g == nil { + regs, err := thread.Registers(true) + if err != nil { + return nil, err + } + it := newStackIterator(thread.BinInfo(), thread, thread.BinInfo().Arch.RegistersToDwarfRegisters(regs), 0, nil, -1, nil) + return it.stacktrace(depth) } - it := newStackIterator(thread.BinInfo(), thread, thread.BinInfo().Arch.RegistersToDwarfRegisters(regs), 0, nil, -1) - return it.stacktrace(depth) + return g.Stacktrace(depth) } func (g *G) stackIterator() (*stackIterator, error) { @@ -58,14 +82,15 @@ func (g *G) stackIterator() (*stackIterator, error) { if err != nil { return nil, err } + if g.Thread != nil { regs, err := g.Thread.Registers(true) if err != nil { return nil, err } - return newStackIterator(g.variable.bi, g.Thread, g.variable.bi.Arch.RegistersToDwarfRegisters(regs), g.stackhi, stkbar, g.stkbarPos), nil + return newStackIterator(g.variable.bi, g.Thread, g.variable.bi.Arch.RegistersToDwarfRegisters(regs), g.stackhi, stkbar, g.stkbarPos, g), nil } - return newStackIterator(g.variable.bi, g.variable.mem, g.variable.bi.Arch.GoroutineToDwarfRegisters(g), g.stackhi, stkbar, g.stkbarPos), nil + return newStackIterator(g.variable.bi, g.variable.mem, g.variable.bi.Arch.GoroutineToDwarfRegisters(g), g.stackhi, stkbar, g.stkbarPos, g), nil } // Stacktrace returns the stack trace for a goroutine. @@ -98,11 +123,15 @@ type stackIterator struct { err error stackhi uint64 + systemstack bool stackBarrierPC uint64 stkbar []savedLR // regs is the register set for the current frame regs op.DwarfRegisters + + g *G // the goroutine being stacktraced, nil if we are stacktracing a goroutine-less thread + g0_sched_sp uint64 // value of g0.sched.sp (see comments around its use) } type savedLR struct { @@ -110,13 +139,13 @@ type savedLR struct { val uint64 } -func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegisters, stackhi uint64, stkbar []savedLR, stkbarPos int) *stackIterator { - stackBarrierFunc := bi.LookupFunc[runtimeStackBarrier] // stack barriers were removed in Go 1.9 +func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegisters, stackhi uint64, stkbar []savedLR, stkbarPos int, g *G) *stackIterator { + stackBarrierFunc := bi.LookupFunc["runtime.stackBarrier"] // stack barriers were removed in Go 1.9 var stackBarrierPC uint64 if stackBarrierFunc != nil && stkbar != nil { stackBarrierPC = stackBarrierFunc.Entry fn := bi.PCToFunc(regs.PC()) - if fn != nil && fn.Name == runtimeStackBarrier { + if fn != nil && fn.Name == "runtime.stackBarrier" { // We caught the goroutine as it's executing the stack barrier, we must // determine whether or not g.stackPos has already been incremented or not. if len(stkbar) > 0 && stkbar[stkbarPos].ptr < regs.SP() { @@ -130,7 +159,19 @@ func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegiste } stkbar = stkbar[stkbarPos:] } - return &stackIterator{pc: regs.PC(), regs: regs, top: true, bi: bi, mem: mem, err: nil, atend: false, stackhi: stackhi, stackBarrierPC: stackBarrierPC, stkbar: stkbar} + var g0_sched_sp uint64 + systemstack := true + if g != nil { + systemstack = g.SystemStack + g0var, _ := g.variable.fieldVariable("m").structMember("g0") + if g0var != nil { + g0, _ := g0var.parseG() + if g0 != nil { + g0_sched_sp = g0.SP + } + } + } + return &stackIterator{pc: regs.PC(), regs: regs, top: true, bi: bi, mem: mem, err: nil, atend: false, stackhi: stackhi, stackBarrierPC: stackBarrierPC, stkbar: stkbar, systemstack: systemstack, g: g, g0_sched_sp: g0_sched_sp} } // Next points the iterator to the next stack frame. @@ -152,9 +193,7 @@ func (it *stackIterator) Next() bool { it.stkbar = it.stkbar[1:] } - // Look for "top of stack" functions. - if it.frame.Current.Fn != nil && (it.frame.Current.Fn.Name == "runtime.goexit" || it.frame.Current.Fn.Name == "runtime.rt0_go" || it.frame.Current.Fn.Name == "runtime.mcall") { - it.atend = true + if it.switchStack() { return true } @@ -164,6 +203,109 @@ func (it *stackIterator) Next() bool { return true } +// asmcgocallSPOffsetSaveSlot is the offset from systemstack.SP where +// (goroutine.SP - StackHi) is saved in runtime.asmcgocall after the stack +// switch happens. +const asmcgocallSPOffsetSaveSlot = 0x28 + +// switchStack will use the current frame to determine if it's time to +// switch between the system stack and the goroutine stack or vice versa. +// Sets it.atend when the top of the stack is reached. +func (it *stackIterator) switchStack() bool { + if it.frame.Current.Fn == nil { + return false + } + switch it.frame.Current.Fn.Name { + case "runtime.asmcgocall": + if it.top || !it.systemstack { + return false + } + + // switch from system stack to goroutine stack + + off, _ := readIntRaw(it.mem, uintptr(it.regs.SP()+asmcgocallSPOffsetSaveSlot), int64(it.bi.Arch.PtrSize())) // reads "offset of SP from StackHi" from where runtime.asmcgocall saved it + oldsp := it.regs.SP() + it.regs.Reg(it.regs.SPRegNum).Uint64Val = uint64(int64(it.stackhi) - off) + + // runtime.asmcgocall can also be called from inside the system stack, + // in that case no stack switch actually happens + if it.regs.SP() == oldsp { + return false + } + it.systemstack = false + + // advances to the next frame in the call stack + it.frame.addrret = uint64(int64(it.regs.SP()) + int64(it.bi.Arch.PtrSize())) + it.frame.Ret, _ = readUintRaw(it.mem, uintptr(it.frame.addrret), int64(it.bi.Arch.PtrSize())) + it.pc = it.frame.Ret + + it.top = false + return true + + case "runtime.mstart": + if it.top || !it.systemstack || it.g == nil { + return false + } + + // Calls to runtime.systemstack will switch to the systemstack then: + // 1. alter the goroutine stack so that it looks like systemstack_switch + // was called + // 2. alter the system stack so that it looks like the bottom-most frame + // belongs to runtime.mstart + // If we find a runtime.mstart frame on the system stack of a goroutine + // parked on runtime.systemstack_switch we assume runtime.systemstack was + // called and continue tracing from the parked position. + + if fn := it.bi.PCToFunc(it.g.PC); fn == nil || fn.Name != "runtime.systemstack_switch" { + return false + } + it.systemstack = false + it.pc = it.g.PC + it.regs.Reg(it.regs.SPRegNum).Uint64Val = it.g.SP + it.regs.Reg(it.regs.BPRegNum).Uint64Val = it.g.BP + it.top = false + return true + + case "runtime.cgocallback_gofunc": + // For a detailed description of how this works read the long comment at + // the start of $GOROOT/src/runtime/cgocall.go and the source code of + // runtime.cgocallback_gofunc in $GOROOT/src/runtime/asm_amd64.s + // + // When a C functions calls back into go it will eventually call into + // runtime.cgocallback_gofunc which is the function that does the stack + // switch from the system stack back into the goroutine stack + // Since we are going backwards on the stack here we see the transition + // as goroutine stack -> system stack. + + if it.top || it.systemstack { + return false + } + + if it.g0_sched_sp <= 0 { + return false + } + // entering the system stack + it.regs.Reg(it.regs.SPRegNum).Uint64Val = it.g0_sched_sp + // reads the previous value of g0.sched.sp that runtime.cgocallback_gofunc saved on the stack + it.g0_sched_sp, _ = readUintRaw(it.mem, uintptr(it.regs.SP()), int64(it.bi.Arch.PtrSize())) + it.top = false + callFrameRegs, ret, retaddr := it.advanceRegs() + frameOnSystemStack := it.newStackframe(ret, retaddr) + it.pc = frameOnSystemStack.Ret + it.regs = callFrameRegs + it.systemstack = true + return true + + case "runtime.goexit", "runtime.rt0_go", "runtime.mcall": + // Look for "top of stack" functions. + it.atend = true + return true + + default: + return false + } +} + // Frame returns the frame the iterator is pointing at. func (it *stackIterator) Frame() Stackframe { return it.frame @@ -199,14 +341,25 @@ func (it *stackIterator) newStackframe(ret, retaddr uint64) Stackframe { } else { it.regs.FrameBase = it.frameBase(fn) } - r := Stackframe{Current: Location{PC: it.pc, File: f, Line: l, Fn: fn}, Regs: it.regs, Ret: ret, addrret: retaddr, StackHi: it.stackhi} + r := Stackframe{Current: Location{PC: it.pc, File: f, Line: l, Fn: fn}, Regs: it.regs, Ret: ret, addrret: retaddr, stackHi: it.stackhi, SystemStack: it.systemstack} if !it.top { - r.Call.File, r.Call.Line, r.Call.Fn = it.bi.PCToLine(it.pc - 1) - if r.Call.Fn == nil { - r.Call.File = "?" - r.Call.Line = -1 + fnname := "" + if r.Current.Fn != nil { + fnname = r.Current.Fn.Name + } + switch fnname { + case "runtime.mstart", "runtime.systemstack_switch": + // these frames are inserted by runtime.systemstack and there is no CALL + // instruction to look for at pc - 1 + r.Call = r.Current + default: + r.Call.File, r.Call.Line, r.Call.Fn = it.bi.PCToLine(it.pc - 1) + if r.Call.Fn == nil { + r.Call.File = "?" + r.Call.Line = -1 + } + r.Call.PC = r.Current.PC } - r.Call.PC = r.Current.PC } else { r.Call = r.Current } @@ -240,9 +393,9 @@ func (it *stackIterator) advanceRegs() (callFrameRegs op.DwarfRegisters, ret uin fde, err := it.bi.frameEntries.FDEForPC(it.pc) var framectx *frame.FrameContext if _, nofde := err.(*frame.NoFDEForPCError); nofde { - framectx = it.bi.Arch.FixFrameUnwindContext(nil) + framectx = it.bi.Arch.FixFrameUnwindContext(nil, it.pc, it.bi) } else { - framectx = it.bi.Arch.FixFrameUnwindContext(fde.EstablishFrame(it.pc)) + framectx = it.bi.Arch.FixFrameUnwindContext(fde.EstablishFrame(it.pc), it.pc, it.bi) } cfareg, err := it.executeFrameRegRule(0, framectx.CFA, 0) @@ -252,7 +405,7 @@ func (it *stackIterator) advanceRegs() (callFrameRegs op.DwarfRegisters, ret uin } it.regs.CFA = int64(cfareg.Uint64Val) - callFrameRegs = op.DwarfRegisters{ByteOrder: it.regs.ByteOrder, PCRegNum: it.regs.PCRegNum, SPRegNum: it.regs.SPRegNum} + callFrameRegs = op.DwarfRegisters{ByteOrder: it.regs.ByteOrder, PCRegNum: it.regs.PCRegNum, SPRegNum: it.regs.SPRegNum, BPRegNum: it.regs.BPRegNum} // According to the standard the compiler should be responsible for emitting // rules for the RSP register so that it can then be used to calculate CFA, @@ -289,7 +442,8 @@ func (it *stackIterator) executeFrameRegRule(regnum uint64, rule frame.DWRule, c case frame.RuleUndefined: return nil, nil case frame.RuleSameVal: - return it.regs.Reg(regnum), nil + reg := *it.regs.Reg(regnum) + return ®, nil case frame.RuleOffset: return it.readRegisterAt(regnum, uint64(cfa+rule.Offset)) case frame.RuleValOffset: @@ -315,11 +469,17 @@ func (it *stackIterator) executeFrameRegRule(regnum uint64, rule frame.DWRule, c return nil, nil } return op.DwarfRegisterFromUint64(uint64(int64(it.regs.Uint64Val(rule.Reg)) + rule.Offset)), nil - case frame.RuleRegOffset: - if it.regs.Reg(rule.Reg) == nil { + case frame.RuleFramePointer: + curReg := it.regs.Reg(rule.Reg) + if curReg == nil { return nil, nil } - return it.readRegisterAt(regnum, uint64(int64(it.regs.Uint64Val(rule.Reg))+rule.Offset)) + if curReg.Uint64Val <= uint64(cfa) { + return it.readRegisterAt(regnum, curReg.Uint64Val) + } else { + newReg := *curReg + return &newReg, nil + } } } diff --git a/pkg/proc/threads.go b/pkg/proc/threads.go index 3b9e7216..9a05f4d4 100644 --- a/pkg/proc/threads.go +++ b/pkg/proc/threads.go @@ -137,8 +137,8 @@ func next(dbp Process, stepInto bool) error { } sameGCond := SameGoroutineCondition(selg) - retFrameCond := andFrameoffCondition(sameGCond, retframe.Regs.CFA-int64(retframe.StackHi)) - sameFrameCond := andFrameoffCondition(sameGCond, topframe.Regs.CFA-int64(topframe.StackHi)) + retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset()) + sameFrameCond := andFrameoffCondition(sameGCond, topframe.FrameOffset()) var sameOrRetFrameCond ast.Expr if sameGCond != nil { sameOrRetFrameCond = &ast.BinaryExpr{ @@ -146,8 +146,8 @@ func next(dbp Process, stepInto bool) error { X: sameGCond, Y: &ast.BinaryExpr{ Op: token.LOR, - X: frameoffCondition(topframe.Regs.CFA - int64(topframe.StackHi)), - Y: frameoffCondition(retframe.Regs.CFA - int64(retframe.StackHi)), + X: frameoffCondition(topframe.FrameOffset()), + Y: frameoffCondition(retframe.FrameOffset()), }, } } @@ -361,11 +361,27 @@ func GetG(thread Thread) (g *G, err error) { } g, err = gaddr.parseG() - if err == nil { - g.Thread = thread - if loc, err := thread.Location(); err == nil { - g.CurrentLoc = *loc + if err != nil { + return + } + if g.ID == 0 { + // The runtime uses a special goroutine with ID == 0 to mark that the + // current goroutine is executing on the system stack (sometimes also + // referred to as the g0 stack or scheduler stack, I'm not sure if there's + // actually any difference between those). + // For our purposes it's better if we always return the real goroutine + // since the rest of the code assumes the goroutine ID is univocal. + // The real 'current goroutine' is stored in g0.m.curg + curgvar, _ := g.variable.fieldVariable("m").structMember("curg") + g, err = curgvar.parseG() + if err != nil { + return } + g.SystemStack = true + } + g.Thread = thread + if loc, err := thread.Location(); err == nil { + g.CurrentLoc = *loc } return } @@ -395,7 +411,7 @@ func GoroutineScope(thread Thread) (*EvalScope, error) { if err != nil { return nil, err } - return &EvalScope{locations[0].Current.PC, locations[0].Regs, thread, g.variable, thread.BinInfo(), g.stackhi}, nil + return &EvalScope{locations[0].Current.PC, locations[0].Regs, thread, g.variable, thread.BinInfo(), locations[0].FrameOffset()}, nil } func onRuntimeBreakpoint(thread Thread) bool { diff --git a/pkg/proc/variables.go b/pkg/proc/variables.go index 3544ae39..1bf7687b 100644 --- a/pkg/proc/variables.go +++ b/pkg/proc/variables.go @@ -138,6 +138,8 @@ type G struct { stkbarPos int // stkbarPos field of g struct stackhi uint64 // value of stack.hi + SystemStack bool // SystemStack is true if this goroutine is currently executing on a system stack. + // Information on goroutine location CurrentLoc Location @@ -155,7 +157,8 @@ type EvalScope struct { Mem MemoryReadWriter // Target's memory Gvar *Variable BinInfo *BinaryInfo - StackHi uint64 + + frameOffset int64 } // IsNilErr is returned when a variable is nil. diff --git a/service/debugger/debugger.go b/service/debugger/debugger.go index 13dd10be..9dd135c8 100644 --- a/service/debugger/debugger.go +++ b/service/debugger/debugger.go @@ -854,7 +854,7 @@ func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadCo for i := range rawlocs { frame := api.Stackframe{ Location: api.ConvertLocation(rawlocs[i].Call), - FrameOffset: rawlocs[i].Regs.CFA - int64(rawlocs[i].StackHi), + FrameOffset: rawlocs[i].FrameOffset(), } if rawlocs[i].Err != nil { frame.Err = rawlocs[i].Err.Error() -- GitLab