proc: support cgo stacktraces

When creating a stack trace we should switch between the goroutine
stack and the system stack (where cgo code is executed) as appropriate
to reconstruct the logical stacktrace.

Goroutines that are currently executing on the system stack will have
the SystemStack flag set, frames of the goroutine stack will have a
negative FrameOffset (like always) and frames of the system stack will
have a positive FrameOffset (which is actually just the CFA value for
the frame).

Updates #935

_fixtures/cgostacktest/hello.c

+#include <stdio.h>
+
+#include "_cgo_export.h"
+
+#define BREAKPOINT asm("int3;")
+
+void helloworld_pt2(int x) {
+	BREAKPOINT;
+	helloWorld(x+1);
+}
+
+void helloworld(int x) {
+	helloworld_pt2(x+1);
+}
+
+void helloworld_pt4(int x) {
+	BREAKPOINT;
+	helloWorld2(x+1);
+}
+
+void helloworld_pt3(int x) {
+	helloworld_pt4(x+1);
+}

_fixtures/cgostacktest/hello.h

+#ifndef __HELLO_H__
+#define __HELLO_H__
+
+void helloworld(int);
+void helloworld_pt3(int);
+
+#endif

_fixtures/cgostacktest/main.go

+package main
+
+// #include <hello.h>
+import "C"
+
+import (
+	"fmt"
+	"runtime"
+)
+
+func main() {
+	runtime.Breakpoint()
+	C.helloworld(2)
+}
+
+//export helloWorld
+func helloWorld(x C.int) {
+	helloWorldS(x)
+}
+
+func helloWorldS(x C.int) {
+	runtime.Breakpoint()
+	C.helloworld_pt3(x - 1)
+}
+
+//export helloWorld2
+func helloWorld2(x C.int) {
+	runtime.Breakpoint()
+	fmt.Printf("hello world %d\n", x)
+}

pkg/dwarf/frame/table.go

@@ -74,8 +74,8 @@ const (
 	RuleExpression
 	RuleValExpression
 	RuleArchitectural
-	RuleCFA       // Value is rule.Reg + rule.Offset
-	RuleRegOffset // Value is stored at address rule.Reg + rule.Offset
+	RuleCFA          // Value is rule.Reg + rule.Offset
+	RuleFramePointer // Value is stored at address rule.Reg + rule.Offset, but only if it's less than the current CFA, otherwise same value
 )
 
 const low_6_offset = 0x3f

pkg/dwarf/op/regs.go

@@ -14,6 +14,7 @@ type DwarfRegisters struct {
 	ByteOrder binary.ByteOrder
 	PCRegNum  uint64
 	SPRegNum  uint64
+	BPRegNum  uint64
 }
 
 type DwarfRegister struct {
@@ -61,6 +62,10 @@ func (regs *DwarfRegisters) SP() uint64 {
 	return regs.Uint64Val(regs.SPRegNum)
 }
 
+func (regs *DwarfRegisters) BP() uint64 {
+	return regs.Uint64Val(regs.BPRegNum)
+}
+
 // AddReg adds register idx to regs.
 func (regs *DwarfRegisters) AddReg(idx uint64, reg *DwarfRegister) {
 	if idx >= uint64(len(regs.Regs)) {

pkg/proc/arch.go

@@ -15,7 +15,7 @@ type Arch interface {
 	BreakpointInstruction() []byte
 	BreakpointSize() int
 	DerefTLS() bool
-	FixFrameUnwindContext(*frame.FrameContext) *frame.FrameContext
+	FixFrameUnwindContext(fctxt *frame.FrameContext, pc uint64, bi *BinaryInfo) *frame.FrameContext
 	RegSize(uint64) int
 	RegistersToDwarfRegisters(Registers) op.DwarfRegisters
 	GoroutineToDwarfRegisters(*G) op.DwarfRegisters
@@ -29,6 +29,12 @@ type AMD64 struct {
 	gStructOffset           uint64
 	hardwareBreakpointUsage []bool
 	goos                    string
+
+	// crosscall2fn is the DIE of crosscall2, a function used by the go runtime
+	// to call C functions. This function in go 1.9 (and previous versions) had
+	// a bad frame descriptor which needs to be fixed to generate good stack
+	// traces.
+	crosscall2fn *Function
 }
 
 const (
@@ -75,9 +81,15 @@ func (a *AMD64) DerefTLS() bool {
 	return a.goos == "windows"
 }
 
+const (
+	crosscall2SPOffsetBad        = 0x8
+	crosscall2SPOffsetWindows    = 0x118
+	crosscall2SPOffsetNonWindows = 0x58
+)
+
 // FixFrameUnwindContext adds default architecture rules to fctxt or returns
 // the default frame unwind context if fctxt is nil.
-func (a *AMD64) FixFrameUnwindContext(fctxt *frame.FrameContext) *frame.FrameContext {
+func (a *AMD64) FixFrameUnwindContext(fctxt *frame.FrameContext, pc uint64, bi *BinaryInfo) *frame.FrameContext {
 	if fctxt == nil {
 		// When there's no frame descriptor entry use BP (the frame pointer) instead
 		// - return register is [bp + a.PtrSize()] (i.e. [cfa-a.PtrSize()])
@@ -109,13 +121,30 @@ func (a *AMD64) FixFrameUnwindContext(fctxt *frame.FrameContext) *frame.FrameCon
 		}
 	}
 
+	if a.crosscall2fn == nil {
+		a.crosscall2fn = bi.LookupFunc["crosscall2"]
+	}
+
+	if a.crosscall2fn != nil && pc >= a.crosscall2fn.Entry && pc < a.crosscall2fn.End {
+		rule := fctxt.CFA
+		if rule.Offset == crosscall2SPOffsetBad {
+			switch a.goos {
+			case "windows":
+				rule.Offset += crosscall2SPOffsetWindows
+			default:
+				rule.Offset += crosscall2SPOffsetNonWindows
+			}
+		}
+		fctxt.CFA = rule
+	}
+
 	// We assume that RBP is the frame pointer and we want to keep it updated,
 	// so that we can use it to unwind the stack even when we encounter frames
 	// without descriptor entries.
 	// If there isn't a rule already we emit one.
 	if fctxt.Regs[amd64DwarfBPRegNum].Rule == frame.RuleUndefined {
 		fctxt.Regs[amd64DwarfBPRegNum] = frame.DWRule{
-			Rule:   frame.RuleRegOffset,
+			Rule:   frame.RuleFramePointer,
 			Reg:    amd64DwarfBPRegNum,
 			Offset: 0,
 		}
@@ -241,7 +270,7 @@ func (a *AMD64) RegistersToDwarfRegisters(regs Registers) op.DwarfRegisters {
 		}
 	}
 
-	return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum}
+	return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum, BPRegNum: amd64DwarfBPRegNum}
 }
 
 // GoroutineToDwarfRegisters extract the saved DWARF registers from a parked
@@ -251,5 +280,5 @@ func (a *AMD64) GoroutineToDwarfRegisters(g *G) op.DwarfRegisters {
 	dregs[amd64DwarfIPRegNum] = op.DwarfRegisterFromUint64(g.PC)
 	dregs[amd64DwarfSPRegNum] = op.DwarfRegisterFromUint64(g.SP)
 	dregs[amd64DwarfBPRegNum] = op.DwarfRegisterFromUint64(g.BP)
-	return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum}
+	return op.DwarfRegisters{Regs: dregs, ByteOrder: binary.LittleEndian, PCRegNum: amd64DwarfIPRegNum, SPRegNum: amd64DwarfSPRegNum, BPRegNum: amd64DwarfBPRegNum}
 }

pkg/proc/eval.go

@@ -183,7 +183,7 @@ func (scope *EvalScope) evalAST(t ast.Expr) (*Variable, error) {
 				}
 				return scope.Gvar.clone(), nil
 			} else if maybePkg.Name == "runtime" && node.Sel.Name == "frameoff" {
-				return newConstant(constant.MakeInt64(scope.Regs.CFA-int64(scope.StackHi)), scope.Mem), nil
+				return newConstant(constant.MakeInt64(scope.frameOffset), scope.Mem), nil
 			} else if v, err := scope.packageVarAddr(maybePkg.Name + "." + node.Sel.Name); err == nil {
 				return v, nil
 			}

pkg/proc/proc.go

@@ -286,7 +286,7 @@ func StepOut(dbp Process) error {
 	}
 
 	sameGCond := SameGoroutineCondition(selg)
-	retFrameCond := andFrameoffCondition(sameGCond, retframe.Regs.CFA-int64(retframe.StackHi))
+	retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset())
 
 	var deferpc uint64 = 0
 	if filepath.Ext(topframe.Current.File) == ".go" {
@@ -358,7 +358,7 @@ func GoroutinesInfo(dbp Process) ([]*G, error) {
 	}
 
 	var (
-		threadg = map[int]Thread{}
+		threadg = map[int]*G{}
 		allg    []*G
 		rdr     = dbp.BinInfo().DwarfReader()
 	)
@@ -370,7 +370,7 @@ func GoroutinesInfo(dbp Process) ([]*G, error) {
 		}
 		g, _ := GetG(th)
 		if g != nil {
-			threadg[g.ID] = th
+			threadg[g.ID] = g
 		}
 	}
 
@@ -410,14 +410,15 @@ func GoroutinesInfo(dbp Process) ([]*G, error) {
 		if err != nil {
 			return nil, err
 		}
-		if thread, allocated := threadg[g.ID]; allocated {
-			loc, err := thread.Location()
+		if thg, allocated := threadg[g.ID]; allocated {
+			loc, err := thg.Thread.Location()
 			if err != nil {
 				return nil, err
 			}
-			g.Thread = thread
+			g.Thread = thg.Thread
 			// Prefer actual thread location information.
 			g.CurrentLoc = *loc
+			g.SystemStack = thg.SystemStack
 		}
 		if g.Status != Gdead {
 			allg = append(allg, g)
@@ -481,10 +482,10 @@ func ConvertEvalScope(dbp Process, gid, frame int) (*EvalScope, error) {
 		return nil, fmt.Errorf("Frame %d does not exist in goroutine %d", frame, gid)
 	}
 
-	return &EvalScope{locs[frame].Current.PC, locs[frame].Regs, thread, g.variable, dbp.BinInfo(), locs[frame].StackHi}, nil
+	return &EvalScope{locs[frame].Current.PC, locs[frame].Regs, thread, g.variable, dbp.BinInfo(), locs[frame].FrameOffset()}, nil
 }
 
 // FrameToScope returns a new EvalScope for this frame
 func FrameToScope(p Process, frame Stackframe) *EvalScope {
-	return &EvalScope{frame.Current.PC, frame.Regs, p.CurrentThread(), nil, p.BinInfo(), frame.StackHi}
+	return &EvalScope{frame.Current.PC, frame.Regs, p.CurrentThread(), nil, p.BinInfo(), frame.FrameOffset()}
 }

pkg/proc/proc_test.go

@@ -3121,3 +3121,219 @@ func TestIssue844(t *testing.T) {
 		}
 	})
 }
+
+func logStacktrace(t *testing.T, frames []proc.Stackframe) {
+	for j := range frames {
+		name := "?"
+		if frames[j].Current.Fn != nil {
+			name = frames[j].Current.Fn.Name
+		}
+
+		t.Logf("\t%#x %#x %#x %s at %s:%d\n", frames[j].Call.PC, frames[j].FrameOffset(), frames[j].FramePointerOffset(), name, filepath.Base(frames[j].Call.File), frames[j].Call.Line)
+	}
+}
+
+// stacktraceCheck checks that all the functions listed in tc appear in
+// frames in the same order.
+// Checks that all the functions in tc starting with "C." or with "!" are in
+// a systemstack frame.
+// Returns a slice m where m[i] is the index in frames of the function tc[i]
+// or nil if any check fails.
+func stacktraceCheck(t *testing.T, tc []string, frames []proc.Stackframe) []int {
+	m := make([]int, len(tc))
+	i, j := 0, 0
+	for i < len(tc) {
+		tcname := tc[i]
+		tcsystem := strings.HasPrefix(tcname, "C.")
+		if tcname[0] == '!' {
+			tcsystem = true
+			tcname = tcname[1:]
+		}
+		for j < len(frames) {
+			name := "?"
+			if frames[j].Current.Fn != nil {
+				name = frames[j].Current.Fn.Name
+			}
+			if name == tcname {
+				m[i] = j
+				if tcsystem != frames[j].SystemStack {
+					t.Logf("system stack check failed for frame %d (expected %v got %v)", j, tcsystem, frames[j].SystemStack)
+					t.Logf("expected: %v\n", tc)
+					return nil
+				}
+				break
+			}
+
+			j++
+		}
+		if j >= len(frames) {
+			t.Logf("couldn't find frame %d %s", i, tc)
+			t.Logf("expected: %v\n", tc)
+			return nil
+		}
+
+		i++
+	}
+	return m
+}
+
+func frameInFile(frame proc.Stackframe, file string) bool {
+	for _, loc := range []proc.Location{frame.Current, frame.Call} {
+		if !strings.HasSuffix(loc.File, "/"+file) && !strings.HasSuffix(loc.File, "\\"+file) {
+			return false
+		}
+		if loc.Line <= 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func TestCgoStacktrace(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		ver, _ := goversion.Parse(runtime.Version())
+		if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 9, -1, 0, 0, ""}) {
+			t.Skip("disabled on windows with go before version 1.9")
+		}
+	}
+	if runtime.GOOS == "darwin" {
+		ver, _ := goversion.Parse(runtime.Version())
+		if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 8, -1, 0, 0, ""}) {
+			t.Skip("disabled on macOS with go before version 1.8")
+		}
+	}
+
+	// Tests that:
+	// a) we correctly identify the goroutine while we are executing cgo code
+	// b) that we can stitch together the system stack (where cgo code
+	// executes) and the normal goroutine stack
+
+	// Each test case describes how the stack trace should appear after a
+	// continue. The first function on each test case is the topmost function
+	// that should be found on the stack, the actual stack trace can have more
+	// frame than those listed here but all the frames listed must appear in
+	// the specified order.
+	testCases := [][]string{
+		[]string{"main.main"},
+		[]string{"C.helloworld_pt2", "C.helloworld", "main.main"},
+		[]string{"main.helloWorldS", "main.helloWorld", "C.helloworld_pt2", "C.helloworld", "main.main"},
+		[]string{"C.helloworld_pt4", "C.helloworld_pt3", "main.helloWorldS", "main.helloWorld", "C.helloworld_pt2", "C.helloworld", "main.main"},
+		[]string{"main.helloWorld2", "C.helloworld_pt4", "C.helloworld_pt3", "main.helloWorldS", "main.helloWorld", "C.helloworld_pt2", "C.helloworld", "main.main"}}
+
+	var gid int
+
+	frameOffs := map[string]int64{}
+	framePointerOffs := map[string]int64{}
+
+	withTestProcess("cgostacktest/", t, func(p proc.Process, fixture protest.Fixture) {
+		for itidx, tc := range testCases {
+			assertNoError(proc.Continue(p), t, fmt.Sprintf("Continue at iteration step %d", itidx))
+
+			g, err := proc.GetG(p.CurrentThread())
+			assertNoError(err, t, fmt.Sprintf("GetG at iteration step %d", itidx))
+
+			if itidx == 0 {
+				gid = g.ID
+			} else {
+				if gid != g.ID {
+					t.Fatalf("wrong goroutine id at iteration step %d (expected %d got %d)", itidx, gid, g.ID)
+				}
+			}
+
+			frames, err := g.Stacktrace(100)
+			assertNoError(err, t, fmt.Sprintf("Stacktrace at iteration step %d", itidx))
+
+			t.Logf("iteration step %d", itidx)
+			logStacktrace(t, frames)
+
+			m := stacktraceCheck(t, tc, frames)
+			mismatch := (m == nil)
+
+			for i, j := range m {
+				if strings.HasPrefix(tc[i], "C.hellow") {
+					if !frameInFile(frames[j], "hello.c") {
+						t.Logf("position in %q is %s:%d (call %s:%d)", tc[i], frames[j].Current.File, frames[j].Current.Line, frames[j].Call.File, frames[j].Call.Line)
+						mismatch = true
+						break
+					}
+				}
+				if frameOff, ok := frameOffs[tc[i]]; ok {
+					if frameOff != frames[j].FrameOffset() {
+						t.Logf("frame %s offset mismatch", tc[i])
+					}
+					if framePointerOffs[tc[i]] != frames[j].FramePointerOffset() {
+						t.Logf("frame %s pointer offset mismatch", tc[i])
+					}
+				} else {
+					frameOffs[tc[i]] = frames[j].FrameOffset()
+					framePointerOffs[tc[i]] = frames[j].FramePointerOffset()
+				}
+			}
+
+			// also check that ThreadStacktrace produces the same list of frames
+			threadFrames, err := proc.ThreadStacktrace(p.CurrentThread(), 100)
+			assertNoError(err, t, fmt.Sprintf("ThreadStacktrace at iteration step %d", itidx))
+
+			if len(threadFrames) != len(frames) {
+				mismatch = true
+			} else {
+				for j := range frames {
+					if frames[j].Current.File != threadFrames[j].Current.File || frames[j].Current.Line != threadFrames[j].Current.Line {
+						t.Logf("stack mismatch between goroutine stacktrace and thread stacktrace")
+						t.Logf("thread stacktrace:")
+						logStacktrace(t, threadFrames)
+						mismatch = true
+						break
+					}
+				}
+			}
+			if mismatch {
+				t.Fatal("see previous loglines")
+			}
+		}
+	})
+}
+
+func TestCgoSources(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		ver, _ := goversion.Parse(runtime.Version())
+		if ver.Major > 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 9, -1, 0, 0, ""}) {
+			t.Skip("disabled on windows with go before version 1.9")
+		}
+	}
+
+	withTestProcess("cgostacktest/", t, func(p proc.Process, fixture protest.Fixture) {
+		sources := p.BinInfo().Sources
+		for _, needle := range []string{"main.go", "hello.c"} {
+			found := false
+			for _, k := range sources {
+				if strings.HasSuffix(k, "/"+needle) || strings.HasSuffix(k, "\\"+needle) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				t.Errorf("File %s not found", needle)
+			}
+		}
+	})
+}
+
+func TestSystemstackStacktrace(t *testing.T) {
+	// check that we can follow a stack switch initiated by runtime.systemstack()
+	withTestProcess("panic", t, func(p proc.Process, fixture protest.Fixture) {
+		_, err := setFunctionBreakpoint(p, "runtime.startpanic_m")
+		assertNoError(err, t, "setFunctionBreakpoint()")
+		assertNoError(proc.Continue(p), t, "first continue")
+		assertNoError(proc.Continue(p), t, "second continue")
+		g, err := proc.GetG(p.CurrentThread())
+		assertNoError(err, t, "GetG")
+		frames, err := g.Stacktrace(100)
+		assertNoError(err, t, "stacktrace")
+		logStacktrace(t, frames)
+		m := stacktraceCheck(t, []string{"!runtime.startpanic_m", "!runtime.systemstack", "runtime.startpanic", "main.main"}, frames)
+		if m == nil {
+			t.Fatal("see previous loglines")
+		}
+	})
+}

pkg/proc/stack.go

@@ -12,8 +12,6 @@ import (
 // This code is partly adaped from runtime.gentraceback in
 // $GOROOT/src/runtime/traceback.go
 
-const runtimeStackBarrier = "runtime.stackBarrier"
-
 // NoReturnAddr is returned when return address
 // could not be found during stack trace.
 type NoReturnAddr struct {
@@ -33,24 +31,50 @@ type Stackframe struct {
 	// Frame registers.
 	Regs op.DwarfRegisters
 	// High address of the stack.
-	StackHi uint64
+	stackHi uint64
 	// Return address for this stack frame (as read from the stack frame itself).
 	Ret uint64
 	// Address to the memory location containing the return address
 	addrret uint64
 	// Err is set if an error occoured during stacktrace
 	Err error
+	// SystemStack is true if this frame belongs to a system stack.
+	SystemStack bool
+}
+
+// FrameOffset returns the address of the stack frame, absolute for system
+// stack frames or as an offset from stackhi for goroutine stacks (a
+// negative value).
+func (frame *Stackframe) FrameOffset() int64 {
+	if frame.SystemStack {
+		return frame.Regs.CFA
+	}
+	return frame.Regs.CFA - int64(frame.stackHi)
+}
+
+// FramePointerOffset returns the value of the frame pointer, absolute for
+// system stack frames or as an offset from stackhi for goroutine stacks (a
+// negative value).
+func (frame *Stackframe) FramePointerOffset() int64 {
+	if frame.SystemStack {
+		return int64(frame.Regs.BP())
+	}
+	return int64(frame.Regs.BP()) - int64(frame.stackHi)
 }
 
 // ThreadStacktrace returns the stack trace for thread.
 // Note the locations in the array are return addresses not call addresses.
 func ThreadStacktrace(thread Thread, depth int) ([]Stackframe, error) {
-	regs, err := thread.Registers(true)
-	if err != nil {
-		return nil, err
+	g, _ := GetG(thread)
+	if g == nil {
+		regs, err := thread.Registers(true)
+		if err != nil {
+			return nil, err
+		}
+		it := newStackIterator(thread.BinInfo(), thread, thread.BinInfo().Arch.RegistersToDwarfRegisters(regs), 0, nil, -1, nil)
+		return it.stacktrace(depth)
 	}
-	it := newStackIterator(thread.BinInfo(), thread, thread.BinInfo().Arch.RegistersToDwarfRegisters(regs), 0, nil, -1)
-	return it.stacktrace(depth)
+	return g.Stacktrace(depth)
 }
 
 func (g *G) stackIterator() (*stackIterator, error) {
@@ -58,14 +82,15 @@ func (g *G) stackIterator() (*stackIterator, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	if g.Thread != nil {
 		regs, err := g.Thread.Registers(true)
 		if err != nil {
 			return nil, err
 		}
-		return newStackIterator(g.variable.bi, g.Thread, g.variable.bi.Arch.RegistersToDwarfRegisters(regs), g.stackhi, stkbar, g.stkbarPos), nil
+		return newStackIterator(g.variable.bi, g.Thread, g.variable.bi.Arch.RegistersToDwarfRegisters(regs), g.stackhi, stkbar, g.stkbarPos, g), nil
 	}
-	return newStackIterator(g.variable.bi, g.variable.mem, g.variable.bi.Arch.GoroutineToDwarfRegisters(g), g.stackhi, stkbar, g.stkbarPos), nil
+	return newStackIterator(g.variable.bi, g.variable.mem, g.variable.bi.Arch.GoroutineToDwarfRegisters(g), g.stackhi, stkbar, g.stkbarPos, g), nil
 }
 
 // Stacktrace returns the stack trace for a goroutine.
@@ -98,11 +123,15 @@ type stackIterator struct {
 	err   error
 
 	stackhi        uint64
+	systemstack    bool
 	stackBarrierPC uint64
 	stkbar         []savedLR
 
 	// regs is the register set for the current frame
 	regs op.DwarfRegisters
+
+	g           *G     // the goroutine being stacktraced, nil if we are stacktracing a goroutine-less thread
+	g0_sched_sp uint64 // value of g0.sched.sp (see comments around its use)
 }
 
 type savedLR struct {
@@ -110,13 +139,13 @@ type savedLR struct {
 	val uint64
 }
 
-func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegisters, stackhi uint64, stkbar []savedLR, stkbarPos int) *stackIterator {
-	stackBarrierFunc := bi.LookupFunc[runtimeStackBarrier] // stack barriers were removed in Go 1.9
+func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegisters, stackhi uint64, stkbar []savedLR, stkbarPos int, g *G) *stackIterator {
+	stackBarrierFunc := bi.LookupFunc["runtime.stackBarrier"] // stack barriers were removed in Go 1.9
 	var stackBarrierPC uint64
 	if stackBarrierFunc != nil && stkbar != nil {
 		stackBarrierPC = stackBarrierFunc.Entry
 		fn := bi.PCToFunc(regs.PC())
-		if fn != nil && fn.Name == runtimeStackBarrier {
+		if fn != nil && fn.Name == "runtime.stackBarrier" {
 			// We caught the goroutine as it's executing the stack barrier, we must
 			// determine whether or not g.stackPos has already been incremented or not.
 			if len(stkbar) > 0 && stkbar[stkbarPos].ptr < regs.SP() {
@@ -130,7 +159,19 @@ func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegiste
 		}
 		stkbar = stkbar[stkbarPos:]
 	}
-	return &stackIterator{pc: regs.PC(), regs: regs, top: true, bi: bi, mem: mem, err: nil, atend: false, stackhi: stackhi, stackBarrierPC: stackBarrierPC, stkbar: stkbar}
+	var g0_sched_sp uint64
+	systemstack := true
+	if g != nil {
+		systemstack = g.SystemStack
+		g0var, _ := g.variable.fieldVariable("m").structMember("g0")
+		if g0var != nil {
+			g0, _ := g0var.parseG()
+			if g0 != nil {
+				g0_sched_sp = g0.SP
+			}
+		}
+	}
+	return &stackIterator{pc: regs.PC(), regs: regs, top: true, bi: bi, mem: mem, err: nil, atend: false, stackhi: stackhi, stackBarrierPC: stackBarrierPC, stkbar: stkbar, systemstack: systemstack, g: g, g0_sched_sp: g0_sched_sp}
 }
 
 // Next points the iterator to the next stack frame.
@@ -152,9 +193,7 @@ func (it *stackIterator) Next() bool {
 		it.stkbar = it.stkbar[1:]
 	}
 
-	// Look for "top of stack" functions.
-	if it.frame.Current.Fn != nil && (it.frame.Current.Fn.Name == "runtime.goexit" || it.frame.Current.Fn.Name == "runtime.rt0_go" || it.frame.Current.Fn.Name == "runtime.mcall") {
-		it.atend = true
+	if it.switchStack() {
 		return true
 	}
 
@@ -164,6 +203,109 @@ func (it *stackIterator) Next() bool {
 	return true
 }
 
+// asmcgocallSPOffsetSaveSlot is the offset from systemstack.SP where
+// (goroutine.SP - StackHi) is saved in runtime.asmcgocall after the stack
+// switch happens.
+const asmcgocallSPOffsetSaveSlot = 0x28
+
+// switchStack will use the current frame to determine if it's time to
+// switch between the system stack and the goroutine stack or vice versa.
+// Sets it.atend when the top of the stack is reached.
+func (it *stackIterator) switchStack() bool {
+	if it.frame.Current.Fn == nil {
+		return false
+	}
+	switch it.frame.Current.Fn.Name {
+	case "runtime.asmcgocall":
+		if it.top || !it.systemstack {
+			return false
+		}
+
+		// switch from system stack to goroutine stack
+
+		off, _ := readIntRaw(it.mem, uintptr(it.regs.SP()+asmcgocallSPOffsetSaveSlot), int64(it.bi.Arch.PtrSize())) // reads "offset of SP from StackHi" from where runtime.asmcgocall saved it
+		oldsp := it.regs.SP()
+		it.regs.Reg(it.regs.SPRegNum).Uint64Val = uint64(int64(it.stackhi) - off)
+
+		// runtime.asmcgocall can also be called from inside the system stack,
+		// in that case no stack switch actually happens
+		if it.regs.SP() == oldsp {
+			return false
+		}
+		it.systemstack = false
+
+		// advances to the next frame in the call stack
+		it.frame.addrret = uint64(int64(it.regs.SP()) + int64(it.bi.Arch.PtrSize()))
+		it.frame.Ret, _ = readUintRaw(it.mem, uintptr(it.frame.addrret), int64(it.bi.Arch.PtrSize()))
+		it.pc = it.frame.Ret
+
+		it.top = false
+		return true
+
+	case "runtime.mstart":
+		if it.top || !it.systemstack || it.g == nil {
+			return false
+		}
+
+		// Calls to runtime.systemstack will switch to the systemstack then:
+		// 1. alter the goroutine stack so that it looks like systemstack_switch
+		//    was called
+		// 2. alter the system stack so that it looks like the bottom-most frame
+		//    belongs to runtime.mstart
+		// If we find a runtime.mstart frame on the system stack of a goroutine
+		// parked on runtime.systemstack_switch we assume runtime.systemstack was
+		// called and continue tracing from the parked position.
+
+		if fn := it.bi.PCToFunc(it.g.PC); fn == nil || fn.Name != "runtime.systemstack_switch" {
+			return false
+		}
+		it.systemstack = false
+		it.pc = it.g.PC
+		it.regs.Reg(it.regs.SPRegNum).Uint64Val = it.g.SP
+		it.regs.Reg(it.regs.BPRegNum).Uint64Val = it.g.BP
+		it.top = false
+		return true
+
+	case "runtime.cgocallback_gofunc":
+		// For a detailed description of how this works read the long comment at
+		// the start of $GOROOT/src/runtime/cgocall.go and the source code of
+		// runtime.cgocallback_gofunc in $GOROOT/src/runtime/asm_amd64.s
+		//
+		// When a C functions calls back into go it will eventually call into
+		// runtime.cgocallback_gofunc which is the function that does the stack
+		// switch from the system stack back into the goroutine stack
+		// Since we are going backwards on the stack here we see the transition
+		// as goroutine stack -> system stack.
+
+		if it.top || it.systemstack {
+			return false
+		}
+
+		if it.g0_sched_sp <= 0 {
+			return false
+		}
+		// entering the system stack
+		it.regs.Reg(it.regs.SPRegNum).Uint64Val = it.g0_sched_sp
+		// reads the previous value of g0.sched.sp that runtime.cgocallback_gofunc saved on the stack
+		it.g0_sched_sp, _ = readUintRaw(it.mem, uintptr(it.regs.SP()), int64(it.bi.Arch.PtrSize()))
+		it.top = false
+		callFrameRegs, ret, retaddr := it.advanceRegs()
+		frameOnSystemStack := it.newStackframe(ret, retaddr)
+		it.pc = frameOnSystemStack.Ret
+		it.regs = callFrameRegs
+		it.systemstack = true
+		return true
+
+	case "runtime.goexit", "runtime.rt0_go", "runtime.mcall":
+		// Look for "top of stack" functions.
+		it.atend = true
+		return true
+
+	default:
+		return false
+	}
+}
+
 // Frame returns the frame the iterator is pointing at.
 func (it *stackIterator) Frame() Stackframe {
 	return it.frame
@@ -199,14 +341,25 @@ func (it *stackIterator) newStackframe(ret, retaddr uint64) Stackframe {
 	} else {
 		it.regs.FrameBase = it.frameBase(fn)
 	}
-	r := Stackframe{Current: Location{PC: it.pc, File: f, Line: l, Fn: fn}, Regs: it.regs, Ret: ret, addrret: retaddr, StackHi: it.stackhi}
+	r := Stackframe{Current: Location{PC: it.pc, File: f, Line: l, Fn: fn}, Regs: it.regs, Ret: ret, addrret: retaddr, stackHi: it.stackhi, SystemStack: it.systemstack}
 	if !it.top {
-		r.Call.File, r.Call.Line, r.Call.Fn = it.bi.PCToLine(it.pc - 1)
-		if r.Call.Fn == nil {
-			r.Call.File = "?"
-			r.Call.Line = -1
+		fnname := ""
+		if r.Current.Fn != nil {
+			fnname = r.Current.Fn.Name
+		}
+		switch fnname {
+		case "runtime.mstart", "runtime.systemstack_switch":
+			// these frames are inserted by runtime.systemstack and there is no CALL
+			// instruction to look for at pc - 1
+			r.Call = r.Current
+		default:
+			r.Call.File, r.Call.Line, r.Call.Fn = it.bi.PCToLine(it.pc - 1)
+			if r.Call.Fn == nil {
+				r.Call.File = "?"
+				r.Call.Line = -1
+			}
+			r.Call.PC = r.Current.PC
 		}
-		r.Call.PC = r.Current.PC
 	} else {
 		r.Call = r.Current
 	}
@@ -240,9 +393,9 @@ func (it *stackIterator) advanceRegs() (callFrameRegs op.DwarfRegisters, ret uin
 	fde, err := it.bi.frameEntries.FDEForPC(it.pc)
 	var framectx *frame.FrameContext
 	if _, nofde := err.(*frame.NoFDEForPCError); nofde {
-		framectx = it.bi.Arch.FixFrameUnwindContext(nil)
+		framectx = it.bi.Arch.FixFrameUnwindContext(nil, it.pc, it.bi)
 	} else {
-		framectx = it.bi.Arch.FixFrameUnwindContext(fde.EstablishFrame(it.pc))
+		framectx = it.bi.Arch.FixFrameUnwindContext(fde.EstablishFrame(it.pc), it.pc, it.bi)
 	}
 
 	cfareg, err := it.executeFrameRegRule(0, framectx.CFA, 0)
@@ -252,7 +405,7 @@ func (it *stackIterator) advanceRegs() (callFrameRegs op.DwarfRegisters, ret uin
 	}
 	it.regs.CFA = int64(cfareg.Uint64Val)
 
-	callFrameRegs = op.DwarfRegisters{ByteOrder: it.regs.ByteOrder, PCRegNum: it.regs.PCRegNum, SPRegNum: it.regs.SPRegNum}
+	callFrameRegs = op.DwarfRegisters{ByteOrder: it.regs.ByteOrder, PCRegNum: it.regs.PCRegNum, SPRegNum: it.regs.SPRegNum, BPRegNum: it.regs.BPRegNum}
 
 	// According to the standard the compiler should be responsible for emitting
 	// rules for the RSP register so that it can then be used to calculate CFA,
@@ -289,7 +442,8 @@ func (it *stackIterator) executeFrameRegRule(regnum uint64, rule frame.DWRule, c
 	case frame.RuleUndefined:
 		return nil, nil
 	case frame.RuleSameVal:
-		return it.regs.Reg(regnum), nil
+		reg := *it.regs.Reg(regnum)
+		return &reg, nil
 	case frame.RuleOffset:
 		return it.readRegisterAt(regnum, uint64(cfa+rule.Offset))
 	case frame.RuleValOffset:
@@ -315,11 +469,17 @@ func (it *stackIterator) executeFrameRegRule(regnum uint64, rule frame.DWRule, c
 			return nil, nil
 		}
 		return op.DwarfRegisterFromUint64(uint64(int64(it.regs.Uint64Val(rule.Reg)) + rule.Offset)), nil
-	case frame.RuleRegOffset:
-		if it.regs.Reg(rule.Reg) == nil {
+	case frame.RuleFramePointer:
+		curReg := it.regs.Reg(rule.Reg)
+		if curReg == nil {
 			return nil, nil
 		}
-		return it.readRegisterAt(regnum, uint64(int64(it.regs.Uint64Val(rule.Reg))+rule.Offset))
+		if curReg.Uint64Val <= uint64(cfa) {
+			return it.readRegisterAt(regnum, curReg.Uint64Val)
+		} else {
+			newReg := *curReg
+			return &newReg, nil
+		}
 	}
 }
 

pkg/proc/threads.go

@@ -137,8 +137,8 @@ func next(dbp Process, stepInto bool) error {
 	}
 
 	sameGCond := SameGoroutineCondition(selg)
-	retFrameCond := andFrameoffCondition(sameGCond, retframe.Regs.CFA-int64(retframe.StackHi))
-	sameFrameCond := andFrameoffCondition(sameGCond, topframe.Regs.CFA-int64(topframe.StackHi))
+	retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset())
+	sameFrameCond := andFrameoffCondition(sameGCond, topframe.FrameOffset())
 	var sameOrRetFrameCond ast.Expr
 	if sameGCond != nil {
 		sameOrRetFrameCond = &ast.BinaryExpr{
@@ -146,8 +146,8 @@ func next(dbp Process, stepInto bool) error {
 			X:  sameGCond,
 			Y: &ast.BinaryExpr{
 				Op: token.LOR,
-				X:  frameoffCondition(topframe.Regs.CFA - int64(topframe.StackHi)),
-				Y:  frameoffCondition(retframe.Regs.CFA - int64(retframe.StackHi)),
+				X:  frameoffCondition(topframe.FrameOffset()),
+				Y:  frameoffCondition(retframe.FrameOffset()),
 			},
 		}
 	}
@@ -361,11 +361,27 @@ func GetG(thread Thread) (g *G, err error) {
 	}
 
 	g, err = gaddr.parseG()
-	if err == nil {
-		g.Thread = thread
-		if loc, err := thread.Location(); err == nil {
-			g.CurrentLoc = *loc
+	if err != nil {
+		return
+	}
+	if g.ID == 0 {
+		// The runtime uses a special goroutine with ID == 0 to mark that the
+		// current goroutine is executing on the system stack (sometimes also
+		// referred to as the g0 stack or scheduler stack, I'm not sure if there's
+		// actually any difference between those).
+		// For our purposes it's better if we always return the real goroutine
+		// since the rest of the code assumes the goroutine ID is univocal.
+		// The real 'current goroutine' is stored in g0.m.curg
+		curgvar, _ := g.variable.fieldVariable("m").structMember("curg")
+		g, err = curgvar.parseG()
+		if err != nil {
+			return
 		}
+		g.SystemStack = true
+	}
+	g.Thread = thread
+	if loc, err := thread.Location(); err == nil {
+		g.CurrentLoc = *loc
 	}
 	return
 }
@@ -395,7 +411,7 @@ func GoroutineScope(thread Thread) (*EvalScope, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &EvalScope{locations[0].Current.PC, locations[0].Regs, thread, g.variable, thread.BinInfo(), g.stackhi}, nil
+	return &EvalScope{locations[0].Current.PC, locations[0].Regs, thread, g.variable, thread.BinInfo(), locations[0].FrameOffset()}, nil
 }
 
 func onRuntimeBreakpoint(thread Thread) bool {

pkg/proc/variables.go

@@ -138,6 +138,8 @@ type G struct {
 	stkbarPos  int       // stkbarPos field of g struct
 	stackhi    uint64    // value of stack.hi
 
+	SystemStack bool // SystemStack is true if this goroutine is currently executing on a system stack.
+
 	// Information on goroutine location
 	CurrentLoc Location
 
@@ -155,7 +157,8 @@ type EvalScope struct {
 	Mem     MemoryReadWriter // Target's memory
 	Gvar    *Variable
 	BinInfo *BinaryInfo
-	StackHi uint64
+
+	frameOffset int64
 }
 
 // IsNilErr is returned when a variable is nil.

service/debugger/debugger.go

@@ -854,7 +854,7 @@ func (d *Debugger) convertStacktrace(rawlocs []proc.Stackframe, cfg *proc.LoadCo
 	for i := range rawlocs {
 		frame := api.Stackframe{
 			Location:    api.ConvertLocation(rawlocs[i].Call),
-			FrameOffset: rawlocs[i].Regs.CFA - int64(rawlocs[i].StackHi),
+			FrameOffset: rawlocs[i].FrameOffset(),
 		}
 		if rawlocs[i].Err != nil {
 			frame.Err = rawlocs[i].Err.Error()