proc: support inlining

Go 1.10 added inlined calls to debug_info, this commit adds support
for DW_TAG_inlined_call to delve, both for stack traces (where
inlined calls will appear as normal stack frames) and to correct
the behavior of next, step and stepout.

The calls to Next and Frame of stackIterator continue to work
unchanged and only return real stack frames, after reading each line
appendInlinedCalls is called to unpacked all the inlined calls that
involve the current PC.

The fake stack frames produced by appendInlinedCalls are
distinguished from real stack frames by having the Inlined attribute
set to true. Also their Current and Call locations are treated
differently. The Call location will be changed to represent the
position inside the inlined call, while the Current location will
always reference the real stack frame. This is done because:

* next, step and stepout need to access the debug_info entry of
the real function they are stepping through
* we are already manipulating Call in different ways while Current
is just what we read from the call stack

The strategy remains mostly the same, we disassemble the function
and we set a breakpoint on each instruction corresponding to a
different file:line. The function in question will be the one
corresponding to the first real (i.e. non-inlined) stack frame.

* If the current function contains inlined calls, 'next' will not
set any breakpoints on instructions that belong to inlined calls. We
do not do this for 'step'.

* If we are inside an inlined call that makes other inlined
functions, 'next' will not set any breakpoints that belong to
inlined calls that are children of the current inlined call.

* If the current function is inlined the breakpoint on the return
address won't be set, because inlined frames don't have a return
address.

* The code we use for stepout doesn't work at all if we are inside
an inlined call, instead we call 'next' but instruct it to remove
all PCs belonging to the current inlined call.

_fixtures/testinline.go

+package main
+
+import "fmt"
+
+func inlineThis(a int) int {
+	z := a * a
+	return z + a/a
+}
+
+func initialize(a, b *int) {
+	*a = 3
+	*b = 4
+}
+
+func main() {
+	var a, b int
+	initialize(&a, &b)
+	a = inlineThis(a)
+	b = inlineThis(b)
+	fmt.Printf("%d %d\n", a, b)
+}

pkg/dwarf/line/state_machine.go

@@ -119,7 +119,6 @@ func (lineInfo *DebugLineInfo) AllPCsForFileLine(f string, l int) (pcs []uint64)
 			if sm.valid {
 				pcs = append(pcs, sm.address)
 			}
-			line := sm.line
 			// Keep going until we're on a different line. We only care about
 			// when a line comes back around (i.e. for loop) so get to next line,
 			// and try to find the line we care about again.
@@ -127,7 +126,7 @@ func (lineInfo *DebugLineInfo) AllPCsForFileLine(f string, l int) (pcs []uint64)
 				if err := sm.next(); err != nil {
 					break
 				}
-				if line < sm.line {
+				if l != sm.line {
 					break
 				}
 			}

pkg/dwarf/reader/reader.go

@@ -365,3 +365,82 @@ func LoadAbstractOrigin(entry *dwarf.Entry, aordr *dwarf.Reader) (Entry, dwarf.O
 
 	return compositeEntry(r), entry.Offset
 }
+
+// InlineStackReader provides a way to read the stack of inlined calls at a
+// specified PC address.
+type InlineStackReader struct {
+	dwarf  *dwarf.Data
+	reader *dwarf.Reader
+	entry  *dwarf.Entry
+	depth  int
+	pc     uint64
+	err    error
+}
+
+// InlineStack returns an InlineStackReader for the specified function and
+// PC address.
+// If pc is 0 then all inlined calls will be returned.
+func InlineStack(dwarf *dwarf.Data, fnoff dwarf.Offset, pc uint64) *InlineStackReader {
+	reader := dwarf.Reader()
+	reader.Seek(fnoff)
+	return &InlineStackReader{dwarf: dwarf, reader: reader, entry: nil, depth: 0, pc: pc}
+}
+
+// Next reads next inlined call in the stack, returns false if there aren't any.
+func (irdr *InlineStackReader) Next() bool {
+	if irdr.err != nil {
+		return false
+	}
+
+	for {
+		irdr.entry, irdr.err = irdr.reader.Next()
+		if irdr.entry == nil || irdr.err != nil {
+			return false
+		}
+
+		switch irdr.entry.Tag {
+		case 0:
+			irdr.depth--
+			if irdr.depth == 0 {
+				return false
+			}
+
+		case dwarf.TagLexDwarfBlock, dwarf.TagSubprogram, dwarf.TagInlinedSubroutine:
+			var recur bool
+			if irdr.pc != 0 {
+				recur, irdr.err = entryRangesContains(irdr.dwarf, irdr.entry, irdr.pc)
+			} else {
+				recur = true
+			}
+			if recur {
+				irdr.depth++
+				if irdr.entry.Tag == dwarf.TagInlinedSubroutine {
+					return true
+				}
+			} else {
+				if irdr.depth == 0 {
+					return false
+				}
+				irdr.reader.SkipChildren()
+			}
+
+		default:
+			irdr.reader.SkipChildren()
+		}
+	}
+}
+
+// Entry returns the DIE for the current inlined call.
+func (irdr *InlineStackReader) Entry() *dwarf.Entry {
+	return irdr.entry
+}
+
+// Err returns an error, if any was encountered.
+func (irdr *InlineStackReader) Err() error {
+	return irdr.err
+}
+
+// SkipChildren skips all children of the current inlined call.
+func (irdr *InlineStackReader) SkipChildren() {
+	irdr.reader.SkipChildren()
+}

pkg/dwarf/reader/variables.go

@@ -47,10 +47,10 @@ func (vrdr *VariableReader) Next() bool {
 				return false
 			}
 
-		case dwarf.TagLexDwarfBlock, dwarf.TagSubprogram:
+		case dwarf.TagLexDwarfBlock, dwarf.TagSubprogram, dwarf.TagInlinedSubroutine:
 			recur := true
 			if vrdr.onlyVisible {
-				recur, vrdr.err = vrdr.entryRangesContains()
+				recur, vrdr.err = entryRangesContains(vrdr.dwarf, vrdr.entry, vrdr.pc)
 				if vrdr.err != nil {
 					return false
 				}
@@ -77,13 +77,13 @@ func (vrdr *VariableReader) Next() bool {
 	}
 }
 
-func (vrdr *VariableReader) entryRangesContains() (bool, error) {
-	rngs, err := vrdr.dwarf.Ranges(vrdr.entry)
+func entryRangesContains(dwarf *dwarf.Data, entry *dwarf.Entry, pc uint64) (bool, error) {
+	rngs, err := dwarf.Ranges(entry)
 	if err != nil {
 		return false, err
 	}
 	for _, rng := range rngs {
-		if vrdr.pc >= rng[0] && vrdr.pc < rng[1] {
+		if pc >= rng[0] && pc < rng[1] {
 			return true, nil
 		}
 	}

pkg/proc/bininfo.go

@@ -296,6 +296,17 @@ func (bi *BinaryInfo) LineToPC(filename string, lineno int) (pc uint64, fn *Func
 	return
 }
 
+// AllPCsForFileLine returns all PC addresses for the given filename:lineno.
+func (bi *BinaryInfo) AllPCsForFileLine(filename string, lineno int) []uint64 {
+	r := make([]uint64, 0, 1)
+	for _, cu := range bi.compileUnits {
+		if cu.lineInfo.Lookup[filename] != nil {
+			r = append(r, cu.lineInfo.AllPCsForFileLine(filename, lineno)...)
+		}
+	}
+	return r
+}
+
 // PCToFunc returns the function containing the given PC address
 func (bi *BinaryInfo) PCToFunc(pc uint64) *Function {
 	i := sort.Search(len(bi.Functions), func(i int) bool {

pkg/proc/dwarf_expr_test.go

@@ -79,8 +79,8 @@ func uintExprCheck(t *testing.T, scope *proc.EvalScope, expr string, tgt uint64)
 	}
 }
 
-func dwarfExprCheck(t *testing.T, mem proc.MemoryReadWriter, regs op.DwarfRegisters, bi *proc.BinaryInfo, testCases map[string]uint16) *proc.EvalScope {
-	scope := &proc.EvalScope{PC: 0x40100, Regs: regs, Mem: mem, Gvar: nil, BinInfo: bi}
+func dwarfExprCheck(t *testing.T, mem proc.MemoryReadWriter, regs op.DwarfRegisters, bi *proc.BinaryInfo, testCases map[string]uint16, fn *proc.Function) *proc.EvalScope {
+	scope := &proc.EvalScope{Location: proc.Location{PC: 0x40100, Fn: fn}, Regs: regs, Mem: mem, Gvar: nil, BinInfo: bi}
 	for name, value := range testCases {
 		uintExprCheck(t, scope, name, uint64(value))
 	}
@@ -116,12 +116,14 @@ func TestDwarfExprRegisters(t *testing.T) {
 
 	bi := fakeBinaryInfo(t, dwb)
 
+	mainfn := bi.LookupFunc["main.main"]
+
 	mem := newFakeMemory(defaultCFA, uint64(0), uint64(testCases["b"]), uint16(testCases["pair.v"]))
 	regs := core.Registers{LinuxCoreRegisters: &core.LinuxCoreRegisters{}}
 	regs.Rax = uint64(testCases["a"])
 	regs.Rdx = uint64(testCases["c"])
 
-	dwarfExprCheck(t, mem, dwarfRegisters(&regs), bi, testCases)
+	dwarfExprCheck(t, mem, dwarfRegisters(&regs), bi, testCases, mainfn)
 }
 
 func TestDwarfExprComposite(t *testing.T) {
@@ -169,6 +171,8 @@ func TestDwarfExprComposite(t *testing.T) {
 
 	bi := fakeBinaryInfo(t, dwb)
 
+	mainfn := bi.LookupFunc["main.main"]
+
 	mem := newFakeMemory(defaultCFA, uint64(0), uint64(0), uint16(testCases["pair.v"]), []byte(stringVal))
 	var regs core.Registers
 	regs.LinuxCoreRegisters = &core.LinuxCoreRegisters{}
@@ -177,7 +181,7 @@ func TestDwarfExprComposite(t *testing.T) {
 	regs.Rcx = uint64(testCases["pair.k"])
 	regs.Rbx = uint64(testCases["n"])
 
-	scope := dwarfExprCheck(t, mem, dwarfRegisters(&regs), bi, testCases)
+	scope := dwarfExprCheck(t, mem, dwarfRegisters(&regs), bi, testCases, mainfn)
 
 	thevar, err := scope.EvalExpression("s", normalLoadConfig)
 	assertNoError(err, t, fmt.Sprintf("EvalExpression(%s)", "s"))
@@ -207,10 +211,12 @@ func TestDwarfExprLoclist(t *testing.T) {
 
 	bi := fakeBinaryInfo(t, dwb)
 
+	mainfn := bi.LookupFunc["main.main"]
+
 	mem := newFakeMemory(defaultCFA, uint16(before), uint16(after))
 	regs := core.Registers{LinuxCoreRegisters: &core.LinuxCoreRegisters{}}
 
-	scope := &proc.EvalScope{PC: 0x40100, Regs: dwarfRegisters(&regs), Mem: mem, Gvar: nil, BinInfo: bi}
+	scope := &proc.EvalScope{Location: proc.Location{PC: 0x40100, Fn: mainfn}, Regs: dwarfRegisters(&regs), Mem: mem, Gvar: nil, BinInfo: bi}
 
 	uintExprCheck(t, scope, "a", before)
 	scope.PC = 0x40800

pkg/proc/eval.go

@@ -584,9 +584,8 @@ func (scope *EvalScope) evalIdent(node *ast.Ident) (*Variable, error) {
 	}
 
 	// if it's not a local variable then it could be a package variable w/o explicit package name
-	_, _, fn := scope.BinInfo.PCToLine(scope.PC)
-	if fn != nil {
-		if v, err := scope.findGlobal(fn.PackageName() + "." + node.Name); err == nil {
+	if scope.Fn != nil {
+		if v, err := scope.findGlobal(scope.Fn.PackageName() + "." + node.Name); err == nil {
 			v.Name = node.Name
 			return v, nil
 		}

pkg/proc/proc.go

@@ -72,7 +72,7 @@ func Next(dbp Process) (err error) {
 		return fmt.Errorf("next while nexting")
 	}
 
-	if err = next(dbp, false); err != nil {
+	if err = next(dbp, false, false); err != nil {
 		dbp.ClearInternalBreakpoints()
 		return
 	}
@@ -226,7 +226,7 @@ func Step(dbp Process) (err error) {
 		return fmt.Errorf("next while nexting")
 	}
 
-	if err = next(dbp, true); err != nil {
+	if err = next(dbp, true, false); err != nil {
 		switch err.(type) {
 		case ThreadBlockedError: // Noop
 		default:
@@ -293,6 +293,22 @@ func StepOut(dbp Process) error {
 		return err
 	}
 
+	success := false
+	defer func() {
+		if !success {
+			dbp.ClearInternalBreakpoints()
+		}
+	}()
+
+	if topframe.Inlined {
+		if err := next(dbp, false, true); err != nil {
+			return err
+		}
+
+		success = true
+		return Continue(dbp)
+	}
+
 	sameGCond := SameGoroutineCondition(selg)
 	retFrameCond := andFrameoffCondition(sameGCond, retframe.FrameOffset())
 
@@ -310,15 +326,10 @@ func StepOut(dbp Process) error {
 		}
 	}
 
-	if topframe.Ret == 0 && deferpc == 0 {
-		return errors.New("nothing to stepout to")
-	}
-
 	if deferpc != 0 && deferpc != topframe.Current.PC {
 		bp, err := dbp.SetBreakpoint(deferpc, NextDeferBreakpoint, sameGCond)
 		if err != nil {
 			if _, ok := err.(BreakpointExistsError); !ok {
-				dbp.ClearInternalBreakpoints()
 				return err
 			}
 		}
@@ -330,11 +341,14 @@ func StepOut(dbp Process) error {
 		}
 	}
 
+	if topframe.Ret == 0 && deferpc == 0 {
+		return errors.New("nothing to stepout to")
+	}
+
 	if topframe.Ret != 0 {
 		_, err := dbp.SetBreakpoint(topframe.Ret, NextBreakpoint, retFrameCond)
 		if err != nil {
 			if _, isexists := err.(BreakpointExistsError); !isexists {
-				dbp.ClearInternalBreakpoints()
 				return err
 			}
 		}
@@ -344,6 +358,7 @@ func StepOut(dbp Process) error {
 		curthread.SetCurrentBreakpoint()
 	}
 
+	success = true
 	return Continue(dbp)
 }
 
@@ -499,6 +514,7 @@ func FrameToScope(bi *BinaryInfo, thread MemoryReadWriter, g *G, frame Stackfram
 	if g != nil {
 		gvar = g.variable
 	}
-	s := &EvalScope{PC: frame.Call.PC, Regs: frame.Regs, Mem: thread, Gvar: gvar, BinInfo: bi, frameOffset: frame.FrameOffset()}
+	s := &EvalScope{Location: frame.Call, Regs: frame.Regs, Mem: thread, Gvar: gvar, BinInfo: bi, frameOffset: frame.FrameOffset()}
+	s.PC = frame.lastpc
 	return s
 }

pkg/proc/proc_test.go

@@ -3473,3 +3473,191 @@ func TestDisassembleGlobalVars(t *testing.T) {
 		}
 	})
 }
+
+func checkFrame(frame proc.Stackframe, fnname, file string, line int, inlined bool) error {
+	if frame.Call.Fn == nil || frame.Call.Fn.Name != fnname {
+		return fmt.Errorf("wrong function name: %s", fnname)
+	}
+	if frame.Call.File != file || frame.Call.Line != line {
+		return fmt.Errorf("wrong file:line %s:%d", frame.Call.File, frame.Call.Line)
+	}
+	if frame.Inlined != inlined {
+		if inlined {
+			return fmt.Errorf("not inlined")
+		} else {
+			return fmt.Errorf("inlined")
+		}
+	}
+	return nil
+}
+
+func TestInlinedStacktraceAndVariables(t *testing.T) {
+	if ver, _ := goversion.Parse(runtime.Version()); ver.Major >= 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 10, -1, 0, 0, ""}) {
+		// Versions of go before 1.10 do not have DWARF information for inlined calls
+		t.Skip("inlining not supported")
+	}
+
+	firstCallCheck := &scopeCheck{
+		line: 7,
+		ok:   false,
+		varChecks: []varCheck{
+			varCheck{
+				name:   "a",
+				typ:    "int",
+				kind:   reflect.Int,
+				hasVal: true,
+				intVal: 3,
+			},
+			varCheck{
+				name:   "z",
+				typ:    "int",
+				kind:   reflect.Int,
+				hasVal: true,
+				intVal: 9,
+			},
+		},
+	}
+
+	secondCallCheck := &scopeCheck{
+		line: 7,
+		ok:   false,
+		varChecks: []varCheck{
+			varCheck{
+				name:   "a",
+				typ:    "int",
+				kind:   reflect.Int,
+				hasVal: true,
+				intVal: 4,
+			},
+			varCheck{
+				name:   "z",
+				typ:    "int",
+				kind:   reflect.Int,
+				hasVal: true,
+				intVal: 16,
+			},
+		},
+	}
+
+	withTestProcessArgs("testinline", t, ".", []string{}, protest.EnableInlining, func(p proc.Process, fixture protest.Fixture) {
+		pcs := p.BinInfo().AllPCsForFileLine(fixture.Source, 7)
+		if len(pcs) < 2 {
+			t.Fatalf("expected at least two locations for %s:%d (got %d: %#x)", fixture.Source, 6, len(pcs), pcs)
+		}
+		for _, pc := range pcs {
+			_, err := p.SetBreakpoint(pc, proc.UserBreakpoint, nil)
+			assertNoError(err, t, fmt.Sprintf("SetBreakpoint(%#x)", pc))
+		}
+
+		// first inlined call
+		assertNoError(proc.Continue(p), t, "Continue")
+		frames, err := proc.ThreadStacktrace(p.CurrentThread(), 20)
+		assertNoError(err, t, "ThreadStacktrace")
+		t.Logf("Stacktrace:\n")
+		for i := range frames {
+			t.Logf("\t%s at %s:%d\n", frames[i].Call.Fn.Name, frames[i].Call.File, frames[i].Call.Line)
+		}
+
+		if err := checkFrame(frames[0], "main.inlineThis", fixture.Source, 7, true); err != nil {
+			t.Fatalf("Wrong frame 0: %v", err)
+		}
+		if err := checkFrame(frames[1], "main.main", fixture.Source, 18, false); err != nil {
+			t.Fatalf("Wrong frame 1: %v", err)
+		}
+
+		if avar, _ := constant.Int64Val(evalVariable(p, t, "a").Value); avar != 3 {
+			t.Fatalf("value of 'a' variable is not 3 (%d)", avar)
+		}
+		if zvar, _ := constant.Int64Val(evalVariable(p, t, "z").Value); zvar != 9 {
+			t.Fatalf("value of 'z' variable is not 9 (%d)", zvar)
+		}
+
+		if _, ok := firstCallCheck.checkLocalsAndArgs(p, t); !ok {
+			t.Fatalf("exiting for past errors")
+		}
+
+		// second inlined call
+		assertNoError(proc.Continue(p), t, "Continue")
+		frames, err = proc.ThreadStacktrace(p.CurrentThread(), 20)
+		assertNoError(err, t, "ThreadStacktrace (2)")
+		t.Logf("Stacktrace 2:\n")
+		for i := range frames {
+			t.Logf("\t%s at %s:%d\n", frames[i].Call.Fn.Name, frames[i].Call.File, frames[i].Call.Line)
+		}
+
+		if err := checkFrame(frames[0], "main.inlineThis", fixture.Source, 7, true); err != nil {
+			t.Fatalf("Wrong frame 0: %v", err)
+		}
+		if err := checkFrame(frames[1], "main.main", fixture.Source, 19, false); err != nil {
+			t.Fatalf("Wrong frame 1: %v", err)
+		}
+
+		if avar, _ := constant.Int64Val(evalVariable(p, t, "a").Value); avar != 4 {
+			t.Fatalf("value of 'a' variable is not 3 (%d)", avar)
+		}
+		if zvar, _ := constant.Int64Val(evalVariable(p, t, "z").Value); zvar != 16 {
+			t.Fatalf("value of 'z' variable is not 9 (%d)", zvar)
+		}
+		if bvar, err := evalVariableOrError(p, "b"); err == nil {
+			t.Fatalf("expected error evaluating 'b', but it succeeded instead: %v", bvar)
+		}
+
+		if _, ok := secondCallCheck.checkLocalsAndArgs(p, t); !ok {
+			t.Fatalf("exiting for past errors")
+		}
+	})
+}
+
+func TestInlineStep(t *testing.T) {
+	if ver, _ := goversion.Parse(runtime.Version()); ver.Major >= 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 10, -1, 0, 0, ""}) {
+		// Versions of go before 1.10 do not have DWARF information for inlined calls
+		t.Skip("inlining not supported")
+	}
+	testseq2Args(".", []string{}, protest.EnableInlining, t, "testinline", "", []seqTest{
+		{contContinue, 18},
+		{contStep, 6},
+		{contStep, 7},
+		{contStep, 18},
+		{contStep, 19},
+	})
+}
+
+func TestInlineNext(t *testing.T) {
+	if ver, _ := goversion.Parse(runtime.Version()); ver.Major >= 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 10, -1, 0, 0, ""}) {
+		// Versions of go before 1.10 do not have DWARF information for inlined calls
+		t.Skip("inlining not supported")
+	}
+	testseq2Args(".", []string{}, protest.EnableInlining, t, "testinline", "", []seqTest{
+		{contContinue, 18},
+		{contStep, 6},
+		{contNext, 7},
+		{contNext, 18},
+		{contNext, 19},
+	})
+}
+
+func TestInlineStepOver(t *testing.T) {
+	if ver, _ := goversion.Parse(runtime.Version()); ver.Major >= 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 10, -1, 0, 0, ""}) {
+		// Versions of go before 1.10 do not have DWARF information for inlined calls
+		t.Skip("inlining not supported")
+	}
+	testseq2Args(".", []string{}, protest.EnableInlining, t, "testinline", "", []seqTest{
+		{contContinue, 18},
+		{contNext, 18},
+		{contNext, 19},
+		{contNext, 19},
+		{contNext, 20},
+	})
+}
+
+func TestInlineStepOut(t *testing.T) {
+	if ver, _ := goversion.Parse(runtime.Version()); ver.Major >= 0 && !ver.AfterOrEqual(goversion.GoVersion{1, 10, -1, 0, 0, ""}) {
+		// Versions of go before 1.10 do not have DWARF information for inlined calls
+		t.Skip("inlining not supported")
+	}
+	testseq2Args(".", []string{}, protest.EnableInlining, t, "testinline", "", []seqTest{
+		{contContinue, 18},
+		{contStep, 6},
+		{contStepout, 18},
+	})
+}

pkg/proc/scope_test.go

@@ -92,27 +92,7 @@ func TestScope(t *testing.T) {
 				t.Errorf("unknown stop position %s:%d %#x", bp.File, bp.Line, bp.Addr)
 			}
 
-			scope, err := proc.GoroutineScope(p.CurrentThread())
-			assertNoError(err, t, "GoroutineScope()")
-
-			args, err := scope.FunctionArguments(normalLoadConfig)
-			assertNoError(err, t, "FunctionArguments()")
-			locals, err := scope.LocalVariables(normalLoadConfig)
-			assertNoError(err, t, "LocalVariables()")
-
-			for _, arg := range args {
-				scopeCheck.checkVar(arg, t)
-			}
-
-			for _, local := range locals {
-				scopeCheck.checkVar(local, t)
-			}
-
-			for i := range scopeCheck.varChecks {
-				if !scopeCheck.varChecks[i].ok {
-					t.Errorf("%d: variable %s not found", scopeCheck.line, scopeCheck.varChecks[i].name)
-				}
-			}
+			scope, _ := scopeCheck.checkLocalsAndArgs(p, t)
 
 			var prev *varCheck
 			for i := range scopeCheck.varChecks {
@@ -127,7 +107,7 @@ func TestScope(t *testing.T) {
 			}
 
 			scopeCheck.ok = true
-			_, err = p.ClearBreakpoint(bp.Addr)
+			_, err := p.ClearBreakpoint(bp.Addr)
 			assertNoError(err, t, "ClearBreakpoint")
 		}
 	})
@@ -254,6 +234,35 @@ func (check *scopeCheck) Parse(descr string, t *testing.T) {
 	}
 }
 
+func (scopeCheck *scopeCheck) checkLocalsAndArgs(p proc.Process, t *testing.T) (*proc.EvalScope, bool) {
+	scope, err := proc.GoroutineScope(p.CurrentThread())
+	assertNoError(err, t, "GoroutineScope()")
+
+	ok := true
+
+	args, err := scope.FunctionArguments(normalLoadConfig)
+	assertNoError(err, t, "FunctionArguments()")
+	locals, err := scope.LocalVariables(normalLoadConfig)
+	assertNoError(err, t, "LocalVariables()")
+
+	for _, arg := range args {
+		scopeCheck.checkVar(arg, t)
+	}
+
+	for _, local := range locals {
+		scopeCheck.checkVar(local, t)
+	}
+
+	for i := range scopeCheck.varChecks {
+		if !scopeCheck.varChecks[i].ok {
+			t.Errorf("%d: variable %s not found", scopeCheck.line, scopeCheck.varChecks[i].name)
+			ok = false
+		}
+	}
+
+	return scope, ok
+}
+
 func (check *scopeCheck) checkVar(v *proc.Variable, t *testing.T) {
 	var varCheck *varCheck
 	for i := range check.varChecks {

pkg/proc/stack.go

@@ -8,17 +8,30 @@ import (
 
 	"github.com/derekparker/delve/pkg/dwarf/frame"
 	"github.com/derekparker/delve/pkg/dwarf/op"
+	"github.com/derekparker/delve/pkg/dwarf/reader"
 )
 
 // This code is partly adapted from runtime.gentraceback in
 // $GOROOT/src/runtime/traceback.go
 
 // Stackframe represents a frame in a system stack.
+//
+// Each stack frame has two locations Current and Call.
+//
+// For the topmost stackframe Current and Call are the same location.
+//
+// For stackframes after the first Current is the location corresponding to
+// the return address and Call is the location of the CALL instruction that
+// was last executed on the frame. Note however that Call.PC is always equal
+// to Current.PC, because finding the correct value for Call.PC would
+// require disassembling each function in the stacktrace.
+//
+// For synthetic stackframes generated for inlined function calls Current.Fn
+// is the function containing the inlining and Call.Fn in the inlined
+// function.
 type Stackframe struct {
-	// Address the function above this one on the call stack will return to.
-	Current Location
-	// Address of the call instruction for the function above on the call stack.
-	Call Location
+	Current, Call Location
+
 	// Frame registers.
 	Regs op.DwarfRegisters
 	// High address of the stack.
@@ -31,6 +44,19 @@ type Stackframe struct {
 	Err error
 	// SystemStack is true if this frame belongs to a system stack.
 	SystemStack bool
+	// Inlined is true if this frame is actually an inlined call.
+	Inlined bool
+
+	// lastpc is a memory address guaranteed to belong to the last instruction
+	// executed in this stack frame.
+	// For the topmost stack frame this will be the same as Current.PC and
+	// Call.PC, for other stack frames it will usually be Current.PC-1, but
+	// could be different when inlined calls are involved in the stacktrace.
+	// Note that this address isn't guaranteed to belong to the start of an
+	// instruction and, for this reason, should not be propagated outside of
+	// pkg/proc.
+	// Use this value to determine active lexical scopes for the stackframe.
+	lastpc uint64
 }
 
 // FrameOffset returns the address of the stack frame, absolute for system
@@ -123,6 +149,8 @@ type stackIterator struct {
 
 	g           *G     // the goroutine being stacktraced, nil if we are stacktracing a goroutine-less thread
 	g0_sched_sp uint64 // value of g0.sched.sp (see comments around its use)
+
+	dwarfReader *dwarf.Reader
 }
 
 type savedLR struct {
@@ -162,7 +190,7 @@ func newStackIterator(bi *BinaryInfo, mem MemoryReadWriter, regs op.DwarfRegiste
 			}
 		}
 	}
-	return &stackIterator{pc: regs.PC(), regs: regs, top: true, bi: bi, mem: mem, err: nil, atend: false, stackhi: stackhi, stackBarrierPC: stackBarrierPC, stkbar: stkbar, systemstack: systemstack, g: g, g0_sched_sp: g0_sched_sp}
+	return &stackIterator{pc: regs.PC(), regs: regs, top: true, bi: bi, mem: mem, err: nil, atend: false, stackhi: stackhi, stackBarrierPC: stackBarrierPC, stkbar: stkbar, systemstack: systemstack, g: g, g0_sched_sp: g0_sched_sp, dwarfReader: bi.dwarf.Reader()}
 }
 
 // Next points the iterator to the next stack frame.
@@ -305,9 +333,8 @@ func (it *stackIterator) Err() error {
 // frameBase calculates the frame base pseudo-register for DWARF for fn and
 // the current frame.
 func (it *stackIterator) frameBase(fn *Function) int64 {
-	rdr := it.bi.dwarf.Reader()
-	rdr.Seek(fn.offset)
-	e, err := rdr.Next()
+	it.dwarfReader.Seek(fn.offset)
+	e, err := it.dwarfReader.Next()
 	if err != nil {
 		return 0
 	}
@@ -327,7 +354,7 @@ func (it *stackIterator) newStackframe(ret, retaddr uint64) Stackframe {
 	} else {
 		it.regs.FrameBase = it.frameBase(fn)
 	}
-	r := Stackframe{Current: Location{PC: it.pc, File: f, Line: l, Fn: fn}, Regs: it.regs, Ret: ret, addrret: retaddr, stackHi: it.stackhi, SystemStack: it.systemstack}
+	r := Stackframe{Current: Location{PC: it.pc, File: f, Line: l, Fn: fn}, Regs: it.regs, Ret: ret, addrret: retaddr, stackHi: it.stackhi, SystemStack: it.systemstack, lastpc: it.pc}
 	if !it.top {
 		fnname := ""
 		if r.Current.Fn != nil {
@@ -339,6 +366,7 @@ func (it *stackIterator) newStackframe(ret, retaddr uint64) Stackframe {
 			// instruction to look for at pc - 1
 			r.Call = r.Current
 		default:
+			r.lastpc = it.pc - 1
 			r.Call.File, r.Call.Line, r.Call.Fn = it.bi.PCToLine(it.pc - 1)
 			if r.Call.Fn == nil {
 				r.Call.File = "?"
@@ -358,7 +386,7 @@ func (it *stackIterator) stacktrace(depth int) ([]Stackframe, error) {
 	}
 	frames := make([]Stackframe, 0, depth+1)
 	for it.Next() {
-		frames = append(frames, it.Frame())
+		frames = it.appendInlineCalls(frames, it.Frame())
 		if len(frames) >= depth+1 {
 			break
 		}
@@ -372,6 +400,60 @@ func (it *stackIterator) stacktrace(depth int) ([]Stackframe, error) {
 	return frames, nil
 }
 
+func (it *stackIterator) appendInlineCalls(frames []Stackframe, frame Stackframe) []Stackframe {
+	if frame.Call.Fn == nil {
+		return append(frames, frame)
+	}
+	if frame.Call.Fn.cu.lineInfo == nil {
+		return append(frames, frame)
+	}
+
+	callpc := frame.Call.PC
+	if len(frames) > 0 {
+		callpc--
+	}
+
+	irdr := reader.InlineStack(it.bi.dwarf, frame.Call.Fn.offset, callpc)
+	for irdr.Next() {
+		entry, offset := reader.LoadAbstractOrigin(irdr.Entry(), it.dwarfReader)
+
+		fnname, okname := entry.Val(dwarf.AttrName).(string)
+		fileidx, okfileidx := entry.Val(dwarf.AttrCallFile).(int64)
+		line, okline := entry.Val(dwarf.AttrCallLine).(int64)
+
+		if !okname || !okfileidx || !okline {
+			break
+		}
+		if fileidx-1 < 0 || fileidx-1 >= int64(len(frame.Current.Fn.cu.lineInfo.FileNames)) {
+			break
+		}
+
+		inlfn := &Function{Name: fnname, Entry: frame.Call.Fn.Entry, End: frame.Call.Fn.End, offset: offset, cu: frame.Call.Fn.cu}
+		frames = append(frames, Stackframe{
+			Current: frame.Current,
+			Call: Location{
+				frame.Call.PC,
+				frame.Call.File,
+				frame.Call.Line,
+				inlfn,
+			},
+			Regs:        frame.Regs,
+			stackHi:     frame.stackHi,
+			Ret:         frame.Ret,
+			addrret:     frame.addrret,
+			Err:         frame.Err,
+			SystemStack: frame.SystemStack,
+			Inlined:     true,
+			lastpc:      frame.lastpc,
+		})
+
+		frame.Call.File = frame.Current.Fn.cu.lineInfo.FileNames[fileidx-1].Path
+		frame.Call.Line = int(line)
+	}
+
+	return append(frames, frame)
+}
+
 // advanceRegs calculates it.callFrameRegs using it.regs and the frame
 // descriptor entry for the current stack frame.
 // it.regs.CallFrameCFA is updated.

pkg/proc/test/support.go

@@ -48,6 +48,7 @@ type BuildFlags uint32
 const (
 	LinkStrip BuildFlags = 1 << iota
 	EnableCGOOptimization
+	EnableInlining
 )
 
 func BuildFixture(name string, flags BuildFlags) Fixture {
@@ -81,7 +82,11 @@ func BuildFixture(name string, flags BuildFlags) Fixture {
 	if flags&LinkStrip != 0 {
 		buildFlags = append(buildFlags, "-ldflags=-s")
 	}
-	buildFlags = append(buildFlags, "-gcflags=-N -l", "-o", tmpfile)
+	gcflags := "-gcflags=-N -l"
+	if flags&EnableInlining != 0 {
+		gcflags = "-gcflags=-N"
+	}
+	buildFlags = append(buildFlags, gcflags, "-o", tmpfile)
 	if *EnableRace {
 		buildFlags = append(buildFlags, "-race")
 	}

pkg/proc/threads.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/derekparker/delve/pkg/dwarf/godwarf"
+	"github.com/derekparker/delve/pkg/dwarf/reader"
 )
 
 // Thread represents a thread.
@@ -100,7 +101,17 @@ func (err *NoSourceForPCError) Error() string {
 // - a breakpoint on the return address of the function, with a condition
 //   checking that we move to the previous stack frame and stay on the same
 //   goroutine.
-func next(dbp Process, stepInto bool) error {
+//
+// The breakpoint on the return address is *not* set if the current frame is
+// an inlined call. For inlined calls topframe.Current.Fn is the function
+// where the inlining happened and the second set of breakpoints will also
+// cover the "return address".
+//
+// If inlinedStepOut is true this function implements the StepOut operation
+// for an inlined function call. Everything works the same as normal except
+// when removing instructions belonging to inlined calls we also remove all
+// instructions belonging to the current inlined call.
+func next(dbp Process, stepInto, inlinedStepOut bool) error {
 	selg := dbp.SelectedGoroutine()
 	curthread := dbp.CurrentThread()
 	topframe, retframe, err := topframe(selg, curthread)
@@ -112,6 +123,11 @@ func next(dbp Process, stepInto bool) error {
 		return &NoSourceForPCError{topframe.Current.PC}
 	}
 
+	// sanity check
+	if inlinedStepOut && !topframe.Inlined {
+		panic("next called with inlinedStepOut but topframe was not inlined")
+	}
+
 	success := false
 	defer func() {
 		if !success {
@@ -141,14 +157,18 @@ func next(dbp Process, stepInto bool) error {
 	sameFrameCond := andFrameoffCondition(sameGCond, topframe.FrameOffset())
 	var sameOrRetFrameCond ast.Expr
 	if sameGCond != nil {
-		sameOrRetFrameCond = &ast.BinaryExpr{
-			Op: token.LAND,
-			X:  sameGCond,
-			Y: &ast.BinaryExpr{
-				Op: token.LOR,
-				X:  frameoffCondition(topframe.FrameOffset()),
-				Y:  frameoffCondition(retframe.FrameOffset()),
-			},
+		if topframe.Inlined {
+			sameOrRetFrameCond = sameFrameCond
+		} else {
+			sameOrRetFrameCond = &ast.BinaryExpr{
+				Op: token.LAND,
+				X:  sameGCond,
+				Y: &ast.BinaryExpr{
+					Op: token.LOR,
+					X:  frameoffCondition(topframe.FrameOffset()),
+					Y:  frameoffCondition(retframe.FrameOffset()),
+				},
+			}
 		}
 	}
 
@@ -216,6 +236,18 @@ func next(dbp Process, stepInto bool) error {
 		return err
 	}
 
+	if !stepInto {
+		// Removing any PC range belonging to an inlined call
+		frame := topframe
+		if inlinedStepOut {
+			frame = retframe
+		}
+		pcs, err = removeInlinedCalls(dbp, pcs, frame)
+		if err != nil {
+			return err
+		}
+	}
+
 	if !csource {
 		var covered bool
 		for i := range pcs {
@@ -233,7 +265,6 @@ func next(dbp Process, stepInto bool) error {
 		}
 	}
 
-	// Add a breakpoint on the return address for the current frame
 	for _, pc := range pcs {
 		if _, err := dbp.SetBreakpoint(pc, NextBreakpoint, sameFrameCond); err != nil {
 			if _, ok := err.(BreakpointExistsError); !ok {
@@ -243,18 +274,24 @@ func next(dbp Process, stepInto bool) error {
 		}
 
 	}
-	if bp, err := dbp.SetBreakpoint(topframe.Ret, NextBreakpoint, retFrameCond); err != nil {
-		if _, isexists := err.(BreakpointExistsError); isexists {
-			if bp.Kind == NextBreakpoint {
-				// If the return address shares the same address with one of the lines
-				// of the function (because we are stepping through a recursive
-				// function) then the corresponding breakpoint should be active both on
-				// this frame and on the return frame.
-				bp.Cond = sameOrRetFrameCond
+	if !topframe.Inlined {
+		// Add a breakpoint on the return address for the current frame.
+		// For inlined functions there is no need to do this, the set of PCs
+		// returned by the AllPCsBetween call above already cover all instructions
+		// of the containing function.
+		if bp, err := dbp.SetBreakpoint(topframe.Ret, NextBreakpoint, retFrameCond); err != nil {
+			if _, isexists := err.(BreakpointExistsError); isexists {
+				if bp.Kind == NextBreakpoint {
+					// If the return address shares the same address with one of the lines
+					// of the function (because we are stepping through a recursive
+					// function) then the corresponding breakpoint should be active both on
+					// this frame and on the return frame.
+					bp.Cond = sameOrRetFrameCond
+				}
 			}
+			// Return address could be wrong, if we are unable to set a breakpoint
+			// there it's ok.
 		}
-		// Return address could be wrong, if we are unable to set a breakpoint
-		// there it's ok.
 	}
 
 	if bp := curthread.Breakpoint(); bp.Breakpoint == nil {
@@ -264,6 +301,39 @@ func next(dbp Process, stepInto bool) error {
 	return nil
 }
 
+// Removes instructions belonging to inlined calls of topframe from pcs.
+// If includeCurrentFn is true it will also remove all instructions
+// belonging to the current function.
+func removeInlinedCalls(dbp Process, pcs []uint64, topframe Stackframe) ([]uint64, error) {
+	bi := dbp.BinInfo()
+	irdr := reader.InlineStack(bi.dwarf, topframe.Call.Fn.offset, 0)
+	for irdr.Next() {
+		e := irdr.Entry()
+		if e.Offset == topframe.Call.Fn.offset {
+			continue
+		}
+		ranges, err := bi.dwarf.Ranges(e)
+		if err != nil {
+			return pcs, err
+		}
+		for _, rng := range ranges {
+			pcs = removePCsBetween(pcs, rng[0], rng[1])
+		}
+		irdr.SkipChildren()
+	}
+	return pcs, irdr.Err()
+}
+
+func removePCsBetween(pcs []uint64, start, end uint64) []uint64 {
+	out := pcs[:0]
+	for _, pc := range pcs {
+		if pc < start || pc >= end {
+			out = append(out, pc)
+		}
+	}
+	return out
+}
+
 func setStepIntoBreakpoint(dbp Process, text []AsmInstruction, cond ast.Expr) error {
 	if len(text) <= 0 {
 		return nil

pkg/proc/variables.go

@@ -156,13 +156,15 @@ type G struct {
 // EvalScope is the scope for variable evaluation. Contains the thread,
 // current location (PC), and canonical frame address.
 type EvalScope struct {
-	PC      uint64 // Current instruction of the evaluation frame
+	Location
 	Regs    op.DwarfRegisters
 	Mem     MemoryReadWriter // Target's memory
 	Gvar    *Variable
 	BinInfo *BinaryInfo
 
 	frameOffset int64
+
+	aordr *dwarf.Reader // extra reader to load DW_AT_abstract_origin entries, do not initialize
 }
 
 // IsNilErr is returned when a variable is nil.
@@ -175,7 +177,7 @@ func (err *IsNilErr) Error() string {
 }
 
 func globalScope(bi *BinaryInfo, mem MemoryReadWriter) *EvalScope {
-	return &EvalScope{PC: 0, Regs: op.DwarfRegisters{}, Mem: mem, Gvar: nil, BinInfo: bi, frameOffset: 0}
+	return &EvalScope{Location: Location{}, Regs: op.DwarfRegisters{}, Mem: mem, Gvar: nil, BinInfo: bi, frameOffset: 0}
 }
 
 func (scope *EvalScope) newVariable(name string, addr uintptr, dwarfType godwarf.Type, mem MemoryReadWriter) *Variable {
@@ -1873,16 +1875,13 @@ func (v *variablesByDepth) Swap(i int, j int) {
 
 // Fetches all variables of a specific type in the current function scope
 func (scope *EvalScope) variablesByTag(tag dwarf.Tag, cfg *LoadConfig) ([]*Variable, error) {
-	fn := scope.BinInfo.PCToFunc(scope.PC)
-	if fn == nil {
+	if scope.Fn == nil {
 		return nil, errors.New("unable to find function context")
 	}
 
-	_, line, _ := scope.BinInfo.PCToLine(scope.PC)
-
 	var vars []*Variable
 	var depths []int
-	varReader := reader.Variables(scope.BinInfo.dwarf, fn.offset, scope.PC, line, tag == dwarf.TagVariable)
+	varReader := reader.Variables(scope.BinInfo.dwarf, scope.Fn.offset, scope.PC, scope.Line, tag == dwarf.TagVariable)
 	hasScopes := false
 	for varReader.Next() {
 		entry := varReader.Entry()